From f1d58da877a886ac381ff36dbca2ae8a2ab5e298 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 12 Jul 2024 12:10:46 -0300
Subject: [PATCH 001/656] feat: send doc event emails via celery (#7680)

* feat: notify_event_to_subscribers_task

* fix: avoid circular import, handle error

* fix: don't queue task in test mode

* fix: don't even send mail in test mode

* test: separately test signal

* fix: if/else error

* test: better naming

* test: test the new task

* test: better test name

* test: refactor notify email test

* fix: save, not update

* test: restore template coverage
---
 ietf/community/models.py |  11 +++-
 ietf/community/tasks.py  |  15 ++++++
 ietf/community/tests.py  | 109 ++++++++++++++++++++++++++++++++++-----
 3 files changed, 119 insertions(+), 16 deletions(-)
 create mode 100644 ietf/community/tasks.py

diff --git a/ietf/community/models.py b/ietf/community/models.py
index 2e40031768..b1295461d6 100644
--- a/ietf/community/models.py
+++ b/ietf/community/models.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 
+from django.conf import settings
 from django.db import models
 from django.db.models import signals
 from django.urls import reverse as urlreverse
@@ -11,6 +12,9 @@
 from ietf.person.models import Person, Email
 from ietf.utils.models import ForeignKey
 
+from .tasks import notify_event_to_subscribers_task
+
+
 class CommunityList(models.Model):
     person = ForeignKey(Person, blank=True, null=True)
     group = ForeignKey(Group, blank=True, null=True)
@@ -106,8 +110,11 @@ def notify_events(sender, instance, **kwargs):
     if getattr(instance, "skip_community_list_notification", False):
         return
 
-    from ietf.community.utils import notify_event_to_subscribers
-    notify_event_to_subscribers(instance)
+    # kludge alert: queuing a celery task in response to a signal can cause unexpected attempts to
+    # start a Celery task during tests. To prevent this, don't queue a celery task if we're running
+    # tests.
+    if settings.SERVER_MODE != "test":
+        notify_event_to_subscribers_task.delay(event_id=instance.pk)
 
 
 signals.post_save.connect(notify_events)
diff --git a/ietf/community/tasks.py b/ietf/community/tasks.py
new file mode 100644
index 0000000000..763a596495
--- /dev/null
+++ b/ietf/community/tasks.py
@@ -0,0 +1,15 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+from celery import shared_task
+
+from ietf.doc.models import DocEvent 
+from ietf.utils.log import log
+
+
+@shared_task
+def notify_event_to_subscribers_task(event_id):
+    from .utils import notify_event_to_subscribers
+    event = DocEvent.objects.filter(pk=event_id).first()
+    if event is None:
+        log(f"Unable to send subscriber notifications because DocEvent {event_id} was not found")
+    else:
+        notify_event_to_subscribers(event)
diff --git a/ietf/community/tests.py b/ietf/community/tests.py
index 387877887f..d76347b70a 100644
--- a/ietf/community/tests.py
+++ b/ietf/community/tests.py
@@ -2,15 +2,18 @@
 # -*- coding: utf-8 -*-
 
 
+import mock
 from pyquery import PyQuery
 
+from django.test.utils import override_settings
 from django.urls import reverse as urlreverse
 
 import debug                            # pyflakes:ignore
 
 from ietf.community.models import CommunityList, SearchRule, EmailSubscription
 from ietf.community.utils import docs_matching_community_list_rule, community_list_rules_matching_doc
-from ietf.community.utils import reset_name_contains_index_for_rule
+from ietf.community.utils import reset_name_contains_index_for_rule, notify_event_to_subscribers
+from ietf.community.tasks import notify_event_to_subscribers_task
 import ietf.community.views
 from ietf.group.models import Group
 from ietf.group.utils import setup_default_community_list_for_group
@@ -18,8 +21,7 @@
 from ietf.doc.utils import add_state_change_event
 from ietf.person.models import Person, Email, Alias
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized
-from ietf.utils.mail import outbox
-from ietf.doc.factories import WgDraftFactory
+from ietf.doc.factories import DocEventFactory, WgDraftFactory
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.person.factories import PersonFactory, EmailFactory, AliasFactory
 
@@ -423,7 +425,49 @@ def test_subscription_for_group(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
 
-    def test_notification(self):
+    @mock.patch("ietf.community.models.notify_event_to_subscribers_task")
+    def test_notification_signal_receiver(self, mock_notify_task):
+        """Saving a DocEvent should notify subscribers
+        
+        This implicitly tests that notify_events is hooked up to the post_save signal.
+        """
+        # Arbitrary model that's not a DocEvent
+        p = PersonFactory()
+        mock_notify_task.reset_mock()  # clear any calls that resulted from the factories
+        # be careful overriding SERVER_MODE - we do it here because the method
+        # under test does not make this call when in "test" mode
+        with override_settings(SERVER_MODE="not-test"):
+            p.save()
+        self.assertFalse(mock_notify_task.delay.called)
+        
+        d = DocEventFactory()
+        mock_notify_task.reset_mock()  # clear any calls that resulted from the factories
+        # be careful overriding SERVER_MODE - we do it here because the method
+        # under test does not make this call when in "test" mode
+        with override_settings(SERVER_MODE="not-test"):
+            d.save()
+        self.assertEqual(mock_notify_task.delay.call_count, 1)
+        self.assertEqual(mock_notify_task.delay.call_args, mock.call(event_id = d.pk))
+        
+        mock_notify_task.reset_mock()
+        d.skip_community_list_notification = True
+        # be careful overriding SERVER_MODE - we do it here because the method
+        # under test does not make this call when in "test" mode
+        with override_settings(SERVER_MODE="not-test"):
+            d.save()
+        self.assertFalse(mock_notify_task.delay.called)
+
+        del(d.skip_community_list_notification)
+        d.doc.type_id="rfc"  # not "draft"
+        d.doc.save()
+        # be careful overriding SERVER_MODE - we do it here because the method
+        # under test does not make this call when in "test" mode
+        with override_settings(SERVER_MODE="not-test"):
+            d.save()
+        self.assertFalse(mock_notify_task.delay.called)
+
+    @mock.patch("ietf.utils.mail.send_mail_text")
+    def test_notify_event_to_subscribers(self, mock_send_mail_text):
         person = PersonFactory(user__username='plain')
         draft = WgDraftFactory()
 
@@ -431,18 +475,55 @@ def test_notification(self):
         if not draft in clist.added_docs.all():
             clist.added_docs.add(draft)
 
-        EmailSubscription.objects.create(community_list=clist, email=Email.objects.filter(person__user__username="plain").first(), notify_on="significant")
+        sub_to_significant = EmailSubscription.objects.create(
+            community_list=clist,
+            email=Email.objects.filter(person__user__username="plain").first(),
+            notify_on="significant",
+        )
+        sub_to_all = EmailSubscription.objects.create(
+            community_list=clist,
+            email=Email.objects.filter(person__user__username="plain").first(),
+            notify_on="all",
+        )
 
-        mailbox_before = len(outbox)
         active_state = State.objects.get(type="draft", slug="active")
         system = Person.objects.get(name="(System)")
-        add_state_change_event(draft, system, None, active_state)
-        self.assertEqual(len(outbox), mailbox_before)
+        event = add_state_change_event(draft, system, None, active_state)
+        notify_event_to_subscribers(event)
+        self.assertEqual(mock_send_mail_text.call_count, 1)
+        address = mock_send_mail_text.call_args[0][1]
+        subject = mock_send_mail_text.call_args[0][3]
+        content = mock_send_mail_text.call_args[0][4]
+        self.assertEqual(address, sub_to_all.email.address)
+        self.assertIn(draft.name, subject)
+        self.assertIn(clist.long_name(), content)
 
-        mailbox_before = len(outbox)
         rfc_state = State.objects.get(type="draft", slug="rfc")
-        add_state_change_event(draft, system, active_state, rfc_state)
-        self.assertEqual(len(outbox), mailbox_before + 1)
-        self.assertTrue(draft.name in outbox[-1]["Subject"])
-        
-        
+        event = add_state_change_event(draft, system, active_state, rfc_state)
+        mock_send_mail_text.reset_mock()
+        notify_event_to_subscribers(event)
+        self.assertEqual(mock_send_mail_text.call_count, 2)
+        addresses = [call_args[0][1] for call_args in mock_send_mail_text.call_args_list]
+        subjects = {call_args[0][3] for call_args in mock_send_mail_text.call_args_list}
+        contents = {call_args[0][4] for call_args in mock_send_mail_text.call_args_list}
+        self.assertCountEqual(
+            addresses, 
+            [sub_to_significant.email.address, sub_to_all.email.address],
+        )
+        self.assertEqual(len(subjects), 1)
+        self.assertIn(draft.name, subjects.pop())
+        self.assertEqual(len(contents), 1)
+        self.assertIn(clist.long_name(), contents.pop())
+
+    @mock.patch("ietf.community.utils.notify_event_to_subscribers")
+    def test_notify_event_to_subscribers_task(self, mock_notify):
+        d = DocEventFactory()
+        notify_event_to_subscribers_task(event_id=d.pk)
+        self.assertEqual(mock_notify.call_count, 1)
+        self.assertEqual(mock_notify.call_args, mock.call(d))
+        mock_notify.reset_mock()
+
+        d.delete()
+        notify_event_to_subscribers_task(event_id=d.pk)
+        self.assertFalse(mock_notify.called)
+

From 26ca821f313d8583e47c7a28e31bcc37475a900a Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 15 Jul 2024 12:16:13 -0500
Subject: [PATCH 002/656] Added a note on updating older clones

---
 docker/README.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docker/README.md b/docker/README.md
index 2e4b50d8ff..14fcc38995 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -156,6 +156,11 @@ On Windows:
 docker compose down -v --rmi all
 docker image prune
 ```
+### Updating an older environment
+
+If you already have a clone, such as from a previous codesprint, and are updating that clone, before starting the datatracker from the updated image:
+* rm ietf/settings_local.py   # The startup script will put a new one, appropriate to the current release, in place
+* Execute the `Clean all` sequence above.
 
 ### Accessing PostgreSQL Port
 
@@ -197,4 +202,4 @@ drwxrwxr-x   5  100999  100999    4096 May 25 07:56 client
 (etc...)
 ```
 
-Try uninstalling Docker Desktop and installing Docker Compose manually. The Docker Compose bundled with Docker Desktop is incompatible with our software. See also [Rootless Docker: file ownership changes #3343](https://github.com/lando/lando/issues/3343), [Docker context desktop-linux has container permission issues #75](https://github.com/docker/desktop-linux/issues/75).
\ No newline at end of file
+Try uninstalling Docker Desktop and installing Docker Compose manually. The Docker Compose bundled with Docker Desktop is incompatible with our software. See also [Rootless Docker: file ownership changes #3343](https://github.com/lando/lando/issues/3343), [Docker context desktop-linux has container permission issues #75](https://github.com/docker/desktop-linux/issues/75).

From 392511f78fc2000b79c45a46b946cbd434d133c9 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Mon, 15 Jul 2024 15:03:07 -0400
Subject: [PATCH 003/656] chore: Update dependabot.yml

---
 .github/dependabot.yml | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 3d36676762..17d89f1aab 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -9,14 +9,10 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
   - package-ecosystem: "docker"
     directory: "/docker"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
@@ -27,8 +23,6 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
     groups:
       yarn:
         patterns:
@@ -37,8 +31,6 @@ updates:
     directory: "/playwright"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
     groups:
       npm:
         patterns:
@@ -47,8 +39,6 @@ updates:
     directory: "/dev/coverage-action"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
     groups:
       npm:
         patterns:
@@ -57,8 +47,6 @@ updates:
     directory: "/dev/deploy-to-container"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
     groups:
       npm:
         patterns:
@@ -67,8 +55,6 @@ updates:
     directory: "/dev/diff"
     schedule:
       interval: "weekly"
-    reviewers:
-      - "ngpixel"
     groups:
       npm:
         patterns:

From 475cf2fc8516f8eb796ae20a554819dbc2e6e0b1 Mon Sep 17 00:00:00 2001
From: Rich Salz <rsalz@akamai.com>
Date: Mon, 15 Jul 2024 17:32:28 -0400
Subject: [PATCH 004/656] fix: Ensure called from the docker directory (#7689)

---
 docker/cleanall | 7 ++++++-
 docker/cleandb  | 8 +++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/docker/cleanall b/docker/cleanall
index 91eac1764b..c6104aaef9 100755
--- a/docker/cleanall
+++ b/docker/cleanall
@@ -1,5 +1,11 @@
 #!/bin/bash
 
+if test $(basename $PWD ) != "docker"
+then
+    echo "Run this from the docker directory" 1>&2
+    exit 1
+fi
+
 read -p "Stop and remove all containers, volumes and images for this project? [y/N] " -n 1 -r
 echo 
 if [[ $REPLY =~ ^[Yy]$ ]]
@@ -7,6 +13,5 @@ then
     cd ..
     echo "Shutting down any instance still running and purge images..."
     docker compose down -v --rmi all
-    cd docker
     echo "Done!"
 fi
diff --git a/docker/cleandb b/docker/cleandb
index 322e4639a2..c881503eae 100755
--- a/docker/cleandb
+++ b/docker/cleandb
@@ -1,5 +1,11 @@
 #!/bin/bash
 
+if test $(basename $PWD ) != "docker"
+then
+    echo "Run this from the docker directory" 1>&2
+    exit 1
+fi
+
 cd ..
 echo "Shutting down any instance still running..."
 docker compose down
@@ -9,5 +15,5 @@ docker volume rm -f "${PROJNAME}_postgresdb-data"
 echo "Rebuilding the DB image..."
 docker compose pull db
 docker compose build --no-cache db
-cd docker
+
 echo "Done!"

From c9dab33f8387b944925127eaf435e301ec83dd7a Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Mon, 15 Jul 2024 17:32:47 -0400
Subject: [PATCH 005/656] ci: migrate to using secret instead of configmap
 (#7685)

---
 k8s/README.md                            |  5 ++
 k8s/auth.yaml                            | 58 ++++++++++++------------
 k8s/beat.yaml                            |  7 ++-
 k8s/celery.yaml                          | 54 +++++++++++-----------
 k8s/datatracker.yaml                     | 54 +++++++++++-----------
 k8s/kustomization.yaml                   |  1 -
 k8s/memcached.yaml                       | 34 ++++++++------
 k8s/rabbitmq.yaml                        | 41 +++++++++--------
 k8s/{django-config.yaml => secrets.yaml} | 10 ++--
 9 files changed, 140 insertions(+), 124 deletions(-)
 create mode 100644 k8s/README.md
 rename k8s/{django-config.yaml => secrets.yaml} (93%)

diff --git a/k8s/README.md b/k8s/README.md
new file mode 100644
index 0000000000..73b5978675
--- /dev/null
+++ b/k8s/README.md
@@ -0,0 +1,5 @@
+# Kustomize deployment
+
+## Run locally
+
+The `secrets.yaml` file is provided as a reference only and must be referenced manually in the `kustomization.yaml` file.
\ No newline at end of file
diff --git a/k8s/auth.yaml b/k8s/auth.yaml
index 8aa1d53cbd..bc99af79d3 100644
--- a/k8s/auth.yaml
+++ b/k8s/auth.yaml
@@ -19,34 +19,9 @@ spec:
         runAsNonRoot: true
       containers:
         # -----------------------------------------------------
-        # ScoutAPM Container
-        # -----------------------------------------------------
-        - name: scoutapm
-          image: "scoutapp/scoutapm:version-1.4.0"
-          imagePullPolicy: IfNotPresent
-          # Replace command with one that will shut down on a TERM signal
-          # The ./core-agent start command line is from the scoutapm docker image
-          command:
-            - "sh"
-            - "-c"
-            - >-
-              trap './core-agent shutdown --tcp 0.0.0.0:6590' TERM;
-              ./core-agent start --daemonize false --log-level debug --tcp 0.0.0.0:6590 &
-              wait $!
-          livenessProbe:
-            exec:
-              command:
-                - "sh"
-                - "-c"
-                - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'"
-          securityContext:
-            readOnlyRootFilesystem: true
-            runAsUser: 65534 # "nobody" user by default
-            runAsGroup: 65534  # "nogroup" group by default
-        # -----------------------------------------------------
-        # Datatracker Container
+        # Auth Container
         # -----------------------------------------------------
-        - name: datatracker
+        - name: auth
           image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
           imagePullPolicy: Always
           ports:
@@ -72,8 +47,8 @@ spec:
             - name: "DEPLOY_UID"
               value: "$DEPLOY_UID"
           envFrom:
-            - configMapRef:
-                name: django-config
+            - secretRef:
+                name: dt-secrets-env
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:
@@ -82,6 +57,31 @@ spec:
             readOnlyRootFilesystem: true
             runAsUser: 1000
             runAsGroup: 1000
+        # -----------------------------------------------------
+        # ScoutAPM Container
+        # -----------------------------------------------------
+        - name: scoutapm
+          image: "scoutapp/scoutapm:version-1.4.0"
+          imagePullPolicy: IfNotPresent
+          # Replace command with one that will shut down on a TERM signal
+          # The ./core-agent start command line is from the scoutapm docker image
+          command:
+            - "sh"
+            - "-c"
+            - >-
+              trap './core-agent shutdown --tcp 0.0.0.0:6590' TERM;
+              ./core-agent start --daemonize false --log-level debug --tcp 0.0.0.0:6590 &
+              wait $!
+          livenessProbe:
+            exec:
+              command:
+                - "sh"
+                - "-c"
+                - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'"
+          securityContext:
+            readOnlyRootFilesystem: true
+            runAsUser: 65534 # "nobody" user by default
+            runAsGroup: 65534  # "nogroup" group by default
       volumes:
         # To be overriden with the actual shared volume
         - name: dt-vol
diff --git a/k8s/beat.yaml b/k8s/beat.yaml
index 99317ab77a..72d74e11e4 100644
--- a/k8s/beat.yaml
+++ b/k8s/beat.yaml
@@ -20,6 +20,9 @@ spec:
       securityContext:
         runAsNonRoot: true
       containers:
+        # -----------------------------------------------------
+        # Beat Container
+        # -----------------------------------------------------
         - name: beat
           image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
           imagePullPolicy: Always
@@ -39,8 +42,8 @@ spec:
             - name: "CONTAINER_ROLE"
               value: "beat"
           envFrom:
-            - configMapRef:
-                name: django-config
+            - secretRef:
+                name: dt-secrets-env
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:
diff --git a/k8s/celery.yaml b/k8s/celery.yaml
index dfb20fa40a..10f58f0161 100644
--- a/k8s/celery.yaml
+++ b/k8s/celery.yaml
@@ -21,31 +21,6 @@ spec:
         runAsNonRoot: true
       containers:
         # -----------------------------------------------------
-        # ScoutAPM Container
-        # -----------------------------------------------------
-        - name: scoutapm
-          image: "scoutapp/scoutapm:version-1.4.0"
-          imagePullPolicy: IfNotPresent
-          # Replace command with one that will shut down on a TERM signal
-          # The ./core-agent start command line is from the scoutapm docker image
-          command:
-            - "sh"
-            - "-c"
-            - >-
-              trap './core-agent shutdown --tcp 0.0.0.0:6590' TERM;
-              ./core-agent start --daemonize false --log-level debug --tcp 0.0.0.0:6590 &
-              wait $!
-          livenessProbe:
-            exec:
-              command:
-                - "sh"
-                - "-c"
-                - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'"
-          securityContext:
-            readOnlyRootFilesystem: true
-            runAsUser: 65534 # "nobody" user by default
-            runAsGroup: 65534  # "nogroup" group by default
-        # -----------------------------------------------------
         # Celery Container
         # -----------------------------------------------------
         - name: celery
@@ -71,8 +46,8 @@ spec:
             - name: "CONTAINER_ROLE"
               value: "celery"
           envFrom:
-            - configMapRef:
-                name: django-config
+            - secretRef:
+                name: dt-secrets-env
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:
@@ -81,6 +56,31 @@ spec:
             readOnlyRootFilesystem: true
             runAsUser: 1000
             runAsGroup: 1000
+        # -----------------------------------------------------
+        # ScoutAPM Container
+        # -----------------------------------------------------
+        - name: scoutapm
+          image: "scoutapp/scoutapm:version-1.4.0"
+          imagePullPolicy: IfNotPresent
+          # Replace command with one that will shut down on a TERM signal
+          # The ./core-agent start command line is from the scoutapm docker image
+          command:
+            - "sh"
+            - "-c"
+            - >-
+              trap './core-agent shutdown --tcp 0.0.0.0:6590' TERM;
+              ./core-agent start --daemonize false --log-level debug --tcp 0.0.0.0:6590 &
+              wait $!
+          livenessProbe:
+            exec:
+              command:
+                - "sh"
+                - "-c"
+                - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'"
+          securityContext:
+            readOnlyRootFilesystem: true
+            runAsUser: 65534 # "nobody" user by default
+            runAsGroup: 65534  # "nogroup" group by default
       volumes:
         # To be overriden with the actual shared volume
         - name: dt-vol
diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml
index 5ad4336614..81dc048d0b 100644
--- a/k8s/datatracker.yaml
+++ b/k8s/datatracker.yaml
@@ -19,31 +19,6 @@ spec:
         runAsNonRoot: true
       containers:
         # -----------------------------------------------------
-        # ScoutAPM Container
-        # -----------------------------------------------------
-        - name: scoutapm
-          image: "scoutapp/scoutapm:version-1.4.0"
-          imagePullPolicy: IfNotPresent
-          # Replace command with one that will shut down on a TERM signal
-          # The ./core-agent start command line is from the scoutapm docker image
-          command:
-            - "sh"
-            - "-c"
-            - >-
-              trap './core-agent shutdown --tcp 0.0.0.0:6590' TERM;
-              ./core-agent start --daemonize false --log-level debug --tcp 0.0.0.0:6590 &
-              wait $!
-          livenessProbe:
-            exec:
-              command:
-                - "sh"
-                - "-c"
-                - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'"
-          securityContext:
-            readOnlyRootFilesystem: true
-            runAsUser: 65534 # "nobody" user by default
-            runAsGroup: 65534  # "nogroup" group by default
-        # -----------------------------------------------------
         # Datatracker Container
         # -----------------------------------------------------
         - name: datatracker
@@ -72,8 +47,8 @@ spec:
             - name: "DEPLOY_UID"
               value: "$DEPLOY_UID"
           envFrom:
-            - configMapRef:
-                name: django-config
+            - secretRef:
+                name: dt-secrets-env
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:
@@ -82,6 +57,31 @@ spec:
             readOnlyRootFilesystem: true
             runAsUser: 1000
             runAsGroup: 1000
+        # -----------------------------------------------------
+        # ScoutAPM Container
+        # -----------------------------------------------------
+        - name: scoutapm
+          image: "scoutapp/scoutapm:version-1.4.0"
+          imagePullPolicy: IfNotPresent
+          # Replace command with one that will shut down on a TERM signal
+          # The ./core-agent start command line is from the scoutapm docker image
+          command:
+            - "sh"
+            - "-c"
+            - >-
+              trap './core-agent shutdown --tcp 0.0.0.0:6590' TERM;
+              ./core-agent start --daemonize false --log-level debug --tcp 0.0.0.0:6590 &
+              wait $!
+          livenessProbe:
+            exec:
+              command:
+                - "sh"
+                - "-c"
+                - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'"
+          securityContext:
+            readOnlyRootFilesystem: true
+            runAsUser: 65534 # "nobody" user by default
+            runAsGroup: 65534  # "nogroup" group by default
       initContainers:
         - name: migration
           image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml
index cfc17f35d7..ab381369b2 100644
--- a/k8s/kustomization.yaml
+++ b/k8s/kustomization.yaml
@@ -9,6 +9,5 @@ resources:
   - beat.yaml
   - celery.yaml
   - datatracker.yaml
-  - django-config.yaml
   - memcached.yaml
   - rabbitmq.yaml
diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml
index e94066c9e0..4b362c88c1 100644
--- a/k8s/memcached.yaml
+++ b/k8s/memcached.yaml
@@ -16,12 +16,16 @@ spec:
       securityContext:
         runAsNonRoot: true
       containers:
-        - image: "quay.io/prometheus/memcached-exporter:v0.14.3"
+        # -----------------------------------------------------
+        # Memcached
+        # -----------------------------------------------------
+        - image: "memcached:1.6-alpine"
           imagePullPolicy: IfNotPresent
-          name: memcached-exporter
+          args: ["-m", "1024"]
+          name: memcached
           ports:
-            - name: metrics
-              containerPort: 9150
+            - name: memcached
+              containerPort: 11211
               protocol: TCP
           securityContext:
             allowPrivilegeEscalation: false
@@ -29,15 +33,18 @@ spec:
               drop:
                 - ALL
             readOnlyRootFilesystem: true
-            runAsUser: 65534  # nobody
-            runAsGroup: 65534  # nobody
-        - image: "memcached:1.6-alpine"
+            # memcached image sets up uid/gid 11211
+            runAsUser: 11211
+            runAsGroup: 11211
+        # -----------------------------------------------------
+        # Memcached Exporter for Prometheus
+        # -----------------------------------------------------
+        - image: "quay.io/prometheus/memcached-exporter:v0.14.3"
           imagePullPolicy: IfNotPresent
-          args: ["-m", "1024"]
-          name: memcached
+          name: memcached-exporter
           ports:
-            - name: memcached
-              containerPort: 11211
+            - name: metrics
+              containerPort: 9150
               protocol: TCP
           securityContext:
             allowPrivilegeEscalation: false
@@ -45,9 +52,8 @@ spec:
               drop:
                 - ALL
             readOnlyRootFilesystem: true
-            # memcached image sets up uid/gid 11211
-            runAsUser: 11211
-            runAsGroup: 11211
+            runAsUser: 65534  # nobody
+            runAsGroup: 65534  # nobody
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       terminationGracePeriodSeconds: 30
diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml
index 132ca79ded..b016b3a5ab 100644
--- a/k8s/rabbitmq.yaml
+++ b/k8s/rabbitmq.yaml
@@ -15,23 +15,6 @@ spec:
     spec:
       securityContext:
         runAsNonRoot: true
-      initContainers:
-        # -----------------------------------------------------
-        # Init RabbitMQ data
-        # -----------------------------------------------------
-        - name: init-rabbitmq
-          image: busybox:stable
-          command:
-            - "sh"
-            - "-c"
-            - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq"
-          securityContext:
-            runAsNonRoot: false
-            runAsUser: 0
-            readOnlyRootFilesystem: true
-          volumeMounts:
-            - name: "rabbitmq-data"
-              mountPath: "/mnt"
       containers:
         # -----------------------------------------------------
         # RabbitMQ Container
@@ -52,8 +35,11 @@ spec:
             - name: rabbitmq-config
               mountPath: "/etc/rabbitmq"
           env:
-            - name: "CELERY_PASSWORD"
-              value: "this-is-a-secret"
+            - name: CELERY_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: dt-secrets-env
+                  key: CELERY_PASSWORD
           livenessProbe:
             exec:
               command: ["rabbitmq-diagnostics", "-q", "ping"]
@@ -76,6 +62,23 @@ spec:
             # rabbitmq image sets up uid/gid 100/101
             runAsUser: 100
             runAsGroup: 101
+      initContainers:
+        # -----------------------------------------------------
+        # Init RabbitMQ data
+        # -----------------------------------------------------
+        - name: init-rabbitmq
+          image: busybox:stable
+          command:
+            - "sh"
+            - "-c"
+            - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq"
+          securityContext:
+            runAsNonRoot: false
+            runAsUser: 0
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: "rabbitmq-data"
+              mountPath: "/mnt"
       volumes:
         - name: rabbitmq-tmp
           emptyDir:
diff --git a/k8s/django-config.yaml b/k8s/secrets.yaml
similarity index 93%
rename from k8s/django-config.yaml
rename to k8s/secrets.yaml
index 07e2d710de..4e76a86a53 100644
--- a/k8s/django-config.yaml
+++ b/k8s/secrets.yaml
@@ -1,9 +1,9 @@
 apiVersion: v1
-kind: ConfigMap
+kind: Secret
 metadata:
-  name: django-config
-data:
-    # n.b., these are debug values / non-secret secrets
+  name: secrets-env
+type: Opaque
+stringData:
   DATATRACKER_SERVER_MODE: "development"  # development for staging, production for production
   DATATRACKER_ADMINS: |-
     Robert Sparks <rjsparks@nostrum.com>
@@ -80,4 +80,4 @@ data:
 
   # Scout configuration
   DATATRACKER_SCOUT_KEY: "this-is-the-scout-key"
-  DATATRACKER_SCOUT_NAME: "StagingDatatracker"
+  DATATRACKER_SCOUT_NAME: "StagingDatatracker"
\ No newline at end of file

From 17e0f573b3e0814e833788d4f022d1ce1e81703b Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 15 Jul 2024 16:33:00 -0500
Subject: [PATCH 006/656] fix: check correct state machine when clearing status
 change ballots (#7684)

* fix: check correct state machine when clearing status change ballots

Fixes #7335

* fix: Improve ballot clearing tests

* fix: look at the right state machines for defer state for a ballot

* fix: also do the right thing with conflrev defers
---
 ietf/doc/tests_ballot.py        |  7 +++++-
 ietf/doc/tests_status_change.py | 40 +++++++++++++++++++++++++++++++++
 ietf/doc/views_ballot.py        | 13 ++++++++++-
 3 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py
index e18b2abfd9..034ba6f4b2 100644
--- a/ietf/doc/tests_ballot.py
+++ b/ietf/doc/tests_ballot.py
@@ -806,7 +806,7 @@ def test_clear_ballot(self):
         ballot = create_ballot_if_not_open(None, draft, ad, 'approve')
         old_ballot_id = ballot.id
         draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="iesg-eva")) 
-        url = urlreverse('ietf.doc.views_ballot.clear_ballot', kwargs=dict(name=draft.name,ballot_type_slug=draft.ballot_open('approve').ballot_type.slug))
+        url = urlreverse('ietf.doc.views_ballot.clear_ballot', kwargs=dict(name=draft.name,ballot_type_slug="approve"))
         login_testing_unauthorized(self, "secretary", url)
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
@@ -816,6 +816,11 @@ def test_clear_ballot(self):
         self.assertIsNotNone(ballot)
         self.assertEqual(ballot.ballotpositiondocevent_set.count(),0)
         self.assertNotEqual(old_ballot_id, ballot.id)
+        # It's not valid to clear a ballot of a type where there's no matching state
+        url = urlreverse('ietf.doc.views_ballot.clear_ballot', kwargs=dict(name=draft.name,ballot_type_slug="statchg"))
+        r = self.client.post(url,{})
+        self.assertEqual(r.status_code, 404)
+ 
 
     def test_ballot_downref_approve(self):
         ad = Person.objects.get(name="Areað Irector")
diff --git a/ietf/doc/tests_status_change.py b/ietf/doc/tests_status_change.py
index 229390447c..bec48ed4ef 100644
--- a/ietf/doc/tests_status_change.py
+++ b/ietf/doc/tests_status_change.py
@@ -484,7 +484,47 @@ def verify_relations(doc,target_name,status):
         verify_relations(doc,'rfc9998','tobcp' )
         verify_relations(doc,'rfc14'  ,'tohist')
         self.assertTrue(doc.latest_event(DocEvent,type="added_comment").desc.startswith('Affected RFC list changed.'))       
+
+    def test_clear_ballot(self):
+        doc = Document.objects.get(name='status-change-imaginary-mid-review')
+        url = urlreverse('ietf.doc.views_ballot.clear_ballot',kwargs=dict(name=doc.name, ballot_type_slug="statchg"))
+        login_testing_unauthorized(self, "secretary", url)
+        
+        # Some additional setup
+        doc.relateddocument_set.create(target=Document.objects.get(name='rfc9999'),relationship_id='tois')
+        doc.relateddocument_set.create(target=Document.objects.get(name='rfc9998'),relationship_id='tohist')
+        create_ballot_if_not_open(None, doc, Person.objects.get(user__username="secretary"), "statchg")
+        doc.set_state(State.objects.get(slug='iesgeval',type='statchg'))
+        old_ballot = doc.ballot_open("statchg")
+        self.assertIsNotNone(old_ballot)
+        
+        r = self.client.post(url, dict())
+        self.assertEqual(r.status_code,302)
+        new_ballot = doc.ballot_open("statchg")
+        self.assertIsNotNone(new_ballot)
+        self.assertNotEqual(new_ballot, old_ballot)
+        self.assertEqual(doc.get_state_slug("statchg"),"iesgeval")
+
+    def test_clear_deferred_ballot(self):
+        doc = Document.objects.get(name='status-change-imaginary-mid-review')
+        url = urlreverse('ietf.doc.views_ballot.clear_ballot',kwargs=dict(name=doc.name, ballot_type_slug="statchg"))
+        login_testing_unauthorized(self, "secretary", url)
+        
+        # Some additional setup
+        doc.relateddocument_set.create(target=Document.objects.get(name='rfc9999'),relationship_id='tois')
+        doc.relateddocument_set.create(target=Document.objects.get(name='rfc9998'),relationship_id='tohist')
+        create_ballot_if_not_open(None, doc, Person.objects.get(user__username="secretary"), "statchg")
+        doc.set_state(State.objects.get(slug='defer',type='statchg'))
+        old_ballot = doc.ballot_open("statchg")
+        self.assertIsNotNone(old_ballot)
         
+        r = self.client.post(url, dict())
+        self.assertEqual(r.status_code,302)
+        new_ballot = doc.ballot_open("statchg")
+        self.assertIsNotNone(new_ballot)
+        self.assertNotEqual(new_ballot, old_ballot)
+        self.assertEqual(doc.get_state_slug("statchg"),"iesgeval")
+
     def setUp(self):
         super().setUp()
         IndividualRfcFactory(rfc_number=14,std_level_id='unkn') # draft was never issued
diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py
index 02b55249d6..83ccb07be6 100644
--- a/ietf/doc/views_ballot.py
+++ b/ietf/doc/views_ballot.py
@@ -399,11 +399,22 @@ def send_ballot_comment(request, name, ballot_id):
 def clear_ballot(request, name, ballot_type_slug):
     """Clear all positions and discusses on every open ballot for a document."""
     doc = get_object_or_404(Document, name=name)
+    # If there's no appropriate ballot type state, clearing would be an invalid action.
+    # This will need to be updated if we ever allow defering IRTF ballots
+    if ballot_type_slug == "approve":
+        state_machine = "draft-iesg"
+    elif ballot_type_slug in ["statchg","conflrev"]:
+        state_machine = ballot_type_slug
+    else:
+        state_machine = None
+    state_slug = state_machine and doc.get_state_slug(state_machine)
+    if state_machine is None or state_slug is None:
+        raise Http404
     if request.method == 'POST':
         by = request.user.person
         if close_ballot(doc, by, ballot_type_slug):
             create_ballot_if_not_open(request, doc, by, ballot_type_slug)
-        if doc.get_state('draft-iesg').slug == 'defer':
+        if state_slug == "defer":
             do_undefer_ballot(request,doc)
         return redirect("ietf.doc.views_doc.document_main", name=doc.name)
 

From 18bb793b2d3cf6e5e4df56eed70577215bb46b25 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 15 Jul 2024 18:45:51 -0300
Subject: [PATCH 007/656] feat: add nginx, robots.txt, HTTP headers (#7683)

* feat: nginx + robots.txt

* feat: minimal /health/ endpoint

* ci: startupProbe for datatracker pod

* ci: probe auth pod; set timeoutSeconds

* feat: add CSP and other headers to nginx

* fix: typo in nginx.conf

* feat: split auth/dt nginx confs

* test: test health endpoint

* ci: auth service on port 80

We'll remove http-old (8080) in the future.

* ci: rename auth container/nginx cfg
---
 ietf/urls.py               |  2 ++
 ietf/utils/tests.py        |  9 ++++++++
 k8s/auth.yaml              | 43 +++++++++++++++++++++++++++++++++-----
 k8s/datatracker.yaml       | 37 ++++++++++++++++++++++++++++----
 k8s/kustomization.yaml     |  2 ++
 k8s/nginx-auth.conf        | 34 ++++++++++++++++++++++++++++++
 k8s/nginx-datatracker.conf | 23 ++++++++++++++++++++
 7 files changed, 141 insertions(+), 9 deletions(-)
 create mode 100644 k8s/nginx-auth.conf
 create mode 100644 k8s/nginx-datatracker.conf

diff --git a/ietf/urls.py b/ietf/urls.py
index 78dbb861b2..4b29a3aa81 100644
--- a/ietf/urls.py
+++ b/ietf/urls.py
@@ -5,6 +5,7 @@
 from django.contrib import admin
 from django.contrib.sitemaps import views as sitemap_views
 from django.contrib.staticfiles.urls import staticfiles_urlpatterns
+from django.http import HttpResponse
 from django.urls import include, path
 from django.views import static as static_view
 from django.views.generic import TemplateView
@@ -35,6 +36,7 @@
 
 urlpatterns = [
     url(r'^$', views_search.frontpage),
+    url(r'^health/', lambda _: HttpResponse()),
     url(r'^accounts/', include('ietf.ietfauth.urls')),
     url(r'^admin/', admin.site.urls),
     url(r'^admin/docs/', include('django.contrib.admindocs.urls')),
diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py
index 08adefc826..476c257a38 100644
--- a/ietf/utils/tests.py
+++ b/ietf/utils/tests.py
@@ -679,3 +679,12 @@ class TestForm(Form):
         self.assertTrue(changed_form.has_changed())
         unchanged_form = TestForm(initial={'test_field': [1]}, data={'test_field': [1]})
         self.assertFalse(unchanged_form.has_changed())
+
+
+class HealthTests(TestCase):
+    def test_health(self):
+        self.assertEqual(
+            self.client.get("/health/").status_code,
+            200,
+        )
+            
diff --git a/k8s/auth.yaml b/k8s/auth.yaml
index bc99af79d3..66627ed450 100644
--- a/k8s/auth.yaml
+++ b/k8s/auth.yaml
@@ -24,10 +24,6 @@ spec:
         - name: auth
           image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
           imagePullPolicy: Always
-          ports:
-            - containerPort: 8000
-              name: http
-              protocol: TCP
           volumeMounts:
             - name: dt-vol
               mountPath: /a
@@ -49,6 +45,14 @@ spec:
           envFrom:
             - secretRef:
                 name: dt-secrets-env
+          startupProbe:
+            httpGet:
+              port: 8000
+              path: /health/
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            failureThreshold: 30
+            timeoutSeconds: 3
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:
@@ -58,6 +62,28 @@ spec:
             runAsUser: 1000
             runAsGroup: 1000
         # -----------------------------------------------------
+        # Nginx Container
+        # -----------------------------------------------------
+        - name: nginx
+          image: "ghcr.io/nginxinc/nginx-unprivileged:1.27"
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              port: 8080
+              path: /health/nginx
+          securityContext:
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: nginx-tmp
+              mountPath: /tmp
+            - name: dt-cfg
+              mountPath: /etc/nginx/conf.d/auth.conf
+              subPath: nginx-auth.conf
+        # -----------------------------------------------------
         # ScoutAPM Container
         # -----------------------------------------------------
         - name: scoutapm
@@ -97,6 +123,9 @@ spec:
         - name: dt-cfg
           configMap:
             name: files-cfgmap
+        - name: nginx-tmp
+          emptyDir:
+            sizeLimit: "500Mi"
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       terminationGracePeriodSeconds: 60
@@ -108,9 +137,13 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 8080
+    - port: 80
       targetPort: http
       protocol: TCP
       name: http
+    - port: 8080
+      targetPort: http
+      protocol: TCP
+      name: http-old
   selector:
     app: auth
diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml
index 81dc048d0b..59a35fbbc6 100644
--- a/k8s/datatracker.yaml
+++ b/k8s/datatracker.yaml
@@ -24,10 +24,6 @@ spec:
         - name: datatracker
           image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
           imagePullPolicy: Always
-          ports:
-            - containerPort: 8000
-              name: http
-              protocol: TCP
           volumeMounts:
             - name: dt-vol
               mountPath: /a
@@ -49,6 +45,14 @@ spec:
           envFrom:
             - secretRef:
                 name: dt-secrets-env
+          startupProbe:
+            httpGet:
+              port: 8000
+              path: /health/
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            failureThreshold: 30
+            timeoutSeconds: 3
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:
@@ -58,6 +62,28 @@ spec:
             runAsUser: 1000
             runAsGroup: 1000
         # -----------------------------------------------------
+        # Nginx Container
+        # -----------------------------------------------------
+        - name: nginx
+          image: "ghcr.io/nginxinc/nginx-unprivileged:1.27"
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              port: 8080
+              path: /health/nginx
+          securityContext:
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: nginx-tmp
+              mountPath: /tmp
+            - name: dt-cfg
+              mountPath: /etc/nginx/conf.d/datatracker.conf
+              subPath: nginx-datatracker.conf
+        # -----------------------------------------------------
         # ScoutAPM Container
         # -----------------------------------------------------
         - name: scoutapm
@@ -126,6 +152,9 @@ spec:
         - name: dt-cfg
           configMap:
             name: files-cfgmap
+        - name: nginx-tmp
+          emptyDir:
+            sizeLimit: "500Mi"
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       terminationGracePeriodSeconds: 60
diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml
index ab381369b2..ba8b8a5826 100644
--- a/k8s/kustomization.yaml
+++ b/k8s/kustomization.yaml
@@ -3,6 +3,8 @@ namePrefix: dt-
 configMapGenerator:
   - name: files-cfgmap
     files:
+      - nginx-auth.conf
+      - nginx-datatracker.conf
       - settings_local.py
 resources:
   - auth.yaml
diff --git a/k8s/nginx-auth.conf b/k8s/nginx-auth.conf
new file mode 100644
index 0000000000..4cbc8a0a51
--- /dev/null
+++ b/k8s/nginx-auth.conf
@@ -0,0 +1,34 @@
+server {
+    listen 8080 default_server;
+    server_name _;
+
+    # Note that regex location matches take priority over non-regex "prefix" matches. Use regexes so that
+    # our deny all rule does not squelch the other locations. 
+    location ~ ^/health/nginx$ {
+        return 200;
+    }
+
+    location ~ ^/robots.txt$ {
+        add_header Content-Type text/plain;
+        return 200 "User-agent: *\nDisallow: /\n";
+    }
+
+    location ~ ^/accounts/create.* {
+        return 302 https://datatracker.ietf.org/accounts/create;
+    }
+
+    # n.b. (?!...) is a negative lookahead group
+    location ~ ^(/(?!(api/openid/|accounts/login/|accounts/logout/|accounts/reset/|person/.*/photo|group/groupmenu.json)).*) {
+        deny all;
+    }
+
+    location / {
+        add_header Content-Security-Policy "default-src 'self' 'unsafe-inline' data: https://datatracker.ietf.org/ https://www.ietf.org/ http://ietf.org/ https://analytics.ietf.org https://static.ietf.org; frame-ancestors 'self' ietf.org *.ietf.org meetecho.com *.meetecho.com gather.town *.gather.town";
+        proxy_set_header Host $${keepempty}host;
+        proxy_set_header Connection close;
+        proxy_set_header X-Request-Start "t=${msec}";
+        proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
+        proxy_set_header X-Real-IP $${keepempty}remote_addr;
+        proxy_pass http://localhost:8000;
+    }
+}
diff --git a/k8s/nginx-datatracker.conf b/k8s/nginx-datatracker.conf
new file mode 100644
index 0000000000..63c985463c
--- /dev/null
+++ b/k8s/nginx-datatracker.conf
@@ -0,0 +1,23 @@
+server {
+    listen 8080 default_server;
+    server_name _;
+
+    location /health/nginx {
+        return 200;
+    }
+
+    location /robots.txt {
+        add_header Content-Type text/plain;
+        return 200 "User-agent: *\nDisallow: /doc/pdf/\n";
+    }
+
+    location / {
+        add_header Content-Security-Policy "default-src 'self' 'unsafe-inline' data: https://datatracker.ietf.org/ https://www.ietf.org/ http://ietf.org/ https://analytics.ietf.org https://static.ietf.org; frame-ancestors 'self' ietf.org *.ietf.org meetecho.com *.meetecho.com";
+        proxy_set_header Host $${keepempty}host;
+        proxy_set_header Connection close;
+        proxy_set_header X-Request-Start "t=${msec}";
+        proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
+        proxy_set_header X-Real-IP $${keepempty}remote_addr;
+        proxy_pass http://localhost:8000;
+    }
+}

From 30510d97d8f0a34e2a683c4b0582e99d5990b541 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 15 Jul 2024 19:57:25 -0300
Subject: [PATCH 008/656] ci: fix migration container secrets (#7694)

---
 k8s/datatracker.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml
index 59a35fbbc6..9e1ead1a90 100644
--- a/k8s/datatracker.yaml
+++ b/k8s/datatracker.yaml
@@ -115,8 +115,8 @@ spec:
             - name: "CONTAINER_ROLE"
               value: "migrations"
           envFrom:
-            - configMapRef:
-                name: django-config
+            - secretRef:
+                name: dt-secrets-env
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:

From b00dfd3c9909cf6d1be5868fcf4b56d672b6fd04 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <Matthew Holloway>
Date: Wed, 17 Jul 2024 12:31:00 +1200
Subject: [PATCH 009/656] feat: Overflow shadows

---
 ietf/static/css/ietf.scss | 17 +++++++++++++++++
 ietf/static/js/ietf.js    | 25 ++++++++++++++++++++++++-
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss
index 062358c0e2..5bd520f041 100644
--- a/ietf/static/css/ietf.scss
+++ b/ietf/static/css/ietf.scss
@@ -1188,3 +1188,20 @@ blockquote {
     padding-left: 1rem;
     border-left: solid 1px var(--bs-body-color);
 }
+
+.overflow-shadows {
+    transition: box-shadow 0.5s;
+}
+
+.overflow-shadows--both {
+    box-shadow: inset 0px 21px 18px -20px var(--bs-body-color),
+                inset 0px -21px 18px -20px var(--bs-body-color);
+}
+
+.overflow-shadows--top-only {
+    box-shadow: inset 0px 21px 18px -20px var(--bs-body-color);
+}
+
+.overflow-shadows--bottom-only {
+    box-shadow: inset 0px -21px 18px -20px var(--bs-body-color);
+}
diff --git a/ietf/static/js/ietf.js b/ietf/static/js/ietf.js
index 74fd39a85f..dde00c6d1e 100644
--- a/ietf/static/js/ietf.js
+++ b/ietf/static/js/ietf.js
@@ -91,6 +91,27 @@ $(document)
         //     });
     });
 
+function overflowShadows(el) {
+    function handleScroll(){
+        const canScrollUp = el.scrollTop > 0
+        const canScrollDown = el.offsetHeight + el.scrollTop < el.scrollHeight
+        el.classList.toggle("overflow-shadows--both", canScrollUp && canScrollDown)
+        el.classList.toggle("overflow-shadows--top-only", canScrollUp && !canScrollDown)
+        el.classList.toggle("overflow-shadows--bottom-only", !canScrollUp && canScrollDown)
+    }
+
+    el.addEventListener("scroll", handleScroll, {passive: true})
+    handleScroll()
+
+    const observer = new IntersectionObserver(handleScroll)
+    observer.observe(el) // el won't have scrollTop etc when hidden, so we need to recalculate when it's revealed
+
+    return () => {
+        el.removeEventListener("scroll", handleScroll)
+        observer.unobserve(el)
+    }
+}
+
 $(document)
     .ready(function () {
         // load data for the menu
@@ -108,7 +129,7 @@ $(document)
                     }
                     attachTo.find(".dropdown-menu")
                         .remove();
-                    var menu = ['<ul class="dropdown-menu ms-n1 mt-n1">'];
+                    var menu = ['<ul class="dropdown-menu ms-n1 mt-n1 overflow-shadows">'];
                     var groups = data[parentId];
                     var gtype = "";
                     for (var i = 0; i < groups.length; ++i) {
@@ -127,6 +148,8 @@ $(document)
                         attachTo.closest(".dropdown-menu");
                     }
                     attachTo.append(menu.join(""));
+
+                    attachTo.find(".overflow-shadows").each(function(){ overflowShadows(this)})
                 }
             }
         });

From d0976ec7aba8a44a30955d6322abde79d996abbf Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 17 Jul 2024 21:54:32 -0400
Subject: [PATCH 010/656] ci: add concurrency group to deploy workflow

---
 .github/workflows/build.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index afb5951aca..8f88c7c22e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -47,6 +47,10 @@ on:
         required: true
         type: boolean
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   # -----------------------------------------------------------------
   # PREPARE

From 02baeef01fe37275cde0d3d90e3ca815408e8832 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 17 Jul 2024 22:00:56 -0400
Subject: [PATCH 011/656] docs: Update README.md postgres badge version

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index af059f972b..133d08f5e7 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@
 [![Python Version](https://img.shields.io/badge/python-3.9-blue?logo=python&logoColor=white)](#prerequisites)
 [![Django Version](https://img.shields.io/badge/django-4.x-51be95?logo=django&logoColor=white)](#prerequisites)
 [![Node Version](https://img.shields.io/badge/node.js-16.x-green?logo=node.js&logoColor=white)](#prerequisites)
-[![MariaDB Version](https://img.shields.io/badge/postgres-14-blue?logo=postgresql&logoColor=white)](#prerequisites)
+[![MariaDB Version](https://img.shields.io/badge/postgres-16-blue?logo=postgresql&logoColor=white)](#prerequisites)
 
 ##### The day-to-day front-end to the IETF database for people who work on IETF standards.
 

From 0555eef13a105dbfd3cd9e19f93d6ce1fd7a41c7 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Sat, 20 Jul 2024 12:47:43 -0700
Subject: [PATCH 012/656] fix: prevent error when no file selected

---
 ietf/meeting/views.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 2948a2e715..253f2852ff 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -2790,7 +2790,8 @@ def clean_content(self):
     def clean_file(self):
         submission_method = self.cleaned_data.get("submission_method")
         if submission_method == "upload":
-            return super().clean_file()
+            if self.cleaned_data.get("file", None) is not None:
+                return super().clean_file()
         return None
 
     def clean(self):

From c7f6bdef0a9427f479b0db806423e8d2d3782bb5 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Sat, 20 Jul 2024 13:02:32 -0700
Subject: [PATCH 013/656] test: test missing file

---
 ietf/meeting/tests_views.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index d783ed9c75..db62fe6204 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -6246,6 +6246,12 @@ def test_upload_minutes_agenda(self):
             q = PyQuery(r.content)
             self.assertTrue(q('form input[type="checkbox"]'))
     
+            # test not submitting a file
+            r = self.client.post(url, dict(submission_method="upload"))
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            self.assertTrue(q("form .is-invalid"))
+    
             test_file = BytesIO(b'this is some text for a test')
             test_file.name = "not_really.json"
             r = self.client.post(url,dict(submission_method="upload",file=test_file))

From a86c62e133008e5fe2a5cce3b99a7329b53462b6 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Sat, 20 Jul 2024 15:17:43 -0700
Subject: [PATCH 014/656] fix: raw_id_fields for EventAdmin classes (#7711)

* fix: raw_id_fields for ReviewRequestDocEventAdmin

* fix: raw_id_fields for ReviewAssignmentDocEventAdmin

* fix: raw_id_fields for AddedMessageEventAdmin

* fix: raw_id_fields for SubmissionDocEventAdmin

* refactor: make inheritance clearer
---
 ietf/doc/admin.py | 44 +++++++++++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py
index 3ad4bee2af..301d32d7cc 100644
--- a/ietf/doc/admin.py
+++ b/ietf/doc/admin.py
@@ -142,6 +142,13 @@ class DocumentActionHolderAdmin(admin.ModelAdmin):
 
 # events
 
+class DeletedEventAdmin(admin.ModelAdmin):
+    list_display = ['id', 'content_type', 'json', 'by', 'time']
+    list_filter = ['time']
+    raw_id_fields = ['content_type', 'by']
+admin.site.register(DeletedEvent, DeletedEventAdmin)
+
+
 class DocEventAdmin(admin.ModelAdmin):
     def event_type(self, obj):
         return str(obj.type)
@@ -159,39 +166,42 @@ def short_desc(self, obj):
 admin.site.register(StateDocEvent, DocEventAdmin)
 admin.site.register(ConsensusDocEvent, DocEventAdmin)
 admin.site.register(BallotDocEvent, DocEventAdmin)
+admin.site.register(IRSGBallotDocEvent, DocEventAdmin)
 admin.site.register(WriteupDocEvent, DocEventAdmin)
 admin.site.register(LastCallDocEvent, DocEventAdmin)
 admin.site.register(TelechatDocEvent, DocEventAdmin)
-admin.site.register(ReviewRequestDocEvent, DocEventAdmin)
-admin.site.register(ReviewAssignmentDocEvent, DocEventAdmin)
 admin.site.register(InitialReviewDocEvent, DocEventAdmin)
-admin.site.register(AddedMessageEvent, DocEventAdmin)
-admin.site.register(SubmissionDocEvent, DocEventAdmin)
 admin.site.register(EditedAuthorsDocEvent, DocEventAdmin)
 admin.site.register(IanaExpertDocEvent, DocEventAdmin)
 
-class DeletedEventAdmin(admin.ModelAdmin):
-    list_display = ['id', 'content_type', 'json', 'by', 'time']
-    list_filter = ['time']
-    raw_id_fields = ['content_type', 'by']
-admin.site.register(DeletedEvent, DeletedEventAdmin)
-
 class BallotPositionDocEventAdmin(DocEventAdmin):
-    raw_id_fields = ["doc", "by", "balloter", "ballot"]
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["balloter", "ballot"]
 admin.site.register(BallotPositionDocEvent, BallotPositionDocEventAdmin)
- 
-class IRSGBallotDocEventAdmin(DocEventAdmin):
-    raw_id_fields = ["doc", "by"]
-admin.site.register(IRSGBallotDocEvent, IRSGBallotDocEventAdmin)
 
 class BofreqEditorDocEventAdmin(DocEventAdmin):
-    raw_id_fields = ["doc", "by", "editors" ]
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["editors"]
 admin.site.register(BofreqEditorDocEvent, BofreqEditorDocEventAdmin)
     
 class BofreqResponsibleDocEventAdmin(DocEventAdmin):
-    raw_id_fields = ["doc", "by", "responsible" ]
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["responsible"]
 admin.site.register(BofreqResponsibleDocEvent, BofreqResponsibleDocEventAdmin)
     
+class ReviewRequestDocEventAdmin(DocEventAdmin):
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["review_request"]
+admin.site.register(ReviewRequestDocEvent, ReviewRequestDocEventAdmin)
+
+class ReviewAssignmentDocEventAdmin(DocEventAdmin):
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["review_assignment"]
+admin.site.register(ReviewAssignmentDocEvent, ReviewAssignmentDocEventAdmin)
+
+class AddedMessageEventAdmin(DocEventAdmin):
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["message"]
+admin.site.register(AddedMessageEvent, AddedMessageEventAdmin)
+
+class SubmissionDocEventAdmin(DocEventAdmin):
+    raw_id_fields = DocEventAdmin.raw_id_fields + ["submission"]
+admin.site.register(SubmissionDocEvent, SubmissionDocEventAdmin)
+
 class DocumentUrlAdmin(admin.ModelAdmin):
     list_display = ['id', 'doc', 'tag', 'url', 'desc', ]
     search_fields = ['doc__name', 'url', ]

From 97837569928580fdbe4a1f40c24e2554396f16ea Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Sat, 20 Jul 2024 17:23:32 -0500
Subject: [PATCH 015/656] Changed milestones to use RFC number if draft is
 published as RFC. (#7718)

Co-authored-by: Tero Kivinen <kivinen@iki.fi>
---
 ietf/templates/group/milestones.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/templates/group/milestones.html b/ietf/templates/group/milestones.html
index 1f259743fe..df03d370fd 100644
--- a/ietf/templates/group/milestones.html
+++ b/ietf/templates/group/milestones.html
@@ -49,7 +49,7 @@ <h2 class="my-3">
                     <td>{{ milestone.desc|urlize_ietf_docs }}</td>
                     <td>
                         {% for d in milestone.docs.all %}
-                            <a href="{% url "ietf.doc.views_doc.document_main" name=d.name %}">{{ d.name }}</a>
+                            <a href="{% url "ietf.doc.views_doc.document_main" name=d.name %}">{% if d.became_rfc %}{{ d.became_rfc }} (was {% endif %}{{ d.name }}{% if d.became_rfc %}){% endif %}</a>
                             <br>
                         {% endfor %}
                     </td>

From 60a3976c3df04d658e63f391bcc2c367cc340dbc Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Sat, 20 Jul 2024 17:23:56 -0500
Subject: [PATCH 016/656] Fixed 'to' to 'review_to' and 'cc' to 'review_cc'.
 (#7710)

Co-authored-by: Tero Kivinen <kivinen@iki.fi>
---
 ietf/templates/doc/review/complete_review.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/templates/doc/review/complete_review.html b/ietf/templates/doc/review/complete_review.html
index 2319c40fd0..f30c641e46 100644
--- a/ietf/templates/doc/review/complete_review.html
+++ b/ietf/templates/doc/review/complete_review.html
@@ -66,7 +66,7 @@ <h1>
         </p>
         <p>
             If you enter the review below, the review will be sent
-            to {% for addr in to %}{{ addr|linkify }}{% if not forloop.last %}, {% endif %}{% endfor %}{% if review_cc %}, with a CC to {% for addr in cc %}{{ addr|linkify }}{% if not forloop.last %}, {% endif %}{% endfor %}{% endif %}.
+            to {% for addr in review_to %}{{ addr|linkify }}{% if not forloop.last %}, {% endif %}{% endfor %}{% if review_cc %}, with a CC to {% for addr in review_cc %}{{ addr|linkify }}{% if not forloop.last %}, {% endif %}{% endfor %}{% endif %}.
         </p>
     {% elif assignment %}
         <p>

From c5ca0ea40575d93c0963480c410c2c278e9fa068 Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Sat, 20 Jul 2024 15:24:14 -0700
Subject: [PATCH 017/656] fix: force choice of From address in Announcement
 form. Fixes #7679. (#7720)

---
 ietf/secr/announcement/forms.py | 7 +++++--
 ietf/secr/announcement/tests.py | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/ietf/secr/announcement/forms.py b/ietf/secr/announcement/forms.py
index 3aacbfe622..3fe58bdaaa 100644
--- a/ietf/secr/announcement/forms.py
+++ b/ietf/secr/announcement/forms.py
@@ -42,8 +42,11 @@ def get_from_choices(user):
     nomcom_choices = get_nomcom_choices(user)
     if nomcom_choices:
         addresses = list(addresses) + nomcom_choices
-    
-    return list(zip(addresses, addresses))
+
+    choices = list(zip(addresses, addresses))
+    if len(choices) > 1:
+        choices.insert(0, ('', '(Choose an option)'))
+    return choices
 
 
 def get_nomcom_choices(user):
diff --git a/ietf/secr/announcement/tests.py b/ietf/secr/announcement/tests.py
index c50e997f97..c147c301b6 100644
--- a/ietf/secr/announcement/tests.py
+++ b/ietf/secr/announcement/tests.py
@@ -48,7 +48,7 @@ def test_main_announce_from(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('#id_frm option')),3)
+        self.assertEqual(len(q('#id_frm option')),4)
 
         # IAB Chair
         self.client.login(username="iab-chair", password="iab-chair+password")

From 363c01e711495cdfe8338c1c1de90255c7083243 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 21 Jul 2024 03:14:40 +0300
Subject: [PATCH 018/656] fix: Explicitly set `executable_path` for Selenium
 (#7715)

* fix: Explicitly set `executable_path` for Selenium

So it finds `geckodriver` again.

* Minimize diff

* fix: use existing executable_name

Co-authored-by: Lars Eggert <lars@eggert.org>

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/utils/jstest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/utils/jstest.py b/ietf/utils/jstest.py
index 07d6ed9dd0..157f97912b 100644
--- a/ietf/utils/jstest.py
+++ b/ietf/utils/jstest.py
@@ -30,7 +30,7 @@
     print("     "+skip_message)
 
 def start_web_driver():
-    service = Service(log_output=f"{executable_name}.log", service_args=['--log-no-truncate'])
+    service = Service(executable_path=f"/usr/bin/{executable_name}", log_output=f"{executable_name}.log", service_args=['--log-no-truncate'])
     options = Options()
     options.add_argument("--headless")
     os.environ["MOZ_REMOTE_SETTINGS_DEVTOOLS"] = "1"

From a3e4e634fce72ee1e9ef75f5f207408999f1a94b Mon Sep 17 00:00:00 2001
From: Sangho Na <sangho@nsh.nz>
Date: Mon, 22 Jul 2024 06:14:02 +1200
Subject: [PATCH 019/656] fix: Exclude replaced documents from IESG discusses
 (#7712)

* fix: Exclude replaced documents from IESG discusses

* test: Add checks for filtering replaced IESG drafts

* chore: Improve replaced draft filter

---------

Co-authored-by: Paul Selkirk <paul@painless-security.com>
---
 ietf/iesg/tests.py | 9 +++++++++
 ietf/iesg/views.py | 1 +
 2 files changed, 10 insertions(+)

diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py
index 7211a6bc06..4579316f22 100644
--- a/ietf/iesg/tests.py
+++ b/ietf/iesg/tests.py
@@ -52,6 +52,15 @@ def test_feed(self):
         self.assertContains(r, draft.name)
         self.assertContains(r, escape(pos.balloter.plain_name()))
 
+        # Mark draft as replaced
+        draft.set_state(State.objects.get(type="draft", slug="repl"))
+
+        r = self.client.get(urlreverse("ietf.iesg.views.discusses"))
+        self.assertEqual(r.status_code, 200)
+
+        self.assertNotContains(r, draft.name)
+        self.assertNotContains(r, escape(pos.balloter.plain_name()))
+
     def test_milestones_needing_review(self):
         draft = WgDraftFactory()
         RoleFactory(name_id='ad',group=draft.group,person=Person.objects.get(user__username='ad'))
diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py
index a219a6b5de..b67ef04a03 100644
--- a/ietf/iesg/views.py
+++ b/ietf/iesg/views.py
@@ -483,6 +483,7 @@ def discusses(request):
                                             models.Q(states__type__in=("statchg", "conflrev"),
                                                      states__slug__in=("iesgeval", "defer")),
                                             docevent__ballotpositiondocevent__pos__blocking=True)
+    possible_docs = possible_docs.exclude(states__in=State.objects.filter(type="draft", slug="repl"))
     possible_docs = possible_docs.select_related("stream", "group", "ad").distinct()
 
     docs = []

From aa36f481e13c99bc0865282a9e890e6b049cdb98 Mon Sep 17 00:00:00 2001
From: Sangho Na <sangho@nsh.nz>
Date: Mon, 22 Jul 2024 06:15:33 +1200
Subject: [PATCH 020/656] chore: Add additional log messages to directauth()
 (#7716)

* chore: Add additional log messages to directauth()

* chore: Keep single log message for each successful response
---
 ietf/api/views.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/ietf/api/views.py b/ietf/api/views.py
index 6aaed4b6a9..62857bff54 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -429,6 +429,7 @@ def directauth(request):
                 data = None
 
         if raw_data is None or data is None:
+            log.log("Request body is either missing or invalid")
             return HttpResponse(json.dumps(dict(result="failure",reason="invalid post")), content_type='application/json')
 
         authtoken = data.get('authtoken', None)
@@ -436,9 +437,11 @@ def directauth(request):
         password = data.get('password', None)
 
         if any([item is None for item in (authtoken, username, password)]):
+            log.log("One or more mandatory fields are missing: authtoken, username, password")
             return HttpResponse(json.dumps(dict(result="failure",reason="invalid post")), content_type='application/json')
 
         if not is_valid_token("ietf.api.views.directauth", authtoken):
+            log.log("Auth token provided is invalid")
             return HttpResponse(json.dumps(dict(result="failure",reason="invalid authtoken")), content_type='application/json')
         
         user_query = User.objects.filter(username__iexact=username)
@@ -449,18 +452,20 @@ def directauth(request):
 
 
         # Note well that we are using user.username, not what was passed to the API.
-        if user_query.count() == 1 and authenticate(username = user_query.first().username, password = password):
+        user_count = user_query.count()
+        if user_count == 1 and authenticate(username = user_query.first().username, password = password):
             user = user_query.get()
             if user_query.filter(person__isnull=True).count() == 1: # Can't inspect user.person direclty here
-                log.log(f"Direct auth of personless user {user.pk}:{user.username}")
+                log.log(f"Direct auth success (personless user): {user.pk}:{user.username}")
             else:
-                log.log(f"Direct auth: {user.pk}:{user.person.plain_name()}")
+                log.log(f"Direct auth success: {user.pk}:{user.person.plain_name()}")
             return HttpResponse(json.dumps(dict(result="success")), content_type='application/json')
 
-        log.log(f"Direct auth failure: {username}")
+        log.log(f"Direct auth failure: {username} ({user_count} user(s) found)")
         return HttpResponse(json.dumps(dict(result="failure", reason="authentication failed")), content_type='application/json') 
 
     else:
+        log.log(f"Request must be POST: {request.method} received")
         return HttpResponse(status=405)
 
 

From d5ceb7b20d89ae0d5b8948c151f48c8ce4d546c9 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 24 Jul 2024 14:20:18 -0700
Subject: [PATCH 021/656] fix: optional / for /person/merge/ URL (#7746)

---
 ietf/person/urls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/person/urls.py b/ietf/person/urls.py
index f37d8b46cf..867646fe39 100644
--- a/ietf/person/urls.py
+++ b/ietf/person/urls.py
@@ -2,7 +2,7 @@
 from ietf.utils.urls import url
 
 urlpatterns = [
-    url(r'^merge/$', views.merge),
+    url(r'^merge/?$', views.merge),
     url(r'^search/(?P<model_name>(person|email))/$', views.ajax_select2_search),
     url(r'^(?P<personid>[0-9]+)/email.json$', ajax.person_email_json),
     url(r'^(?P<email_or_name>[^/]+)$', views.profile),

From b5ab4b66110ed0777dae170e87db9343876b2741 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 25 Jul 2024 15:31:53 -0700
Subject: [PATCH 022/656] chore: update test name fixture (#7751)

---
 ietf/name/fixtures/names.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json
index 913c6c987e..3eb2c38d6f 100644
--- a/ietf/name/fixtures/names.json
+++ b/ietf/name/fixtures/names.json
@@ -3464,7 +3464,7 @@
       "parent_types": [],
       "req_subm_approval": true,
       "role_order": "[\n    \"chair\",\n    \"delegate\"\n]",
-      "session_purposes": "[\n    \"officehours\"\n]",
+      "session_purposes": "[\n    \"officehours\",\n    \"regular\"\n]",
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",

From 247361b7dd8550233991abaa289ca36241c8561f Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 30 Jul 2024 20:55:07 -0300
Subject: [PATCH 023/656] ci: better access logs+redirect auth URLs+fix
 X-Request-Start header (#7700)

* fix: silence nginx healthcheck logs

* fix: nginx logs in JSON

* fix: typos in nginx conf

* refactor: repeat less nginx config

* fix: log more req headers from gunicorn

* fix: redirect auth->datatracker, not deny

* feat: log X-Forwarded-Proto
---
 ietf/utils/jsonlogger.py   |  8 ++++++++
 k8s/auth.yaml              |  3 +++
 k8s/datatracker.yaml       |  3 +++
 k8s/kustomization.yaml     |  1 +
 k8s/nginx-auth.conf        |  8 ++++++--
 k8s/nginx-datatracker.conf |  6 +++++-
 k8s/nginx-logging.conf     | 20 ++++++++++++++++++++
 7 files changed, 46 insertions(+), 3 deletions(-)
 create mode 100644 k8s/nginx-logging.conf

diff --git a/ietf/utils/jsonlogger.py b/ietf/utils/jsonlogger.py
index a9eeb02ba9..9c7949fd58 100644
--- a/ietf/utils/jsonlogger.py
+++ b/ietf/utils/jsonlogger.py
@@ -24,3 +24,11 @@ def add_fields(self, log_record, record, message_dict):
         log_record.setdefault("user_agent", record.args["a"])
         log_record.setdefault("len_bytes", record.args["B"])
         log_record.setdefault("duration_ms", record.args["M"])
+        log_record.setdefault("host", record.args["{host}i"])
+        log_record.setdefault("x_request_start", record.args["{x-request-start}i"])
+        log_record.setdefault("x_real_ip", record.args["{x-real-ip}i"])
+        log_record.setdefault("x_forwarded_for", record.args["{x-forwarded-for}i"])
+        log_record.setdefault("x_forwarded_proto", record.args["{x-forwarded-proto}i"])
+        log_record.setdefault("cf_connecting_ip", record.args["{cf-connecting-ip}i"])
+        log_record.setdefault("cf_connecting_ipv6", record.args["{cf-connecting-ipv6}i"])
+        log_record.setdefault("cf_ray", record.args["{cf-ray}i"])
diff --git a/k8s/auth.yaml b/k8s/auth.yaml
index 66627ed450..c35cdc8ac2 100644
--- a/k8s/auth.yaml
+++ b/k8s/auth.yaml
@@ -80,6 +80,9 @@ spec:
           volumeMounts:
             - name: nginx-tmp
               mountPath: /tmp
+            - name: dt-cfg
+              mountPath: /etc/nginx/conf.d/00logging.conf
+              subPath: nginx-logging.conf
             - name: dt-cfg
               mountPath: /etc/nginx/conf.d/auth.conf
               subPath: nginx-auth.conf
diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml
index 9e1ead1a90..a8a9675687 100644
--- a/k8s/datatracker.yaml
+++ b/k8s/datatracker.yaml
@@ -80,6 +80,9 @@ spec:
           volumeMounts:
             - name: nginx-tmp
               mountPath: /tmp
+            - name: dt-cfg
+              mountPath: /etc/nginx/conf.d/00logging.conf
+              subPath: nginx-logging.conf
             - name: dt-cfg
               mountPath: /etc/nginx/conf.d/datatracker.conf
               subPath: nginx-datatracker.conf
diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml
index ba8b8a5826..4b79f00753 100644
--- a/k8s/kustomization.yaml
+++ b/k8s/kustomization.yaml
@@ -3,6 +3,7 @@ namePrefix: dt-
 configMapGenerator:
   - name: files-cfgmap
     files:
+      - nginx-logging.conf
       - nginx-auth.conf
       - nginx-datatracker.conf
       - settings_local.py
diff --git a/k8s/nginx-auth.conf b/k8s/nginx-auth.conf
index 4cbc8a0a51..6dd5d6ed56 100644
--- a/k8s/nginx-auth.conf
+++ b/k8s/nginx-auth.conf
@@ -2,9 +2,13 @@ server {
     listen 8080 default_server;
     server_name _;
 
+    # Replace default "main" formatter with the ietfjson formatter from nginx-logging.conf
+    access_log  /var/log/nginx/access.log  ietfjson;
+
     # Note that regex location matches take priority over non-regex "prefix" matches. Use regexes so that
     # our deny all rule does not squelch the other locations. 
     location ~ ^/health/nginx$ {
+        access_log off;
         return 200;
     }
 
@@ -19,14 +23,14 @@ server {
 
     # n.b. (?!...) is a negative lookahead group
     location ~ ^(/(?!(api/openid/|accounts/login/|accounts/logout/|accounts/reset/|person/.*/photo|group/groupmenu.json)).*) {
-        deny all;
+        return 302 https://datatracker.ietf.org$${keepempty}request_uri;
     }
 
     location / {
         add_header Content-Security-Policy "default-src 'self' 'unsafe-inline' data: https://datatracker.ietf.org/ https://www.ietf.org/ http://ietf.org/ https://analytics.ietf.org https://static.ietf.org; frame-ancestors 'self' ietf.org *.ietf.org meetecho.com *.meetecho.com gather.town *.gather.town";
         proxy_set_header Host $${keepempty}host;
         proxy_set_header Connection close;
-        proxy_set_header X-Request-Start "t=${msec}";
+        proxy_set_header X-Request-Start "t=$${keepempty}msec";
         proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
         proxy_set_header X-Real-IP $${keepempty}remote_addr;
         proxy_pass http://localhost:8000;
diff --git a/k8s/nginx-datatracker.conf b/k8s/nginx-datatracker.conf
index 63c985463c..ff439fba6a 100644
--- a/k8s/nginx-datatracker.conf
+++ b/k8s/nginx-datatracker.conf
@@ -2,7 +2,11 @@ server {
     listen 8080 default_server;
     server_name _;
 
+    # Replace default "main" formatter with the ietfjson formatter from nginx-logging.conf
+    access_log  /var/log/nginx/access.log  ietfjson;
+
     location /health/nginx {
+        access_log off;
         return 200;
     }
 
@@ -15,7 +19,7 @@ server {
         add_header Content-Security-Policy "default-src 'self' 'unsafe-inline' data: https://datatracker.ietf.org/ https://www.ietf.org/ http://ietf.org/ https://analytics.ietf.org https://static.ietf.org; frame-ancestors 'self' ietf.org *.ietf.org meetecho.com *.meetecho.com";
         proxy_set_header Host $${keepempty}host;
         proxy_set_header Connection close;
-        proxy_set_header X-Request-Start "t=${msec}";
+        proxy_set_header X-Request-Start "t=$${keepempty}msec";
         proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
         proxy_set_header X-Real-IP $${keepempty}remote_addr;
         proxy_pass http://localhost:8000;
diff --git a/k8s/nginx-logging.conf b/k8s/nginx-logging.conf
new file mode 100644
index 0000000000..0938b0530e
--- /dev/null
+++ b/k8s/nginx-logging.conf
@@ -0,0 +1,20 @@
+# Define JSON log format - must be loaded before config that references it
+log_format  ietfjson  escape=json
+  '{'
+    '"time":"$${keepempty}time_iso8601",'
+    '"remote_ip":"$${keepempty}remote_addr",'
+    '"request":"$${keepempty}request",'
+    '"host":"$${keepempty}host",'
+    '"path":"$${keepempty}request_uri",'
+    '"method":"$${keepempty}request_method",'
+    '"status":"$${keepempty}status",'
+    '"len_bytes":"$${keepempty}body_bytes_sent",'
+    '"duration_ms":"$${keepempty}request_time",'
+    '"referer":"$${keepempty}http_referer",'
+    '"user_agent":"$${keepempty}http_user_agent",'
+    '"x_forwarded_for":"$${keepempty}http_x_forwarded_for",'
+    '"x_forwarded_proto":"$${keepempty}http_x_forwarded_proto",'
+    '"cf_connecting_ip":"$${keepempty}http_cf_connecting_ip",'
+    '"cf_connecting_ipv6":"$${keepempty}http_cf_connecting_ipv6",'
+    '"cf_ray":"$${keepempty}http_cf_ray"'
+  '}';

From fb1942a5386873eb48402fd274cc29c5877bf377 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Fri, 2 Aug 2024 07:25:03 +1200
Subject: [PATCH 024/656] fix: Sort RFCs by date (#7766)

* fix: Sort RFCs by date

* fix: concluded wgs and bofs date sort #7350
---
 ietf/static/js/list.js                     | 6 ++++--
 ietf/templates/group/concluded_groups.html | 8 ++++----
 ietf/templates/person/profile.html         | 2 +-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/ietf/static/js/list.js b/ietf/static/js/list.js
index d7e9dc944b..c16111ba63 100644
--- a/ietf/static/js/list.js
+++ b/ietf/static/js/list.js
@@ -5,8 +5,10 @@ import {
 function text_sort(a, b, options) {
 
     function prep(e, options) {
-        return $($.parseHTML(e.values()[options.valueName]))
-            .text()
+        const el = $($.parseHTML(e.values()[options.valueName]));
+        const cell_el = e.elm.querySelector(`.${options.valueName}`)
+        const sort_by_number = cell_el?.getAttribute('data-sort-number')
+        return sort_by_number ?? el.text()
             .trim()
             .replaceAll(/\s+/g, ' ');
     }
diff --git a/ietf/templates/group/concluded_groups.html b/ietf/templates/group/concluded_groups.html
index c748c2061b..725e8bd3cc 100644
--- a/ietf/templates/group/concluded_groups.html
+++ b/ietf/templates/group/concluded_groups.html
@@ -40,8 +40,8 @@ <h2 class="mt-5" id="{{ label|slugify }}">{{ label }}</h2>
                         <tr>
                             <th scope="col" data-sort="group">Group</th>
                             <th scope="col" data-sort="name">Name</th>
-                            <th scope="col" data-sort="date">Start</th>
-                            <th scope="col" data-sort="date">Concluded</th>
+                            <th scope="col" data-sort="date-start">Start</th>
+                            <th scope="col" data-sort="date-concluded">Concluded</th>
                         </tr>
                     </thead>
                     <tbody>
@@ -51,8 +51,8 @@ <h2 class="mt-5" id="{{ label|slugify }}">{{ label }}</h2>
                                     <a href="{{ g.about_url }}">{{ g.acronym }}</a>
                                 </td>
                                 <td>{{ g.name }}</td>
-                                <td>{{ g.start_date|date:"Y-m" }}</td>
-                                <td>{{ g.conclude_date|date:"Y-m" }}</td>
+                                <td data-sort-number="{{ g.start_date|date:"U" }}">{{ g.start_date|date:"Y-m" }}</td>
+                                <td data-sort-number="{{ g.conclude_date|date:"U" }}">{{ g.conclude_date|date:"Y-m" }}</td>
                             </tr>
                         {% endfor %}
                     </tbody>
diff --git a/ietf/templates/person/profile.html b/ietf/templates/person/profile.html
index 42e5d2e43a..1424f037a1 100644
--- a/ietf/templates/person/profile.html
+++ b/ietf/templates/person/profile.html
@@ -106,7 +106,7 @@ <h2 class="mt-5" id="rfcs-{{ forloop.counter }}">
                             <td>
                                 <a class="text-nowrap" href="{{ doc.get_absolute_url }}">RFC {{ doc.rfc_number }}</a>
                             </td>
-                            <td>{{ doc.pub_date|date:"b Y"|title }}</td>
+                            <td data-sort-number="{{ doc.pub_date|date:"U" }}">{{ doc.pub_date|date:"b Y"|title }}</td>
                             <td>{{ doc.title|urlize_ietf_docs }}</td>
                             <td class="text-end">
                                 {% with doc.referenced_by_rfcs_as_rfc_or_draft.count as refbycount %}

From 06677a9863b541d2e322b2ec5fd1032035c175fd Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 1 Aug 2024 17:23:35 -0300
Subject: [PATCH 025/656] fix: require login to pdfize (#7775)

* fix: require login to pdfize

* fix: suppress "pdfized" button when it won't work
---
 ietf/doc/views_doc.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 42898d2098..dfef40e558 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -265,6 +265,8 @@ def document_main(request, name, rev=None, document_html=False):
         can_change_stream = bool(can_edit or roles)
 
         file_urls, found_types = build_file_urls(doc)
+        if not request.user.is_authenticated:
+            file_urls = [fu for fu in file_urls if fu[0] != "pdfized"]
         content = doc.text_or_error() # pyflakes:ignore
         content = markup_txt.markup(maybe_split(content, split=split_content))
 
@@ -406,6 +408,8 @@ def document_main(request, name, rev=None, document_html=False):
         latest_revision = None
 
         file_urls, found_types = build_file_urls(doc)
+        if not request.user.is_authenticated:
+            file_urls = [fu for fu in file_urls if fu[0] != "pdfized"]
         content = doc.text_or_error() # pyflakes:ignore
         content = markup_txt.markup(maybe_split(content, split=split_content))
 
@@ -1039,6 +1043,8 @@ def document_html(request, name, rev=None):
         document_html=True,
     )
 
+
+@login_required
 def document_pdfized(request, name, rev=None, ext=None):
 
     found = fuzzy_find_documents(name, rev)

From d683920cecf2cd877bf4e3b3453de6b7494bed12 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 1 Aug 2024 18:33:49 -0300
Subject: [PATCH 026/656] test: log in before pdfize test (#7778)

---
 ietf/doc/tests.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index 5d0bb9c4f9..5e5b741fe5 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -3013,6 +3013,13 @@ def test_pdfized(self):
             with (Path(dir) / f'{draft.name}-{r:02d}.txt').open('w') as f:
                 f.write('text content')
 
+        self.assertTrue(
+            login_testing_unauthorized(
+                self,
+                PersonFactory().user.username,
+                urlreverse(self.view, kwargs={"name": draft.name}),
+            )
+        )
         self.should_succeed(dict(name=rfc.name))
         self.should_succeed(dict(name=draft.name))
         for r in range(0,2):

From 0b445a9f0985ed6a96569391b39cc1ef4a31a5d0 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 5 Aug 2024 10:47:32 -0300
Subject: [PATCH 027/656] docs: fix email in README.md (#7784)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 133d08f5e7..ee9865ba21 100644
--- a/README.md
+++ b/README.md
@@ -81,7 +81,7 @@ Many developers are using [VS Code](https://code.visualstudio.com/) and taking a
 
 If VS Code is not available to you, in your clone, type `cd docker; ./run`
 
-Once the containers are started, run the tests to make sure your checkout is a good place to start from (all tests should pass - if any fail, ask for help at tools-develop@). Inside the app container's shell type:
+Once the containers are started, run the tests to make sure your checkout is a good place to start from (all tests should pass - if any fail, ask for help at tools-help@). Inside the app container's shell type:
 ```sh
 ietf/manage.py test --settings=settings_test
 ```

From b13a606a247c8603e7494822c58ff5028ccce5f5 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 5 Aug 2024 11:00:15 -0300
Subject: [PATCH 028/656] feat: recognize HTTPS via proxy (#7765)

* feat: set SECURE_PROXY_SSL_HEADER

* chore: update comment
---
 k8s/settings_local.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 6f0956d065..33ac4f1e38 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -17,6 +17,13 @@ def _multiline_to_list(s):
 # Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env!
 SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development")
 
+# Use X-Forwarded-Proto to determine request.is_secure(). This relies on CloudFlare overwriting the
+# value of the header if an incoming request sets it, which it does:
+# https://developers.cloudflare.com/fundamentals/reference/http-request-headers/#x-forwarded-proto
+# See also, especially the warnings:
+# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header
+SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
 # Secrets
 _SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None)
 if _SECRET_KEY is not None:

From 8a5826a9414ddc53cb721b069e19bf96e2c2c71a Mon Sep 17 00:00:00 2001
From: Rich Salz <rsalz@akamai.com>
Date: Mon, 5 Aug 2024 10:48:48 -0400
Subject: [PATCH 029/656] fix: redundant word in banner for Legacy stream
 documents (#7207)

* fix: Remove redundant "stream stream" output

fix: Change "Legacy stream" to "Legacy"

chore: Add "stream" to stream.desc as needed

Fixes: #6902

* chore: Remove unused stream_desc parameter

The stream_desc key isn't used in template/doc/docuemnt_draft.html to
don't pass it in nor compute it

Fixes: #6902

* fix: migrate the legacy StreamName

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/doc/views_doc.py                         |  8 +------
 ietf/name/fixtures/names.json                 |  2 +-
 .../0014_change_legacy_stream_desc.py         | 21 +++++++++++++++++++
 ietf/secr/templates/telechat/doc.html         |  2 +-
 .../doc/mail/last_call_announcement.txt       |  2 +-
 5 files changed, 25 insertions(+), 10 deletions(-)
 create mode 100644 ietf/name/migrations/0014_change_legacy_stream_desc.py

diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index dfef40e558..21c5eb235e 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -607,12 +607,7 @@ def document_main(request, name, rev=None, document_html=False):
         additional_urls = doc.documenturl_set.exclude(tag_id='auth48')
 
         # Stream description and name passing test
-        if doc.stream != None:
-            stream_desc = doc.stream.desc
-            stream = "draft-stream-" + doc.stream.slug
-        else:
-            stream_desc = "(None)"
-            stream = "(None)"
+        stream = ("draft-stream-" + doc.stream.slug) if doc.stream != None else "(None)"
 
         html = None
         js = None
@@ -651,7 +646,6 @@ def document_main(request, name, rev=None, document_html=False):
                                        revisions=simple_diff_revisions if document_html else revisions,
                                        snapshot=snapshot,
                                        stream=stream,
-                                       stream_desc=stream_desc,
                                        latest_revision=latest_revision,
                                        latest_rev=latest_rev,
                                        can_edit=can_edit,
diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json
index 3eb2c38d6f..59b367deb8 100644
--- a/ietf/name/fixtures/names.json
+++ b/ietf/name/fixtures/names.json
@@ -14058,7 +14058,7 @@
   },
   {
     "fields": {
-      "desc": "Legacy stream",
+      "desc": "Legacy",
       "name": "Legacy",
       "order": 6,
       "used": true
diff --git a/ietf/name/migrations/0014_change_legacy_stream_desc.py b/ietf/name/migrations/0014_change_legacy_stream_desc.py
new file mode 100644
index 0000000000..8297e86274
--- /dev/null
+++ b/ietf/name/migrations/0014_change_legacy_stream_desc.py
@@ -0,0 +1,21 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+
+from django.db import migrations
+
+def forward(apps, schema_editor):
+    StreamName = apps.get_model("name", "StreamName")
+    StreamName.objects.filter(pk="legacy").update(desc="Legacy")
+
+def reverse(apps, schema_editor):
+    StreamName = apps.get_model("name", "StreamName")
+    StreamName.objects.filter(pk="legacy").update(desc="Legacy stream")
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("name", "0013_narrativeminutes"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse)
+    ]
diff --git a/ietf/secr/templates/telechat/doc.html b/ietf/secr/templates/telechat/doc.html
index 7891c1b1e7..6727e157f5 100644
--- a/ietf/secr/templates/telechat/doc.html
+++ b/ietf/secr/templates/telechat/doc.html
@@ -86,7 +86,7 @@ <h2 id="writeup">Ballot Writeup</h2>
                 <h2 id="downrefs">Downward References</h2>
                 {% for ref in downrefs %}
                     <p>Add {{ref.target.name}}
-                        ({{ref.target.std_level}} - {{ref.target.stream.desc}})
+                        ({{ref.target.std_level}} - {{ref.target.stream.desc}} stream)
                         to downref registry.<br>
                         {% if not ref.target.std_level %}
                             +++ Warning: The standards level has not been set yet!!!<br>
diff --git a/ietf/templates/doc/mail/last_call_announcement.txt b/ietf/templates/doc/mail/last_call_announcement.txt
index 8f15a8e2a6..5cf2e9c45b 100644
--- a/ietf/templates/doc/mail/last_call_announcement.txt
+++ b/ietf/templates/doc/mail/last_call_announcement.txt
@@ -33,7 +33,7 @@ No IPR declarations have been submitted directly on this I-D.
 {% if downrefs %}
 The document contains these normative downward references.
 See RFC 3967 for additional information: 
-{% for ref in downrefs %}    {{ref.target.name}}: {{ref.target.title}} ({{ref.target.std_level}} - {{ref.target.stream.desc}})
+{% for ref in downrefs %}    {{ref.target.name}}: {{ref.target.title}} ({{ref.target.std_level}} - {{ref.target.stream.desc}} stream)
 {% endfor %}{%endif%}
 
 {% endautoescape %}

From 16ac73d4b748de2ae1162452753a84bc3d6a0369 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 5 Aug 2024 12:18:01 -0300
Subject: [PATCH 030/656] fix: use BOF states in concluded_groups() (#7771)

* fix: use BOF states in concluded_groups()

* fix: handle events for older BOFs

These could be cleaned up in the database, but I think this
change does the right thing for the existing data.

* style: Black
---
 ietf/group/views.py | 79 +++++++++++++++++++++++++++++++++++++--------
 1 file changed, 65 insertions(+), 14 deletions(-)

diff --git a/ietf/group/views.py b/ietf/group/views.py
index f909a31b6d..e3fd7e80d9 100644
--- a/ietf/group/views.py
+++ b/ietf/group/views.py
@@ -334,35 +334,86 @@ def chartering_groups(request):
                   dict(charter_states=charter_states,
                        group_types=group_types))
 
+
 def concluded_groups(request):
     sections = OrderedDict()
 
-    sections['WGs'] = Group.objects.filter(type='wg', state="conclude").select_related("state", "charter").order_by("parent__name","acronym")
-    sections['RGs'] = Group.objects.filter(type='rg', state="conclude").select_related("state", "charter").order_by("parent__name","acronym")
-    sections['BOFs'] = Group.objects.filter(type='wg', state="bof-conc").select_related("state", "charter").order_by("parent__name","acronym")
-    sections['AGs'] = Group.objects.filter(type='ag', state="conclude").select_related("state", "charter").order_by("parent__name","acronym")
-    sections['RAGs'] = Group.objects.filter(type='rag', state="conclude").select_related("state", "charter").order_by("parent__name","acronym")
-    sections['Directorates'] = Group.objects.filter(type='dir', state="conclude").select_related("state", "charter").order_by("parent__name","acronym")
-    sections['Review teams'] = Group.objects.filter(type='review', state="conclude").select_related("state", "charter").order_by("parent__name","acronym")
-    sections['Teams'] = Group.objects.filter(type='team', state="conclude").select_related("state", "charter").order_by("parent__name","acronym")
-    sections['Programs'] = Group.objects.filter(type='program', state="conclude").select_related("state", "charter").order_by("parent__name","acronym")
+    sections["WGs"] = (
+        Group.objects.filter(type="wg", state="conclude")
+        .select_related("state", "charter")
+        .order_by("parent__name", "acronym")
+    )
+    sections["RGs"] = (
+        Group.objects.filter(type="rg", state="conclude")
+        .select_related("state", "charter")
+        .order_by("parent__name", "acronym")
+    )
+    sections["BOFs"] = (
+        Group.objects.filter(type="wg", state="bof-conc")
+        .select_related("state", "charter")
+        .order_by("parent__name", "acronym")
+    )
+    sections["AGs"] = (
+        Group.objects.filter(type="ag", state="conclude")
+        .select_related("state", "charter")
+        .order_by("parent__name", "acronym")
+    )
+    sections["RAGs"] = (
+        Group.objects.filter(type="rag", state="conclude")
+        .select_related("state", "charter")
+        .order_by("parent__name", "acronym")
+    )
+    sections["Directorates"] = (
+        Group.objects.filter(type="dir", state="conclude")
+        .select_related("state", "charter")
+        .order_by("parent__name", "acronym")
+    )
+    sections["Review teams"] = (
+        Group.objects.filter(type="review", state="conclude")
+        .select_related("state", "charter")
+        .order_by("parent__name", "acronym")
+    )
+    sections["Teams"] = (
+        Group.objects.filter(type="team", state="conclude")
+        .select_related("state", "charter")
+        .order_by("parent__name", "acronym")
+    )
+    sections["Programs"] = (
+        Group.objects.filter(type="program", state="conclude")
+        .select_related("state", "charter")
+        .order_by("parent__name", "acronym")
+    )
 
     for name, groups in sections.items():
-        
         # add start/conclusion date
         d = dict((g.pk, g) for g in groups)
 
         for g in groups:
             g.start_date = g.conclude_date = None
 
-        for e in ChangeStateGroupEvent.objects.filter(group__in=groups, state="active").order_by("-time"):
+        # Some older BOFs were created in the "active" state, so consider both "active" and "bof"
+        # ChangeStateGroupEvents when finding the start date. A group with _both_ "active" and "bof"
+        # events should not be in the "bof-conc" state so this shouldn't cause a problem (if it does,
+        # we'll need to clean up the data)
+        for e in ChangeStateGroupEvent.objects.filter(
+            group__in=groups,
+            state__in=["active", "bof"] if name == "BOFs" else ["active"],
+        ).order_by("-time"):
             d[e.group_id].start_date = e.time
 
-        for e in ChangeStateGroupEvent.objects.filter(group__in=groups, state="conclude").order_by("time"):
+        # Similarly, some older BOFs were concluded into the "conclude" state and the event was never
+        # fixed, so consider both "conclude" and "bof-conc" ChangeStateGroupEvents when finding the
+        # concluded date. A group with _both_ "conclude" and "bof-conc" events should not be in the
+        # "bof-conc" state so this shouldn't cause a problem (if it does, we'll need to clean up the
+        # data)
+        for e in ChangeStateGroupEvent.objects.filter(
+            group__in=groups,
+            state__in=["bof-conc", "conclude"] if name == "BOFs" else ["conclude"],
+        ).order_by("time"):
             d[e.group_id].conclude_date = e.time
 
-    return render(request, 'group/concluded_groups.html',
-                  dict(sections=sections))
+    return render(request, "group/concluded_groups.html", dict(sections=sections))
+
 
 def prepare_group_documents(request, group, clist):
     found_docs, meta = prepare_document_table(request, docs_tracked_by_community_list(clist), request.GET, max_results=500)

From 63d13074d1c6496223b85d1bd471db5211004d16 Mon Sep 17 00:00:00 2001
From: Russ Housley <housley@vigilsec.com>
Date: Mon, 5 Aug 2024 12:03:17 -0400
Subject: [PATCH 031/656] fix: use area-acronym-at-the-time in proceedings
 (#7723)

When generating IETF meeting proceedings, use the Area Acronym for each WG at the time of the meeting. Fixes #7706.
---
 ietf/meeting/views.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 253f2852ff..3e483b193b 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -4131,6 +4131,13 @@ def _format_materials(items):
 
 def proceedings(request, num=None):
 
+    def area_and_group_acronyms_from_session(s):
+        area = s.group_parent_at_the_time()
+        if area == None:
+            area = s.group.parent
+        group = s.group_at_the_time()
+        return (area.acronym, group.acronym)
+
     meeting = get_meeting(num)
 
     # Early proceedings were hosted on www.ietf.org rather than the datatracker
@@ -4181,12 +4188,11 @@ def proceedings(request, num=None):
         .exclude(current_status='notmeet')
     )
 
-    ietf = sessions.filter(group__parent__type__slug = 'area').exclude(group__acronym='edu').order_by('group__parent__acronym', 'group__acronym')
+    ietf = sessions.filter(group__parent__type__slug = 'area').exclude(group__acronym__in=['edu','iepg','tools'])
+    ietf = list(ietf)
+    ietf.sort(key=lambda s: area_and_group_acronyms_from_session(s))
     ietf_areas = []
-    for area, area_sessions in itertools.groupby(
-            ietf,
-            key=lambda s: s.group.parent
-    ):
+    for area, area_sessions in itertools.groupby(ietf, key=lambda s: s.group_parent_at_the_time()):
         meeting_groups, not_meeting_groups = organize_proceedings_sessions(area_sessions)
         ietf_areas.append((area, meeting_groups, not_meeting_groups))
 

From 9ef7bff77c1208ed6c9504002aee3320856e6499 Mon Sep 17 00:00:00 2001
From: Jim Fenton <fenton@bluepopcorn.net>
Date: Tue, 6 Aug 2024 08:03:37 -0700
Subject: [PATCH 032/656] feat: Unify slide upload and proposal (#7787)

* attempt at optional approval

* Update of meeting slides propose/upload

* Fix tests and residual coding bugs

* Remove gratuitous blank lines
---
 ietf/meeting/forms.py                         |   7 +-
 ietf/meeting/tests_views.py                   |  45 ++++--
 ietf/meeting/urls.py                          |   1 -
 ietf/meeting/views.py                         | 130 +++++++-----------
 .../meeting/propose_session_slides.html       |  27 ----
 .../meeting/session_details_panel.html        |   2 +-
 ietf/templates/meeting/slides_approved.txt    |   2 +-
 .../meeting/upload_session_slides.html        |  12 +-
 8 files changed, 95 insertions(+), 131 deletions(-)
 delete mode 100644 ietf/templates/meeting/propose_session_slides.html

diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py
index b31ffb6cd7..3b66d2cd29 100644
--- a/ietf/meeting/forms.py
+++ b/ietf/meeting/forms.py
@@ -489,9 +489,12 @@ class UploadAgendaForm(ApplyToAllFileUploadForm):
 class UploadSlidesForm(ApplyToAllFileUploadForm):
     doc_type = 'slides'
     title = forms.CharField(max_length=255)
+    approved = forms.BooleanField(label='Auto-approve', initial=True, required=False)
 
-    def __init__(self, session, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+    def __init__(self, session, show_apply_to_all_checkbox, can_manage, *args, **kwargs):
+        super().__init__(show_apply_to_all_checkbox, *args, **kwargs)
+        if not can_manage:
+            self.fields.pop('approved')
         self.session = session
 
     def clean_title(self):
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index db62fe6204..da82afb329 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -6454,7 +6454,7 @@ def test_upload_slides(self, mock_slides_manager_cls):
         self.assertFalse(session1.presentations.filter(document__type_id='slides'))
         test_file = BytesIO(b'this is not really a slide')
         test_file.name = 'not_really.txt'
-        r = self.client.post(url,dict(file=test_file,title='a test slide file',apply_to_all=True))
+        r = self.client.post(url,dict(file=test_file,title='a test slide file',apply_to_all=True,approved=True))
         self.assertEqual(r.status_code, 302)
         self.assertEqual(session1.presentations.count(),1) 
         self.assertEqual(session2.presentations.count(),1) 
@@ -6477,7 +6477,7 @@ def test_upload_slides(self, mock_slides_manager_cls):
         url = urlreverse('ietf.meeting.views.upload_session_slides',kwargs={'num':session2.meeting.number,'session_id':session2.id})
         test_file = BytesIO(b'some other thing still not slidelike')
         test_file.name = 'also_not_really.txt'
-        r = self.client.post(url,dict(file=test_file,title='a different slide file',apply_to_all=False))
+        r = self.client.post(url,dict(file=test_file,title='a different slide file',apply_to_all=False,approved=True))
         self.assertEqual(r.status_code, 302)
         self.assertEqual(session1.presentations.count(),1)
         self.assertEqual(session2.presentations.count(),2)
@@ -6501,7 +6501,7 @@ def test_upload_slides(self, mock_slides_manager_cls):
         self.assertIn('Revise', str(q("title")))
         test_file = BytesIO(b'new content for the second slide deck')
         test_file.name = 'doesnotmatter.txt'
-        r = self.client.post(url,dict(file=test_file,title='rename the presentation',apply_to_all=False))
+        r = self.client.post(url,dict(file=test_file,title='rename the presentation',apply_to_all=False, approved=True))
         self.assertEqual(r.status_code, 302)
         self.assertEqual(session1.presentations.count(),1)
         self.assertEqual(session2.presentations.count(),2)
@@ -6597,7 +6597,7 @@ def test_propose_session_slides(self):
             newperson = PersonFactory()
             
             session_overview_url = urlreverse('ietf.meeting.views.session_details',kwargs={'num':session.meeting.number,'acronym':session.group.acronym})
-            propose_url = urlreverse('ietf.meeting.views.propose_session_slides', kwargs={'session_id':session.pk, 'num': session.meeting.number})    
+            upload_url = urlreverse('ietf.meeting.views.upload_session_slides', kwargs={'session_id':session.pk, 'num': session.meeting.number})    
 
             r = self.client.get(session_overview_url)
             self.assertEqual(r.status_code,200)
@@ -6612,13 +6612,13 @@ def test_propose_session_slides(self):
             self.assertTrue(q('.proposeslides'))
             self.client.logout()
 
-            login_testing_unauthorized(self,newperson.user.username,propose_url)
-            r = self.client.get(propose_url)
+            login_testing_unauthorized(self,newperson.user.username,upload_url)
+            r = self.client.get(upload_url)
             self.assertEqual(r.status_code,200)
             test_file = BytesIO(b'this is not really a slide')
             test_file.name = 'not_really.txt'
             empty_outbox()
-            r = self.client.post(propose_url,dict(file=test_file,title='a test slide file',apply_to_all=True))
+            r = self.client.post(upload_url,dict(file=test_file,title='a test slide file',apply_to_all=True,approved=False))
             self.assertEqual(r.status_code, 302)
             session = Session.objects.get(pk=session.pk)
             self.assertEqual(session.slidesubmission_set.count(),1)
@@ -6639,6 +6639,25 @@ def test_propose_session_slides(self):
             self.assertEqual(len(q('.proposedslidelist p')), 2)
             self.client.logout()
 
+            login_testing_unauthorized(self,chair.user.username,upload_url)
+            r = self.client.get(upload_url)
+            self.assertEqual(r.status_code,200)
+            test_file = BytesIO(b'this is not really a slide either')
+            test_file.name = 'again_not_really.txt'
+            empty_outbox()
+            r = self.client.post(upload_url,dict(file=test_file,title='a selfapproved test slide file',apply_to_all=True,approved=True))
+            self.assertEqual(r.status_code, 302)
+            self.assertEqual(len(outbox),0)
+            self.assertEqual(session.slidesubmission_set.count(),2)
+            self.client.logout()
+
+            self.client.login(username=chair.user.username, password=chair.user.username+"+password")
+            r = self.client.get(session_overview_url)
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            self.assertEqual(len(q('.uploadslidelist p')), 0)
+            self.client.logout()
+
     def test_disapprove_proposed_slides(self):
         submission = SlideSubmissionFactory()
         submission.session.meeting.importantdate_set.create(name_id='revsub',date=date_today() + datetime.timedelta(days=20))
@@ -6759,12 +6778,12 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls):
         session.meeting.importantdate_set.create(name_id='revsub',date=date_today()+datetime.timedelta(days=20))
         newperson = PersonFactory()
         
-        propose_url = urlreverse('ietf.meeting.views.propose_session_slides', kwargs={'session_id':session.pk, 'num': session.meeting.number})          
+        upload_url = urlreverse('ietf.meeting.views.upload_session_slides', kwargs={'session_id':session.pk, 'num': session.meeting.number})          
         
-        login_testing_unauthorized(self,newperson.user.username,propose_url)
+        login_testing_unauthorized(self,newperson.user.username,upload_url)
         test_file = BytesIO(b'this is not really a slide')
         test_file.name = 'not_really.txt'
-        r = self.client.post(propose_url,dict(file=test_file,title='a test slide file',apply_to_all=True))
+        r = self.client.post(upload_url,dict(file=test_file,title='a test slide file',apply_to_all=True,approved=False))
         self.assertEqual(r.status_code, 302)
         self.client.logout()
 
@@ -6787,15 +6806,15 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls):
         
         self.assertEqual(session.presentations.first().document.rev,'00')
 
-        login_testing_unauthorized(self,newperson.user.username,propose_url)
+        login_testing_unauthorized(self,newperson.user.username,upload_url)
         test_file = BytesIO(b'this is not really a slide, but it is another version of it')
         test_file.name = 'not_really.txt'
-        r = self.client.post(propose_url,dict(file=test_file,title='a test slide file',apply_to_all=True))
+        r = self.client.post(upload_url,dict(file=test_file,title='a test slide file',apply_to_all=True))
         self.assertEqual(r.status_code, 302)
 
         test_file = BytesIO(b'this is not really a slide, but it is third version of it')
         test_file.name = 'not_really.txt'
-        r = self.client.post(propose_url,dict(file=test_file,title='a test slide file',apply_to_all=True))
+        r = self.client.post(upload_url,dict(file=test_file,title='a test slide file',apply_to_all=True))
         self.assertEqual(r.status_code, 302)
         self.client.logout()       
 
diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py
index 26d3d93b20..f2e65578ec 100644
--- a/ietf/meeting/urls.py
+++ b/ietf/meeting/urls.py
@@ -22,7 +22,6 @@ def get_redirect_url(self, *args, **kwargs):
     url(r'^session/(?P<session_id>\d+)/narrativeminutes$', views.upload_session_narrativeminutes),
     url(r'^session/(?P<session_id>\d+)/agenda$', views.upload_session_agenda),
     url(r'^session/(?P<session_id>\d+)/import/minutes$', views.import_session_minutes),
-    url(r'^session/(?P<session_id>\d+)/propose_slides$', views.propose_session_slides),
     url(r'^session/(?P<session_id>\d+)/slides(?:/%(name)s)?$' % settings.URL_REGEXPS, views.upload_session_slides),
     url(r'^session/(?P<session_id>\d+)/add_to_session$', views.ajax_add_slides_to_session),
     url(r'^session/(?P<session_id>\d+)/remove_from_session$', views.ajax_remove_slides_from_session),
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 3e483b193b..c3494a13ba 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -1702,7 +1702,7 @@ def api_get_session_materials(request, session_id=None):
 
     minutes = session.minutes()
     slides_actions = []
-    if can_manage_session_materials(request.user, session.group, session):
+    if can_manage_session_materials(request.user, session.group, session) or not session.is_material_submission_cutoff():
         slides_actions.append(
             {
                 "label": "Upload slides",
@@ -1712,16 +1712,6 @@ def api_get_session_materials(request, session_id=None):
                 ),
             }
         )
-    elif not session.is_material_submission_cutoff():
-        slides_actions.append(
-            {
-                "label": "Propose slides",
-                "url": reverse(
-                    "ietf.meeting.views.propose_session_slides",
-                    kwargs={"num": session.meeting.number, "session_id": session.pk},
-                ),
-            }
-        )
     else:
         pass  # no action available if it's past cutoff
 
@@ -2920,6 +2910,7 @@ def upload_session_agenda(request, session_id, num):
                   })
 
 
+@login_required
 def upload_session_slides(request, session_id, num, name=None):
     """Upload new or replacement slides for a session
     
@@ -2927,10 +2918,7 @@ def upload_session_slides(request, session_id, num, name=None):
     """
     # num is redundant, but we're dragging it along an artifact of where we are in the current URL structure
     session = get_object_or_404(Session, pk=session_id)
-    if not session.can_manage_materials(request.user):
-        permission_denied(
-            request, "You don't have permission to upload slides for this session."
-        )
+    can_manage = session.can_manage_materials(request.user)
     if session.is_material_submission_cutoff() and not has_role(
         request.user, "Secretariat"
     ):
@@ -2955,7 +2943,7 @@ def upload_session_slides(request, session_id, num, name=None):
 
     if request.method == "POST":
         form = UploadSlidesForm(
-            session, show_apply_to_all_checkbox, request.POST, request.FILES
+            session, show_apply_to_all_checkbox, can_manage, request.POST, request.FILES
         )
         if form.is_valid():
             file = request.FILES["file"]
@@ -2963,6 +2951,46 @@ def upload_session_slides(request, session_id, num, name=None):
             apply_to_all = session.type_id == "regular"
             if show_apply_to_all_checkbox:
                 apply_to_all = form.cleaned_data["apply_to_all"]
+            if can_manage:
+                approved = form.cleaned_data["approved"]
+            else:
+                approved = False
+
+            # Propose slides if not auto-approved
+            if not approved:
+                title = form.cleaned_data['title']
+                submission = SlideSubmission.objects.create(session = session, title = title, filename = '', apply_to_all = apply_to_all, submitter=request.user.person)
+
+                if session.meeting.type_id=='ietf':
+                    name = 'slides-%s-%s' % (session.meeting.number, 
+                                         session.group.acronym) 
+                    if not apply_to_all:
+                        name += '-%s' % (session.docname_token(),)
+                else:
+                    name = 'slides-%s-%s' % (session.meeting.number, session.docname_token())
+                name = name + '-' + slugify(title).replace('_', '-')[:128]
+                filename = '%s-ss%d%s'% (name, submission.id, ext)
+                destination = io.open(os.path.join(settings.SLIDE_STAGING_PATH, filename),'wb+')
+                for chunk in file.chunks():
+                    destination.write(chunk)
+                destination.close()
+
+                submission.filename = filename
+                submission.save()
+
+                (to, cc) = gather_address_lists('slides_proposed', group=session.group, proposer=request.user.person).as_strings()
+                msg_txt = render_to_string("meeting/slides_proposed.txt", {
+                        "to": to,
+                        "cc": cc,
+                        "submission": submission,
+                        "settings": settings,
+                     })
+                msg = infer_message(msg_txt)
+                msg.by = request.user.person
+                msg.save()
+                send_mail_message(request, msg)
+                messages.success(request, 'Successfully submitted proposed slides.')
+                return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
 
             # Handle creation / update of the Document (but do not save yet)
             if doc is not None:
@@ -3076,7 +3104,7 @@ def upload_session_slides(request, session_id, num, name=None):
         initial = {}
         if doc is not None:
             initial = {"title": doc.title}
-        form = UploadSlidesForm(session, show_apply_to_all_checkbox, initial=initial)
+        form = UploadSlidesForm(session, show_apply_to_all_checkbox, can_manage, initial=initial)
 
     return render(
         request,
@@ -3085,77 +3113,12 @@ def upload_session_slides(request, session_id, num, name=None):
             "session": session,
             "session_number": session_number,
             "slides_sp": session.presentations.filter(document=doc).first() if doc else None,
+            "manage": session.can_manage_materials(request.user),
             "form": form,
         },
     )
 
 
-@login_required
-def propose_session_slides(request, session_id, num):
-    session = get_object_or_404(Session,pk=session_id)
-    if session.is_material_submission_cutoff() and not has_role(request.user, "Secretariat"):
-        permission_denied(request, "The materials cutoff for this session has passed. Contact the secretariat for further action.")
-
-    session_number = None
-    sessions = get_sessions(session.meeting.number,session.group.acronym)
-    show_apply_to_all_checkbox = len(sessions) > 1 if session.type_id == 'regular' else False
-    if len(sessions) > 1:
-       session_number = 1 + sessions.index(session)
-
-    
-    if request.method == 'POST':
-        form = UploadSlidesForm(session, show_apply_to_all_checkbox,request.POST,request.FILES)
-        if form.is_valid():
-            file = request.FILES['file']
-            _, ext = os.path.splitext(file.name)
-            apply_to_all = session.type_id == 'regular'
-            if show_apply_to_all_checkbox:
-                apply_to_all = form.cleaned_data['apply_to_all']
-            title = form.cleaned_data['title']
-
-            submission = SlideSubmission.objects.create(session = session, title = title, filename = '', apply_to_all = apply_to_all, submitter=request.user.person)
-
-            if session.meeting.type_id=='ietf':
-                name = 'slides-%s-%s' % (session.meeting.number, 
-                                         session.group.acronym) 
-                if not apply_to_all:
-                    name += '-%s' % (session.docname_token(),)
-            else:
-                name = 'slides-%s-%s' % (session.meeting.number, session.docname_token())
-            name = name + '-' + slugify(title).replace('_', '-')[:128]
-            filename = '%s-ss%d%s'% (name, submission.id, ext)
-            destination = io.open(os.path.join(settings.SLIDE_STAGING_PATH, filename),'wb+')
-            for chunk in file.chunks():
-                destination.write(chunk)
-            destination.close()
-
-            submission.filename = filename
-            submission.save()
-
-            (to, cc) = gather_address_lists('slides_proposed', group=session.group, proposer=request.user.person).as_strings()
-            msg_txt = render_to_string("meeting/slides_proposed.txt", {
-                    "to": to,
-                    "cc": cc,
-                    "submission": submission,
-                    "settings": settings,
-                 })
-            msg = infer_message(msg_txt)
-            msg.by = request.user.person
-            msg.save()
-            send_mail_message(request, msg)
-            messages.success(request, 'Successfully submitted proposed slides.')
-            return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
-    else: 
-        initial = {}
-        form = UploadSlidesForm(session, show_apply_to_all_checkbox, initial=initial)
-
-    return render(request, "meeting/propose_session_slides.html", 
-                  {'session': session,
-                   'session_number': session_number,
-                   'form': form,
-                  })
-
-
 def remove_sessionpresentation(request, session_id, num, name):
     sp = get_object_or_404(
         SessionPresentation, session_id=session_id, document__name=name
@@ -5072,6 +5035,7 @@ def approve_proposed_slides(request, slidesubmission_id, num):
                     "cc": cc,
                     "submission": submission,
                     "settings": settings,
+                    "approver": request.user.person
                 })
                 send_mail_text(request, to, None, subject, body, cc=cc)
                 return redirect('ietf.meeting.views.session_details',num=num,acronym=acronym)
diff --git a/ietf/templates/meeting/propose_session_slides.html b/ietf/templates/meeting/propose_session_slides.html
deleted file mode 100644
index e5a0b451e6..0000000000
--- a/ietf/templates/meeting/propose_session_slides.html
+++ /dev/null
@@ -1,27 +0,0 @@
-{% extends "base.html" %}
-{# Copyright The IETF Trust 2015, All Rights Reserved #}
-{% load origin static django_bootstrap5 tz %}
-{% block title %}Propose Slides for {{ session.meeting }} : {{ session.group.acronym }}{% endblock %}
-{% block content %}
-    {% origin %}
-    <h1>
-        Propose Slides for {{ session.meeting }}
-        <br>
-        <small class="text-body-secondary">{{ session.group.acronym }}
-            {% if session.name %}: {{ session.name }}{% endif %}
-        </small>
-    </h1>
-    {% if session_number %}
-        <h2 class="mt-3">
-            Session {{ session_number }} : {{ session.official_timeslotassignment.timeslot.time|timezone:session.meeting.time_zone|date:"D M-d-Y Hi" }}
-        </h2>
-    {% endif %}
-    <p class="alert alert-info my-3">
-        This form will allow you to propose a slide deck to the session chairs. After you upload your proposal, mail will be sent to the session chairs asking for their approval.
-    </p>
-    <form enctype="multipart/form-data" method="post">
-        {% csrf_token %}
-        {% bootstrap_form form %}
-        <button type="submit" class="btn btn-primary">Upload</button>
-    </form>
-{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/meeting/session_details_panel.html b/ietf/templates/meeting/session_details_panel.html
index d053ba1c1c..1dcbded913 100644
--- a/ietf/templates/meeting/session_details_panel.html
+++ b/ietf/templates/meeting/session_details_panel.html
@@ -187,7 +187,7 @@ <h3 class="mt-4">Slides</h3>
             </a>
         {% elif request.user.is_authenticated and not session.is_material_submission_cutoff %}
             <a class="btn btn-primary proposeslides"
-               href="{% url 'ietf.meeting.views.propose_session_slides' session_id=session.pk num=session.meeting.number %}">
+               href="{% url 'ietf.meeting.views.upload_session_slides' session_id=session.pk num=session.meeting.number %}">
                 Propose slides
             </a>
         {% endif %}
diff --git a/ietf/templates/meeting/slides_approved.txt b/ietf/templates/meeting/slides_approved.txt
index db288ad853..61ffafcd18 100644
--- a/ietf/templates/meeting/slides_approved.txt
+++ b/ietf/templates/meeting/slides_approved.txt
@@ -1,4 +1,4 @@
-{% load ietf_filters %}{% autoescape off %}Your proposed slides have been approved for {{ submission.session.meeting }} : {{ submission.session.group.acronym }}{% if submission.session.name %} : {{submission.session.name}}{% endif %}
+{% load ietf_filters %}{% autoescape off %}Your proposed slides have been approved for {{ submission.session.meeting }} : {{ submission.session.group.acronym }}{% if submission.session.name %} : {{submission.session.name}}{% endif %} by {{approver}}
 
 Title: {{submission.title}}
 
diff --git a/ietf/templates/meeting/upload_session_slides.html b/ietf/templates/meeting/upload_session_slides.html
index 8e3e064df3..059ffae16f 100644
--- a/ietf/templates/meeting/upload_session_slides.html
+++ b/ietf/templates/meeting/upload_session_slides.html
@@ -17,15 +17,21 @@ <h1>
         {% else %}
             Upload new
         {% endif %}
-        slides for {{ session.meeting }}
-        <br>
+        slides for {{ session.meeting }}        <br>
         <small class="text-body-secondary">
             {{ session.group.acronym }}
             {% if session.name %}: {{ session.name }}{% endif %}
         </small>
     </h1>
     {% if session_number %}
-        <h2>Session {{ session_number }} : {{ session.official_timeslotassignment.timeslot.time|timezone:session.meeting.time_zone|date:"D M-d-Y Hi" }}</h2>
+        <h2 class="mt-3">
+            Session {{ session_number }} : {{ session.official_timeslotassignment.timeslot.time|timezone:session.meeting.time_zone|date:"D M-d-Y Hi" }}
+        </h2>
+    {% endif %}
+    {% if not manage %}
+    <p class="alert alert-info my-3">
+        This form will allow you to propose a slide deck to the session chairs. After you upload your proposal, mail will be sent to the session chairs asking for their approval.
+    </p>
     {% endif %}
     {% if slides_sp %}<h3>{{ slides_sp.document.name }}</h3>{% endif %}
     <form class="my-3" enctype="multipart/form-data" method="post">

From b90820eb7a74ecaa417df120229ca3f798602e69 Mon Sep 17 00:00:00 2001
From: Sangho Na <sangho@nsh.nz>
Date: Thu, 8 Aug 2024 02:15:01 +1200
Subject: [PATCH 033/656] fix: Hide last modified field in agenda when
 unavailable (#7722)

* test: Update tests to check for Updated field in agenda.txt

* fix: Hide Updated in agenda.txt if too old

* test: Remove confusing tests on CSV agenda

* refactor: Make updated() return None when no valid timestamp found

* refactor: Remove walrus operator
---
 client/agenda/Agenda.vue          |  2 +-
 ietf/meeting/models.py            | 13 ++++----
 ietf/meeting/tests_views.py       | 50 ++++++++++++++++++++++++++++++-
 ietf/meeting/utils.py             |  3 +-
 ietf/templates/meeting/agenda.txt |  2 ++
 5 files changed, 61 insertions(+), 9 deletions(-)

diff --git a/client/agenda/Agenda.vue b/client/agenda/Agenda.vue
index 34a3751f23..40dae008a4 100644
--- a/client/agenda/Agenda.vue
+++ b/client/agenda/Agenda.vue
@@ -323,7 +323,7 @@ const meetingUpdated = computed(() => {
   if (!agendaStore.meeting.updated) { return false }
   
   const updatedDatetime = DateTime.fromISO(agendaStore.meeting.updated).setZone(agendaStore.timezone)
-  if (!updatedDatetime.isValid || updatedDatetime < DateTime.fromISO('1980-01-01')) {
+  if (!updatedDatetime.isValid) {
     return false
   }
   
diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index dd6e2db6c5..fa1ad9d674 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -369,13 +369,14 @@ def vtimezone(self):
 
     def updated(self):
         # should be Meeting.modified, but we don't have that
-        min_time = pytz.utc.localize(datetime.datetime(1970, 1, 1, 0, 0, 0))
-        timeslots_updated = self.timeslot_set.aggregate(Max('modified'))["modified__max"] or min_time
-        sessions_updated = self.session_set.aggregate(Max('modified'))["modified__max"] or min_time
-        assignments_updated = min_time
+        timeslots_updated = self.timeslot_set.aggregate(Max('modified'))["modified__max"]
+        sessions_updated = self.session_set.aggregate(Max('modified'))["modified__max"]
+        assignments_updated = None
         if self.schedule:
-            assignments_updated = SchedTimeSessAssignment.objects.filter(schedule__in=[self.schedule, self.schedule.base if self.schedule else None]).aggregate(Max('modified'))["modified__max"] or min_time
-        return max(timeslots_updated, sessions_updated, assignments_updated)
+            assignments_updated = SchedTimeSessAssignment.objects.filter(schedule__in=[self.schedule, self.schedule.base if self.schedule else None]).aggregate(Max('modified'))["modified__max"]
+        dts = [timeslots_updated, sessions_updated, assignments_updated]
+        valid_only = [dt for dt in dts if dt is not None]
+        return max(valid_only) if valid_only else None
 
     @memoize
     def previous_meeting(self):
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index da82afb329..60eef96b2d 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -294,6 +294,8 @@ def test_meeting_agenda(self):
             (slot.time + slot.duration).astimezone(meeting.tz()).strftime("%H%M"),
         ))
         self.assertContains(r, f"shown in the {meeting.tz()} time zone")
+        updated = meeting.updated().astimezone(meeting.tz()).strftime("%Y-%m-%d %H:%M:%S %Z")
+        self.assertContains(r, f"Updated {updated}")
 
         # text, UTC
         r = self.client.get(urlreverse(
@@ -309,6 +311,16 @@ def test_meeting_agenda(self):
             (slot.time + slot.duration).astimezone(datetime.timezone.utc).strftime("%H%M"),
         ))
         self.assertContains(r, "shown in UTC")
+        updated = meeting.updated().astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z")
+        self.assertContains(r, f"Updated {updated}")
+
+        # text, invalid updated (none)
+        with patch("ietf.meeting.models.Meeting.updated", return_value=None):
+            r = self.client.get(urlreverse(
+                "ietf.meeting.views.agenda_plain",
+                kwargs=dict(num=meeting.number, ext=".txt", utc="-utc"),
+            ))
+            self.assertNotContains(r, "Updated ")
 
         # future meeting, no agenda
         r = self.client.get(urlreverse("ietf.meeting.views.agenda_plain", kwargs=dict(num=future_meeting.number, ext=".txt")))
@@ -859,6 +871,24 @@ def test_important_dates_ical(self):
         for d in meeting.importantdate_set.all():
             self.assertContains(r, d.date.isoformat())
 
+        updated = meeting.updated()
+        self.assertIsNotNone(updated)
+        expected_updated = updated.astimezone(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+        self.assertContains(r, f"DTSTAMP:{expected_updated}")
+        dtstamps_count = r.content.decode("utf-8").count(f"DTSTAMP:{expected_updated}")
+        self.assertEqual(dtstamps_count, meeting.importantdate_set.count())
+
+        # With default cached_updated, 1970-01-01
+        with patch("ietf.meeting.models.Meeting.updated", return_value=None):
+            r = self.client.get(url)
+            for d in meeting.importantdate_set.all():
+                self.assertContains(r, d.date.isoformat())
+
+            expected_updated = "19700101T000000Z"
+            self.assertContains(r, f"DTSTAMP:{expected_updated}")
+            dtstamps_count = r.content.decode("utf-8").count(f"DTSTAMP:{expected_updated}")
+            self.assertEqual(dtstamps_count, meeting.importantdate_set.count())
+
     def test_group_ical(self):
         meeting = make_meeting_test_data()
         s1 = Session.objects.filter(meeting=meeting, group__acronym="mars").first()
@@ -4952,7 +4982,23 @@ def test_upcoming_ical(self):
                                       expected_event_count=len(expected_event_summaries))
         self.assertNotContains(r, 'Remote instructions:')
 
-    def test_upcoming_ical_filter(self):
+        updated = meeting.updated()
+        self.assertIsNotNone(updated)
+        expected_updated = updated.astimezone(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+        self.assertContains(r, f"DTSTAMP:{expected_updated}")
+
+        # With default cached_updated, 1970-01-01
+        with patch("ietf.meeting.models.Meeting.updated", return_value=None):
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+
+            self.assertEqual(meeting.type_id, "ietf")
+
+            expected_updated = "19700101T000000Z"
+            self.assertEqual(1, r.content.decode("utf-8").count(f"DTSTAMP:{expected_updated}"))
+
+    @patch("ietf.meeting.utils.preprocess_meeting_important_dates")
+    def test_upcoming_ical_filter(self, mock_preprocess_meeting_important_dates):
         # Just a quick check of functionality - details tested by test_js.InterimTests
         make_meeting_test_data(create_interims=True)
         url = urlreverse("ietf.meeting.views.upcoming_ical")
@@ -4974,6 +5020,8 @@ def test_upcoming_ical_filter(self):
                                       ],
                                       expected_event_count=2)
 
+        # Verify preprocess_meeting_important_dates isn't being called
+        mock_preprocess_meeting_important_dates.assert_not_called()
 
     def test_upcoming_json(self):
         make_meeting_test_data(create_interims=True)
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index a60d3b010a..470795811d 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -609,7 +609,8 @@ def bulk_create_timeslots(meeting, times, locations, other_props):
 
 def preprocess_meeting_important_dates(meetings):
     for m in meetings:
-        m.cached_updated = m.updated()
+        # cached_updated must be present, set it to 1970-01-01 if necessary
+        m.cached_updated = m.updated() or pytz.utc.localize(datetime.datetime(1970, 1, 1, 0, 0, 0))
         m.important_dates = m.importantdate_set.prefetch_related("name")
         for d in m.important_dates:
             d.midnight_cutoff = "UTC 23:59" in d.name.name
diff --git a/ietf/templates/meeting/agenda.txt b/ietf/templates/meeting/agenda.txt
index d00be80d14..7a49dde0c8 100644
--- a/ietf/templates/meeting/agenda.txt
+++ b/ietf/templates/meeting/agenda.txt
@@ -7,7 +7,9 @@
 {% filter center:72 %}{{ schedule.meeting.agenda_info_note|striptags|wordwrap:72|safe }}{% endfilter %}
 {% endif %}
 {% filter center:72 %}{{ schedule.meeting.date|date:"F j" }}-{% if schedule.meeting.date.month != schedule.meeting.end_date.month %}{{ schedule.meeting.end_date|date:"F " }}{% endif %}{{ schedule.meeting.end_date|date:"j, Y" }}{% endfilter %}
+{% if updated %}
 {% filter center:72 %}Updated {{ updated|date:"Y-m-d H:i:s T" }}{% endfilter %}
+{% endif %}
 
 {% filter center:72 %}IETF agendas are subject to change, up to and during the meeting.{% endfilter %}
 {% filter center:72 %}Times are shown in {% if display_timezone.lower == "utc" %}UTC{% else %}the {{ display_timezone }} time zone{% endif %}.{% endfilter %}

From 95a7e14ada82fd43eafed7218a28ba0554c81e69 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 7 Aug 2024 11:16:40 -0300
Subject: [PATCH 034/656] feat: dev mode admin + refactor api init (#7628)

* feat: style admin site in dev mode

* refactor: eliminate base_site.html

* fix: remove debug

* fix: commit missing __init__.py

* refactor: make method static; fix tests

* refactor: move api init to AppConfig.ready()

Avoids interacting with the app registry before
it's ready.
---
 ietf/admin/__init__.py         |  0
 ietf/admin/apps.py             |  6 +++++
 ietf/admin/sites.py            | 15 ++++++++++++
 ietf/api/__init__.py           | 43 ++++++++++++++--------------------
 ietf/api/__init__.pyi          |  1 +
 ietf/api/apps.py               | 15 ++++++++++++
 ietf/api/urls.py               |  1 +
 ietf/settings.py               |  2 +-
 ietf/templates/admin/base.html | 27 +++++++++++++++++++++
 ietf/urls.py                   |  2 --
 ietf/utils/tests.py            |  3 ++-
 11 files changed, 85 insertions(+), 30 deletions(-)
 create mode 100644 ietf/admin/__init__.py
 create mode 100644 ietf/admin/apps.py
 create mode 100644 ietf/admin/sites.py
 create mode 100644 ietf/api/apps.py
 create mode 100644 ietf/templates/admin/base.html

diff --git a/ietf/admin/__init__.py b/ietf/admin/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/ietf/admin/apps.py b/ietf/admin/apps.py
new file mode 100644
index 0000000000..20b762cfec
--- /dev/null
+++ b/ietf/admin/apps.py
@@ -0,0 +1,6 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+from django.contrib.admin import apps as admin_apps
+
+
+class AdminConfig(admin_apps.AdminConfig):
+    default_site = "ietf.admin.sites.AdminSite" 
diff --git a/ietf/admin/sites.py b/ietf/admin/sites.py
new file mode 100644
index 0000000000..69cb62ae20
--- /dev/null
+++ b/ietf/admin/sites.py
@@ -0,0 +1,15 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+from django.contrib.admin import AdminSite as _AdminSite
+from django.conf import settings
+from django.utils.safestring import mark_safe
+
+
+class AdminSite(_AdminSite):
+    site_title = "Datatracker admin"
+    
+    @staticmethod
+    def site_header():
+        if settings.SERVER_MODE == "production":
+            return "Datatracker administration"
+        else:
+            return mark_safe('Datatracker administration <span class="text-danger">&delta;</span>')
diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py
index 54b4b7424b..81f370121f 100644
--- a/ietf/api/__init__.py
+++ b/ietf/api/__init__.py
@@ -7,8 +7,10 @@
 
 from urllib.parse import urlencode
 
-from django.conf import settings
+from django.apps import apps as django_apps
 from django.core.exceptions import ObjectDoesNotExist
+from django.utils.module_loading import autodiscover_modules
+
 
 import debug                            # pyflakes:ignore
 
@@ -19,40 +21,29 @@
 from tastypie.exceptions import ApiFieldError
 from tastypie.fields import ApiField
 
+
 _api_list = []
 
-for _app in settings.INSTALLED_APPS:
-    _module_dict = globals()
-    if '.' in _app:
-        _root, _name = _app.split('.', 1)
-        if _root == 'ietf':
-            if not '.' in _name:
-                _api = Api(api_name=_name)
-                _module_dict[_name] = _api
-                _api_list.append((_name, _api))
+
+def populate_api_list():
+    for app_config in django_apps.get_app_configs():
+        _module_dict = globals()
+        if '.' in app_config.name:
+            _root, _name = app_config.name.split('.', 1)
+            if _root == 'ietf':
+                if not '.' in _name:
+                    _api = Api(api_name=_name)
+                    _module_dict[_name] = _api
+                    _api_list.append((_name, _api))
 
 def autodiscover():
     """
     Auto-discover INSTALLED_APPS resources.py modules and fail silently when
-    not present. This forces an import on them to register any admin bits they
+    not present. This forces an import on them to register any resources they
     may want.
     """
+    autodiscover_modules("resources")
 
-    from importlib import import_module
-    from django.conf import settings
-    from django.utils.module_loading import module_has_submodule
-
-    for app in settings.INSTALLED_APPS:
-        mod = import_module(app)
-        # Attempt to import the app's admin module.
-        try:
-            import_module('%s.resources' % (app, ))
-        except:
-            # Decide whether to bubble up this error. If the app just
-            # doesn't have an admin module, we can ignore the error
-            # attempting to import it, otherwise we want it to bubble up.
-            if module_has_submodule(mod, "resources"):
-                raise
 
 class ModelResource(tastypie.resources.ModelResource):
     def generate_cache_key(self, *args, **kwargs):
diff --git a/ietf/api/__init__.pyi b/ietf/api/__init__.pyi
index 63d9bc513b..ededea90a7 100644
--- a/ietf/api/__init__.pyi
+++ b/ietf/api/__init__.pyi
@@ -30,4 +30,5 @@ class Serializer(): ...
 class ToOneField(tastypie.fields.ToOneField): ...
 class TimedeltaField(tastypie.fields.ApiField): ...
 
+def populate_api_list() -> None: ...
 def autodiscover() -> None: ...
diff --git a/ietf/api/apps.py b/ietf/api/apps.py
new file mode 100644
index 0000000000..7eca094a62
--- /dev/null
+++ b/ietf/api/apps.py
@@ -0,0 +1,15 @@
+from django.apps import AppConfig
+from . import populate_api_list
+
+
+class ApiConfig(AppConfig):
+    name = "ietf.api"
+
+    def ready(self):
+        """Hook to do init after the app registry is fully populated
+        
+        Importing models or accessing the app registry is ok here, but do not
+        interact with the database. See        
+        https://docs.djangoproject.com/en/4.2/ref/applications/#django.apps.AppConfig.ready
+        """
+        populate_api_list()
diff --git a/ietf/api/urls.py b/ietf/api/urls.py
index fb2184a3f0..3c0fb872c9 100644
--- a/ietf/api/urls.py
+++ b/ietf/api/urls.py
@@ -11,6 +11,7 @@
 from ietf.submit import views as submit_views
 from ietf.utils.urls import url
 
+
 api.autodiscover()
 
 urlpatterns = [
diff --git a/ietf/settings.py b/ietf/settings.py
index 13b7506673..7572b15213 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -436,7 +436,7 @@ def skip_unreadable_post(record):
 
 INSTALLED_APPS = [
     # Django apps
-    'django.contrib.admin',
+    'ietf.admin',  # replaces django.contrib.admin
     'django.contrib.admindocs',
     'django.contrib.auth',
     'django.contrib.contenttypes',
diff --git a/ietf/templates/admin/base.html b/ietf/templates/admin/base.html
new file mode 100644
index 0000000000..9ca7377a54
--- /dev/null
+++ b/ietf/templates/admin/base.html
@@ -0,0 +1,27 @@
+{% extends 'admin/base.html' %}
+{% load static %}
+{% block extrastyle %}{{ block.super }}
+    {% if server_mode and server_mode != "production" %}
+        <style>
+            {# grab colors that match bootstrap so we don't have to import the css #}
+            html, :root{
+                --bs-danger-bg-subtle: #F8D7DAFF; 
+                --bs-danger-text-emphasis: #58151CFF;
+                --bs-danger: #DC3545FF;
+                --bs-secondary: #6C757DFF;
+                --bs-primary-text-emphasis: #052C65FF;
+            }
+            html[data-theme="light"], :root {
+                --primary: var(--bs-danger-bg-subtle);
+                --secondary: var(--bs-danger-bg-subtle);
+                --accent: var(--bs-danger-text-emphasis);
+                --primary-fg: var(--bs-primary-text-emphasis);
+                --link-fg: var(--bs-danger-text-emphasis);
+                --header-color: var(--bs-secondary);
+                --breadcrumbs-fg: var(--bs-secondary);
+                --breadcrumbs-link-fg: var(--link-fg);
+            }
+            span.text-danger { color: var(--bs-danger); }
+        </style>
+    {% endif %}
+{% endblock %}
diff --git a/ietf/urls.py b/ietf/urls.py
index 4b29a3aa81..90b161b530 100644
--- a/ietf/urls.py
+++ b/ietf/urls.py
@@ -20,8 +20,6 @@
 from ietf.utils.urls import url
 
 
-admin.autodiscover()
-
 # sometimes, this code gets called more than once, which is an
 # that seems impossible to work around.
 try:
diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py
index 476c257a38..d435583e89 100644
--- a/ietf/utils/tests.py
+++ b/ietf/utils/tests.py
@@ -36,6 +36,7 @@
 
 import debug                            # pyflakes:ignore
 
+from ietf.admin.sites import AdminSite
 from ietf.person.name import name_parts, unidecode_name
 from ietf.submit.tests import submission_file
 from ietf.utils.draft import PlaintextDraft, getmeta
@@ -325,7 +326,7 @@ def test_all_model_admins_exist(self):
         User.objects.create_superuser('admin', 'admin@example.org', 'admin+password')
         self.client.login(username='admin', password='admin+password')
         rtop = self.client.get("/admin/")
-        self.assertContains(rtop, 'Django administration')
+        self.assertContains(rtop, AdminSite.site_header())
         for name in self.apps:
             app_name = self.apps[name]
             self.assertContains(rtop, name)

From 1de41965bee819d85b5b76f6a2db5c2e5934a6b6 Mon Sep 17 00:00:00 2001
From: Russ Housley <housley@vigilsec.com>
Date: Wed, 7 Aug 2024 10:18:13 -0400
Subject: [PATCH 035/656] fix: add celery tasks for idnits2 and bibxml file
 generation (#7204)

* Add celery tasks for idnits2 and bibxml file generation

* Update tests_tasks.py

Fix typo: bad_vakue -> bad_value

* Update utils.py

Don't raise error inbibxml_for_all_drafts

* chore: fixup merge

* chore: more merge cleanup

* chore: one last merge cleanup

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 ietf/doc/tasks.py       |   2 +
 ietf/doc/tests_tasks.py | 119 ++++++++++++++++++++++------------------
 ietf/doc/utils.py       |   3 +-
 3 files changed, 70 insertions(+), 54 deletions(-)

diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py
index 209db035a4..f1de459dd8 100644
--- a/ietf/doc/tasks.py
+++ b/ietf/doc/tasks.py
@@ -105,6 +105,8 @@ def generate_draft_bibxml_files_task(days=7, process_all=False):
     If process_all is False (the default), processes only docs with new revisions
     in the last specified number of days.
     """
+    if not process_all and days < 1:
+        raise ValueError("Must call with days >= 1 or process_all=True")
     ensure_draft_bibxml_path_exists()
     doc_events = NewRevisionDocEvent.objects.filter(
         type="new_revision",
diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py
index 51a8556e69..b75f58656b 100644
--- a/ietf/doc/tests_tasks.py
+++ b/ietf/doc/tests_tasks.py
@@ -22,8 +22,6 @@
 )
 
 class TaskTests(TestCase):
-    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ["DERIVED_DIR"]
-
     @mock.patch("ietf.doc.tasks.in_draft_expire_freeze")
     @mock.patch("ietf.doc.tasks.get_expired_drafts")
     @mock.patch("ietf.doc.tasks.expirable_drafts")
@@ -63,8 +61,8 @@ def test_expire_ids_task(
 
         # test that an exception is raised
         in_draft_expire_freeze_mock.side_effect = RuntimeError
-        with self.assertRaises(RuntimeError): (
-            expire_ids_task())
+        with self.assertRaises(RuntimeError):
+            expire_ids_task()
 
     @mock.patch("ietf.doc.tasks.send_expire_warning_for_draft")
     @mock.patch("ietf.doc.tasks.get_soon_to_expire_drafts")
@@ -98,37 +96,42 @@ def test_expire_last_calls_task(self, mock_get_expired, mock_expire):
         self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1]))
         self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2]))
 
-    @mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status")
-    def test_generate_idnits2_rfc_status_task(self, mock_generate):
+
+class Idnits2SupportTests(TestCase):
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR']
+
+    @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted")
+    def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate):
         mock_generate.return_value = "dåtå"
-        generate_idnits2_rfc_status_task()
+        generate_idnits2_rfcs_obsoleted_task()
         self.assertEqual(mock_generate.call_count, 1)
         self.assertEqual(
             "dåtå".encode("utf8"),
-            (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").read_bytes(),
+            (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(),
         )
-    
-    @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted")
-    def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate):
+
+    @mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status")
+    def test_generate_idnits2_rfc_status_task(self, mock_generate):
         mock_generate.return_value = "dåtå"
-        generate_idnits2_rfcs_obsoleted_task()
+        generate_idnits2_rfc_status_task()
         self.assertEqual(mock_generate.call_count, 1)
         self.assertEqual(
             "dåtå".encode("utf8"),
-            (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(),
+            (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").read_bytes(),
         )
 
-    @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists")
-    @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file")
-    def test_generate_draft_bibxml_files_task(self, mock_create, mock_ensure_path):
+
+class BIBXMLSupportTests(TestCase):
+    def setUp(self):
+        super().setUp()
         now = timezone.now()
-        very_old_event = NewRevisionDocEventFactory(
+        self.very_old_event = NewRevisionDocEventFactory(
             time=now - datetime.timedelta(days=1000), rev="17"
         )
-        old_event = NewRevisionDocEventFactory(
+        self.old_event = NewRevisionDocEventFactory(
             time=now - datetime.timedelta(days=8), rev="03"
         )
-        young_event = NewRevisionDocEventFactory(
+        self.young_event = NewRevisionDocEventFactory(
             time=now - datetime.timedelta(days=6), rev="06"
         )
         # a couple that should always be ignored
@@ -141,62 +144,72 @@ def test_generate_draft_bibxml_files_task(self, mock_create, mock_ensure_path):
             rev="09",
             doc__type_id="rfc",
         )
-    
         # Get rid of the "00" events created by the factories -- they're just noise for this test
         NewRevisionDocEvent.objects.filter(rev="00").delete()
-    
-        # default args - look back 7 days
-        generate_draft_bibxml_files_task()
-        self.assertTrue(mock_ensure_path.called)
-        self.assertCountEqual(
-            mock_create.call_args_list, [mock.call(young_event.doc, young_event.rev)]
-        )
-        mock_create.reset_mock()
-        mock_ensure_path.reset_mock()
-    
-        # shorter lookback
-        generate_draft_bibxml_files_task(days=5)
-        self.assertTrue(mock_ensure_path.called)
-        self.assertCountEqual(mock_create.call_args_list, [])
-        mock_create.reset_mock()
-        mock_ensure_path.reset_mock()
-    
-        # longer lookback
-        generate_draft_bibxml_files_task(days=9)
+
+    @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists")
+    @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file")
+    def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensure_path):
+        generate_draft_bibxml_files_task(process_all=True)
         self.assertTrue(mock_ensure_path.called)
         self.assertCountEqual(
             mock_create.call_args_list,
             [
-                mock.call(young_event.doc, young_event.rev),
-                mock.call(old_event.doc, old_event.rev),
+                mock.call(self.young_event.doc, self.young_event.rev),
+                mock.call(self.old_event.doc, self.old_event.rev),
+                mock.call(self.very_old_event.doc, self.very_old_event.rev),
             ],
         )
         mock_create.reset_mock()
         mock_ensure_path.reset_mock()
-    
-        # everything
+
+        # everything should still be tried, even if there's an exception
+        mock_create.side_effect = RuntimeError
         generate_draft_bibxml_files_task(process_all=True)
         self.assertTrue(mock_ensure_path.called)
         self.assertCountEqual(
             mock_create.call_args_list,
             [
-                mock.call(young_event.doc, young_event.rev),
-                mock.call(old_event.doc, old_event.rev),
-                mock.call(very_old_event.doc, very_old_event.rev),
+                mock.call(self.young_event.doc, self.young_event.rev),
+                mock.call(self.old_event.doc, self.old_event.rev),
+                mock.call(self.very_old_event.doc, self.very_old_event.rev),
             ],
         )
+
+    @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists")
+    @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file")
+    def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_ensure_path):
+        # default args - look back 7 days
+        generate_draft_bibxml_files_task()
+        self.assertTrue(mock_ensure_path.called)
+        self.assertCountEqual(
+            mock_create.call_args_list, [mock.call(self.young_event.doc, self.young_event.rev)]
+        )
         mock_create.reset_mock()
         mock_ensure_path.reset_mock()
-    
-        # everything should still be tried, even if there's an exception
-        mock_create.side_effect = RuntimeError
-        generate_draft_bibxml_files_task(process_all=True)
+
+        # shorter lookback
+        generate_draft_bibxml_files_task(days=5)
+        self.assertTrue(mock_ensure_path.called)
+        self.assertCountEqual(mock_create.call_args_list, [])
+        mock_create.reset_mock()
+        mock_ensure_path.reset_mock()
+
+        # longer lookback
+        generate_draft_bibxml_files_task(days=9)
         self.assertTrue(mock_ensure_path.called)
         self.assertCountEqual(
             mock_create.call_args_list,
             [
-                mock.call(young_event.doc, young_event.rev),
-                mock.call(old_event.doc, old_event.rev),
-                mock.call(very_old_event.doc, very_old_event.rev),
+                mock.call(self.young_event.doc, self.young_event.rev),
+                mock.call(self.old_event.doc, self.old_event.rev),
             ],
         )
+
+    @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists")
+    @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file")
+    def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value(self, mock_create, mock_ensure_path):
+        with self.assertRaises(ValueError):
+            generate_draft_bibxml_files_task(days=0)
+        self.assertFalse(mock_create.called)
+        self.assertFalse(mock_ensure_path.called)
diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index cd0fbb43b0..a98b46cb50 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2011-2020, All Rights Reserved
+# Copyright The IETF Trust 2011-2024, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -1228,6 +1228,7 @@ def fuzzy_find_documents(name, rev=None):
     FoundDocuments = namedtuple('FoundDocuments', 'documents matched_name matched_rev')
     return FoundDocuments(docs, name, rev)
 
+
 def bibxml_for_draft(doc, rev=None):
 
     if rev is not None and rev != doc.rev:

From 7d6d7e1c4460bdbe00647b89f25363cdc4a600b2 Mon Sep 17 00:00:00 2001
From: Russ Housley <housley@vigilsec.com>
Date: Wed, 7 Aug 2024 11:12:40 -0400
Subject: [PATCH 036/656] Update session_details_panel.html (#7719)

Add rows for interim meeting recordings. Fixes #7699.
---
 ietf/templates/meeting/session_details_panel.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/templates/meeting/session_details_panel.html b/ietf/templates/meeting/session_details_panel.html
index 1dcbded913..808da1438d 100644
--- a/ietf/templates/meeting/session_details_panel.html
+++ b/ietf/templates/meeting/session_details_panel.html
@@ -320,7 +320,7 @@ <h3 class="mt-4">Notes and recordings</h3>
                 </tr>
             {% endif %}
             {# Recordings #}
-            {% if meeting.number|add:"0" >= 80 %}
+            {% if meeting.type.slug == 'interim' or meeting.number|add:"0" >= 80 %}
                 {% with session.recordings as recordings %}
                     {% if recordings %}
                         {# There's no guaranteed order, so this is a bit messy: #}
@@ -370,4 +370,4 @@ <h3 class="mt-4">Notes and recordings</h3>
         </table>
     {% endif %}
 {% endwith %}{% endwith %}
-{% endfor %}
\ No newline at end of file
+{% endfor %}

From 4b912d55a540735418007c4d4751ed753dea431f Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 7 Aug 2024 12:14:39 -0300
Subject: [PATCH 037/656] fix: better handle "entered" agendas (#7721)

* fix: don't assume file has open method

The open method is specific to Django's uploaded
file classes, where it just calls seek(0)

* refactor: better upload/enter agenda abstraction
---
 ietf/meeting/utils.py | 22 ++++++++++++++--------
 ietf/meeting/views.py | 29 ++++++++++++++---------------
 2 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index 470795811d..e3d8c830ee 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -727,7 +727,7 @@ def save_session_minutes_revision(session, file, ext, request, encoding=None, ap
 def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=None):
     """Accept an uploaded materials file
 
-    This function takes a file object, a filename and a meeting object and subdir as string.
+    This function takes a _binary mode_ file object, a filename and a meeting object and subdir as string.
     It saves the file to the appropriate directory, get_materials_path() + subdir.
     If the file is a zip file, it creates a new directory in 'slides', which is the basename of the
     zip file and unzips the file in the new directory.
@@ -749,9 +749,18 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N
                 pass  # if the file is already gone, so be it
 
     with (path / filename).open('wb+') as destination:
+        # prep file for reading
+        if hasattr(file, "chunks"):
+            chunks = file.chunks()
+        else:
+            try:
+                file.seek(0)
+            except AttributeError:
+                pass
+            chunks = [file.read()]  # pretend we have chunks
+
         if filename.suffix in settings.MEETING_VALID_MIME_TYPE_EXTENSIONS['text/html']:
-            file.open()
-            text = file.read()
+            text = b"".join(chunks)
             if encoding:
                 try:
                     text = text.decode(encoding)
@@ -778,11 +787,8 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N
                                      f"please check the resulting content.  "
                                  ))
         else:
-            if hasattr(file, 'chunks'):
-                for chunk in file.chunks():
-                    destination.write(chunk)
-            else:
-                destination.write(file.read())
+            for chunk in chunks:
+                destination.write(chunk)
 
     # unzip zipfile
     if is_zipfile:
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index c3494a13ba..cf7c67ac4b 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -33,6 +33,7 @@
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
 from django.core.exceptions import ValidationError
+from django.core.files.uploadedfile import SimpleUploadedFile
 from django.core.validators import URLValidator
 from django.urls import reverse,reverse_lazy
 from django.db.models import F, Max, Q
@@ -2795,6 +2796,17 @@ def require_field(f):
         elif submission_method == "enter":
             require_field("content")
 
+    def get_file(self):
+        """Get content as a file-like object"""
+        if self.cleaned_data.get("submission_method") == "upload":
+            return self.cleaned_data["file"]
+        else:
+            return SimpleUploadedFile(
+                name="uploaded.md",
+                content=self.cleaned_data["content"].encode("utf-8"),
+                content_type="text/markdown;charset=utf-8",
+            )
+
 def upload_session_agenda(request, session_id, num):
     # num is redundant, but we're dragging it along an artifact of where we are in the current URL structure
     session = get_object_or_404(Session,pk=session_id)
@@ -2815,21 +2827,8 @@ def upload_session_agenda(request, session_id, num):
     if request.method == 'POST':
         form = UploadOrEnterAgendaForm(show_apply_to_all_checkbox,request.POST,request.FILES)
         if form.is_valid():
-            submission_method = form.cleaned_data['submission_method']
-            if submission_method == "upload":
-                file = request.FILES['file']
-                _, ext = os.path.splitext(file.name)
-            else:
-                if agenda_sp:
-                    doc = agenda_sp.document
-                    _, ext = os.path.splitext(doc.uploaded_filename)
-                else:
-                    ext = ".md"
-                fd, name = tempfile.mkstemp(suffix=ext, text=True)
-                os.close(fd)
-                with open(name, "w") as file:
-                    file.write(form.cleaned_data['content'])
-                file = open(name, "rb")
+            file = form.get_file()
+            _, ext = os.path.splitext(file.name)
             apply_to_all = session.type.slug == 'regular'
             if show_apply_to_all_checkbox:
                 apply_to_all = form.cleaned_data['apply_to_all']

From f921cdba5d81fb2c1152ee13da31cde10c5d685a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 7 Aug 2024 12:37:26 -0300
Subject: [PATCH 038/656] fix: disable nginx body size check (#7803)

* fix: increase nginx client_max_body_size

* fix: entirely disable nginx body size check
---
 k8s/nginx-datatracker.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/k8s/nginx-datatracker.conf b/k8s/nginx-datatracker.conf
index ff439fba6a..5cbc22e6c7 100644
--- a/k8s/nginx-datatracker.conf
+++ b/k8s/nginx-datatracker.conf
@@ -23,5 +23,6 @@ server {
         proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
         proxy_set_header X-Real-IP $${keepempty}remote_addr;
         proxy_pass http://localhost:8000;
+        client_max_body_size 0;  # disable size check
     }
 }

From 0c8db80b184b7bf1c12f34b6022bb5d2f24e072e Mon Sep 17 00:00:00 2001
From: Rich Salz <rsalz@akamai.com>
Date: Wed, 7 Aug 2024 12:23:18 -0400
Subject: [PATCH 039/656] fix: Show recordings for interims (#7197)

* fix: Show recordings for interims

Add methods uses_notes(), has_recordings(), and uses_chat_logs() to the
meeting object (with semantically correct tests) and use them consistently
throughout.  List the recordings if the "meeting numnber" starts with
"interim"

Fixes: #6543

* style: Use "is not" and "is" for None comparisons

* None comparison and non-IETF meetings

style: Use "is not None" instead of "!="
For non-IETF meetings assume chat logs exist

* fix: Restore useNotes for JS fields

* fix: uses_notes->useNotes (in JavaScript)

Also add comment about meeting number field in tests

* Missed a uses_notes->useNotes edit

* fix: useNotes->usesNotes

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 client/agenda/AgendaScheduleList.vue            |  2 +-
 client/agenda/store.js                          |  4 ++--
 ietf/doc/views_doc.py                           | 14 +++++++-------
 ietf/meeting/models.py                          | 17 ++++++++++++++++-
 ietf/meeting/tests_views.py                     |  2 +-
 ietf/meeting/views.py                           |  4 +---
 ietf/templates/group/meetings-row.html          |  4 ++--
 .../meeting/interim_session_buttons.html        |  2 +-
 .../meeting/session_buttons_include.html        | 12 ++++++------
 .../meeting/session_details_panel.html          |  8 ++++----
 ietf/templates/meeting/upcoming.html            |  2 +-
 playwright/helpers/meeting.js                   |  2 +-
 12 files changed, 43 insertions(+), 30 deletions(-)

diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index 9db763b74e..a6d2b9a5aa 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -296,7 +296,7 @@ const meetingEvents = computed(() => {
           color: 'red'
         })
       }
-      if (agendaStore.useNotes) {
+      if (agendaStore.usesNotes) {
         links.push({
           id: `lnk-${item.id}-note`,
           label: 'Notepad for note-takers',
diff --git a/client/agenda/store.js b/client/agenda/store.js
index 839d464c43..18c3e8c657 100644
--- a/client/agenda/store.js
+++ b/client/agenda/store.js
@@ -50,7 +50,7 @@ export const useAgendaStore = defineStore('agenda', {
     selectedCatSubs: [],
     settingsShown: false,
     timezone: DateTime.local().zoneName,
-    useNotes: false,
+    usesNotes: false,
     visibleDays: []
   }),
   getters: {
@@ -160,7 +160,7 @@ export const useAgendaStore = defineStore('agenda', {
         this.isCurrentMeeting = agendaData.isCurrentMeeting
         this.meeting = agendaData.meeting
         this.schedule = agendaData.schedule
-        this.useNotes = agendaData.useNotes
+        this.usesNotes = agendaData.usesNotes
 
         // -> Compute current info note hash
         this.infoNoteHash = murmur(agendaData.meeting.infoNote, 0).toString()
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 21c5eb235e..bd49275082 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -996,7 +996,7 @@ def document_raw_id(request, name, rev=None, ext=None):
     for t in possible_types:
         if os.path.exists(base_path + t):
             found_types[t]=base_path+t
-    if ext == None:
+    if ext is None:
         ext = 'txt'
     if not ext in found_types:
         raise Http404('dont have the file for that extension')
@@ -1227,7 +1227,7 @@ def document_bibtex(request, name, rev=None):
         raise Http404()
 
     # Make sure URL_REGEXPS did not grab too much for the rev number
-    if rev != None and len(rev) != 2:
+    if rev is not None and len(rev) != 2:
         mo = re.search(r"^(?P<m>[0-9]{1,2})-(?P<n>[0-9]{2})$", rev)
         if mo:
             name = name+"-"+mo.group(1)
@@ -1250,7 +1250,7 @@ def document_bibtex(request, name, rev=None):
         replaced_by = [d.name for d in doc.related_that("replaces")]
         draft_became_rfc = doc.became_rfc()
 
-        if rev != None and rev != doc.rev:
+        if rev is not None and rev != doc.rev:
             # find the entry in the history
             for h in doc.history_set.order_by("-time"):
                 if rev == h.rev:
@@ -1291,7 +1291,7 @@ def document_bibxml(request, name, rev=None):
         raise Http404()
 
     # Make sure URL_REGEXPS did not grab too much for the rev number
-    if rev != None and len(rev) != 2:
+    if rev is not None and len(rev) != 2:
         mo = re.search(r"^(?P<m>[0-9]{1,2})-(?P<n>[0-9]{2})$", rev)
         if mo:
             name = name+"-"+mo.group(1)
@@ -1439,7 +1439,7 @@ def document_referenced_by(request, name):
     if doc.type_id in ["bcp","std","fyi"]:
         for rfc in doc.contains():
             refs |= rfc.referenced_by()
-    full = ( request.GET.get('full') != None )
+    full = ( request.GET.get('full') is not None )
     numdocs = refs.count()
     if not full and numdocs>250:
        refs=refs[:250]
@@ -1459,7 +1459,7 @@ def document_ballot_content(request, doc, ballot_id, editable=True):
     augment_events_with_revision(doc, all_ballots)
 
     ballot = None
-    if ballot_id != None:
+    if ballot_id is not None:
         ballot_id = int(ballot_id)
         for b in all_ballots:
             if b.id == ballot_id:
@@ -1661,7 +1661,7 @@ def add_comment(request, name):
 
     login = request.user.person
 
-    if doc.type_id == "draft" and doc.group != None:
+    if doc.type_id == "draft" and doc.group is not None:
         can_add_comment = bool(has_role(request.user, ("Area Director", "Secretariat", "IRTF Chair", "IANA", "RFC Editor")) or (
             request.user.is_authenticated and
             Role.objects.filter(name__in=("chair", "secr"),
diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index fa1ad9d674..d8a069ef3c 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -383,7 +383,22 @@ def previous_meeting(self):
         return Meeting.objects.filter(type_id=self.type_id,date__lt=self.date).order_by('-date').first()
 
     def uses_notes(self):
-        return self.date>=datetime.date(2020,7,6)
+        if self.type_id != 'ietf':
+            return True
+        num = self.get_number()
+        return num is not None and num >= 108
+
+    def has_recordings(self):
+        if self.type_id != 'ietf':
+            return True
+        num = self.get_number()
+        return num is not None and num >= 80
+
+    def has_chat_logs(self):
+        if self.type_id != 'ietf':
+            return True;
+        num = self.get_number()
+        return num is not None and num >= 60
 
     def meeting_start(self):
         """Meeting-local midnight at the start of the meeting date"""
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 60eef96b2d..e4f62838de 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -259,7 +259,7 @@ def test_meeting_agenda(self):
                 },
                 "categories": rjson.get("categories"), # Just expect the value to exist
                 "isCurrentMeeting": True,
-                "useNotes": True,
+                "usesNotes": False, # make_meeting_test_data sets number=72
                 "schedule": rjson.get("schedule"), # Just expect the value to exist
                 "floors": []
             }
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index cf7c67ac4b..211cdec9a5 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -1617,7 +1617,6 @@ def agenda_plain(request, num=None, name=None, base=None, ext=None, owner=None,
                 "now": timezone.now().astimezone(meeting.tz()),
                 "display_timezone": display_timezone,
                 "is_current_meeting": is_current_meeting,
-                "use_notes": meeting.uses_notes(),
                 "cache_time": 150 if is_current_meeting else 3600,
             },
             content_type=mimetype[ext],
@@ -1692,7 +1691,7 @@ def api_get_agenda_data (request, num=None):
         },
         "categories": filter_organizer.get_filter_categories(),
         "isCurrentMeeting": is_current_meeting,
-        "useNotes": meeting.uses_notes(),
+        "usesNotes": meeting.uses_notes(),
         "schedule": list(map(agenda_extract_schedule, filtered_assignments)),
         "floors": list(map(agenda_extract_floorplan, floors))
     })
@@ -2489,7 +2488,6 @@ def session_details(request, num, acronym):
                     'can_manage_materials' : can_manage,
                     'can_view_request': can_view_request,
                     'thisweek': datetime_today()-datetime.timedelta(days=7),
-                    'use_notes': meeting.uses_notes(),
                   })
 
 class SessionDraftsForm(forms.Form):
diff --git a/ietf/templates/group/meetings-row.html b/ietf/templates/group/meetings-row.html
index 57c727eeae..3bbe1d425b 100644
--- a/ietf/templates/group/meetings-row.html
+++ b/ietf/templates/group/meetings-row.html
@@ -78,9 +78,9 @@
                 <div class="regular float-end">
                     {# see note in the included templates re: show_agenda parameter and required JS import #}
                     {% if s.meeting.type.slug == 'interim' %}
-                        {% include "meeting/interim_session_buttons.html" with show_agenda=False show_empty=False session=s meeting=s.meeting use_notes=s.meeting.use_notes %}
+                        {% include "meeting/interim_session_buttons.html" with show_agenda=False show_empty=False session=s meeting=s.meeting %}
                     {% else %}
-                        {% include "meeting/session_buttons_include.html" with show_agenda=False item=s.official_timeslotassignment session=s meeting=s.meeting use_notes=s.meeting.use_notes %}
+                        {% include "meeting/session_buttons_include.html" with show_agenda=False item=s.official_timeslotassignment session=s meeting=s.meeting %}
                     {% endif %}
                 </div>
             {% endif %}
diff --git a/ietf/templates/meeting/interim_session_buttons.html b/ietf/templates/meeting/interim_session_buttons.html
index 2f0951338d..a32f4345c9 100644
--- a/ietf/templates/meeting/interim_session_buttons.html
+++ b/ietf/templates/meeting/interim_session_buttons.html
@@ -34,7 +34,7 @@
             </a>
         {% endif %}
         {# notes #}
-        {% if use_notes %}
+        {% if session.agenda.uses_notes %}
             <a class="btn btn-outline-primary"
                href="{{ session.notes_url }}"
                aria-label="Notepad for note-takers"
diff --git a/ietf/templates/meeting/session_buttons_include.html b/ietf/templates/meeting/session_buttons_include.html
index 3dc9c45dcf..bb5144b45d 100644
--- a/ietf/templates/meeting/session_buttons_include.html
+++ b/ietf/templates/meeting/session_buttons_include.html
@@ -41,7 +41,7 @@
                     </a>
                 {% endif %}
                 {# Notes #}
-                {% if use_notes %}
+                {% if meeting.uses_notes %}
                     <a class="btn btn-outline-primary"
                         role="button"
                         href="{{ session.notes_url }}"
@@ -126,7 +126,7 @@
                     </a>
                 {% else %}
                     {# chat logs #}
-                    {% if meeting.number|add:"0" >= 60 %}
+                    {% if meeting.has_chat_logs %}
                         <a class="btn btn-outline-primary"
                             role="button"
                             href="{{session.chat_archive_url}}"
@@ -136,7 +136,7 @@
                         </a>
                     {% endif %}
                     {# Recordings #}
-                    {% if meeting.number|add:"0" >= 80 %}
+		    {% if meeting.has_recordings %}
                         {% with session.recordings as recordings %}
                             {% if recordings %}
                                 {# There's no guaranteed order, so this is a bit messy: #}
@@ -229,7 +229,7 @@
                         </li>
                     {% endif %}
                     {# Notes #}
-                    {% if use_notes %}
+                    {% if meeting.uses_notes %}
                         <li>
                             <a class="dropdown-item" href="{{ session.notes_url }}">
                                 <i class="bi bi-journal-text"></i> Notepad for note-takers
@@ -303,7 +303,7 @@
                         </li>
                     {% else %}
                         {# chat logs #}
-                        {% if meeting.number|add:"0" >= 60 %}
+			{% if meeting.has_chat_logs %}
                             <li>
                                 <a class="dropdown-item"
                                     href="session.chat_room_url">
@@ -312,7 +312,7 @@
                             </li>
                         {% endif %}
                         {# Recordings #}
-                        {% if meeting.number|add:"0" >= 80 %}
+                        {% if meeting.has_recordings %}
                             {% with session.recordings as recordings %}
                                 {% if recordings %}
                                     {# There's no guaranteed order, so this is a bit messy: #}
diff --git a/ietf/templates/meeting/session_details_panel.html b/ietf/templates/meeting/session_details_panel.html
index 808da1438d..52aeaaa8c1 100644
--- a/ietf/templates/meeting/session_details_panel.html
+++ b/ietf/templates/meeting/session_details_panel.html
@@ -9,7 +9,7 @@
                     {% if meeting.type.slug == 'interim' %}
                         {% include "meeting/interim_session_buttons.html" with show_agenda=False show_empty=False %}
                     {% else %}
-                        {% include "meeting/session_buttons_include.html" with show_agenda=False item=session.official_timeslotassignment use_notes=session.meeting.use_notes %}
+                        {% include "meeting/session_buttons_include.html" with show_agenda=False item=session.official_timeslotassignment %}
                     {% endif %}
                 </div>
             {% endif %}
@@ -230,7 +230,7 @@ <h3 class="mt-4">Meeting tools</h3>
         <table class="table table-sm table-striped meeting-tools"
                id="meeting_tools_{{ session.pk }}">
             <tbody>
-            {% if use_notes %}
+            {% if meeting.uses_notes %}
                 <tr>
                     <td>
                         <a href="{{ session.notes_url }}">
@@ -310,7 +310,7 @@ <h3 class="mt-4">Notes and recordings</h3>
         <table class="table table-sm table-striped meeting-tools"
                id="notes_and_recordings_{{ session.pk }}">
             <tbody>
-            {% if use_notes %}
+            {% if session.uses_notes %}
                 <tr>
                     <td>
                         <a href="{{ session.notes_url }}">
@@ -320,7 +320,7 @@ <h3 class="mt-4">Notes and recordings</h3>
                 </tr>
             {% endif %}
             {# Recordings #}
-            {% if meeting.type.slug == 'interim' or meeting.number|add:"0" >= 80 %}
+            {% if session.has_recordings %}
                 {% with session.recordings as recordings %}
                     {% if recordings %}
                         {# There's no guaranteed order, so this is a bit messy: #}
diff --git a/ietf/templates/meeting/upcoming.html b/ietf/templates/meeting/upcoming.html
index 802b1b03ca..13a27ed910 100644
--- a/ietf/templates/meeting/upcoming.html
+++ b/ietf/templates/meeting/upcoming.html
@@ -89,7 +89,7 @@ <h1>Upcoming Meetings</h1>
                                         <span class="badge rounded-pill text-bg-warning">Cancelled</span>
                                     </td>
                                 {% else %}
-                                    <td class="text-end">{% include "meeting/interim_session_buttons.html" with show_agenda=True use_notes=meeting.uses_notes %}</td>
+                                    <td class="text-end">{% include "meeting/interim_session_buttons.html" with show_agenda=True %}</td>
                                 {% endif %}
                             {% endwith %}
                         {% else %}
diff --git a/playwright/helpers/meeting.js b/playwright/helpers/meeting.js
index f07228b473..52bc331fd6 100644
--- a/playwright/helpers/meeting.js
+++ b/playwright/helpers/meeting.js
@@ -630,7 +630,7 @@ module.exports = {
       },
       categories,
       isCurrentMeeting: dateMode !== 'past',
-      useNotes: true,
+      usesNotes: true,
       schedule,
       floors
     }

From 30970749e30d89111bc58f4ef80c76ef88c70397 Mon Sep 17 00:00:00 2001
From: Emelia Smith <ThisIsMissEm@users.noreply.github.com>
Date: Wed, 7 Aug 2024 20:25:08 +0200
Subject: [PATCH 040/656] fix: Send create user email for password resets where
 we have an email and person, but no user. (#7729)

* fix: Send create user email for password resets where we have an email and person, but no user account

This fixes https://github.com/ietf-tools/datatracker/issues/6458

* fix: create User straight away and use nomral password reset

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/ietfauth/tests.py | 18 ++++++++++++++++++
 ietf/ietfauth/views.py | 16 +++++++++++++---
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py
index 6a85c6eb13..722c1e8b6f 100644
--- a/ietf/ietfauth/tests.py
+++ b/ietf/ietfauth/tests.py
@@ -527,6 +527,24 @@ def test_reset_password_without_username(self):
         self.assertIn(secondary_address, to)
         self.assertNotIn(inactive_secondary_address, to)
 
+    def test_reset_password_without_user(self):
+        """Reset password using email address for person without a user account"""
+        url = urlreverse('ietf.ietfauth.views.password_reset')
+        email = EmailFactory()
+        person = email.person
+        # Remove the user object from the person to get a Email/Person without User:
+        person.user = None
+        person.save()
+        # Remove the remaining User record, since reset_password looks for that by username:
+        User.objects.filter(username__iexact=email.address).delete()
+        empty_outbox()
+        r = self.client.post(url, { 'username': email.address })
+        self.assertEqual(len(outbox), 1)
+        lastReceivedEmail = outbox[-1]
+        self.assertIn(email.address, lastReceivedEmail.get('To'))
+        self.assertTrue(lastReceivedEmail.get('Subject').startswith("Confirm password reset"))
+        self.assertContains(r, "Your password reset request has been successfully received", status_code=200)
+
     def test_review_overview(self):
         review_req = ReviewRequestFactory()
         assignment = ReviewAssignmentFactory(review_request=review_req,reviewer=EmailFactory(person__user__username='reviewer'))
diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py
index 61c7b929b1..32bb5c92be 100644
--- a/ietf/ietfauth/views.py
+++ b/ietf/ietfauth/views.py
@@ -491,9 +491,19 @@ def password_reset(request):
             if not user:
                 # try to find user ID from the email address
                 email = Email.objects.filter(address=submitted_username).first()
-                if email and email.person and email.person.user:
-                    user = email.person.user
-
+                if email and email.person:
+                    if email.person.user:
+                        user = email.person.user
+                    else: 
+                        # Create a User record with this (conditioned by way of Email) username
+                        # Don't bother setting the name or email fields on User - rely on the
+                        # Person pointer.
+                        user = User.objects.create(
+                            username=email.address.lower(), 
+                            is_active=True,
+                        )
+                        email.person.user = user
+                        email.person.save()
             if user and user.person.email_set.filter(active=True).exists():
                 data = {
                     'username': user.username,

From e5e6c9bc89b227e2587a831c1a40d386147acb39 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Thu, 8 Aug 2024 06:36:21 +1200
Subject: [PATCH 041/656] feat: Site status message (#7659)

* Status WIP

* feat: Status

* fix: Status tests

* feat: status redirect

* chore: Status tests

* chore: Status tests

* feat: Status tests

* chore: Status playwright tests

* fix: PR feedback, mostly Vue and copyright dates

* fix: Status model migration tidy up

* chore: Status - one migration

* feat: status on doc/html pages

* chore: Resetting Status migration

* chore: removing unused FieldError

* fix: Update Status test to remove 'by'

* chore: fixing API test to exclude 'status'

* chore: fixing status_page test

* feat: Site Status PR feedback. URL coverage debugging

* Adding ietf.status to Tastypie omitted apps

* feat: Site Status PR feedback

* chore: correct copyright year on newly created files

* chore: repair merge damage

* chore: repair more merge damage

* fix: reconcile the api init refactor with ignoring apps

---------

Co-authored-by: Matthew Holloway <Matthew Holloway>
Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 client/Embedded.vue                    |   8 +-
 client/components/Status.vue           |  80 +++++++++++++++++
 client/index.html                      |   2 +
 client/shared/json-wrapper.js          |  20 +++++
 client/shared/local-storage-wrapper.js |  42 +++++++++
 client/shared/status-common.js         |   5 ++
 ietf/api/__init__.py                   |   6 +-
 ietf/api/tests.py                      |   1 +
 ietf/settings.py                       |   1 +
 ietf/static/css/ietf.scss              |   7 ++
 ietf/status/__init__.py                |   0
 ietf/status/admin.py                   |  19 ++++
 ietf/status/apps.py                    |   9 ++
 ietf/status/migrations/0001_initial.py |  75 ++++++++++++++++
 ietf/status/migrations/__init__.py     |   0
 ietf/status/models.py                  |  24 +++++
 ietf/status/tests.py                   | 120 +++++++++++++++++++++++++
 ietf/status/urls.py                    |  12 +++
 ietf/status/views.py                   |  46 ++++++++++
 ietf/templates/base.html               |   5 +-
 ietf/templates/base/status.html        |   2 +
 ietf/templates/doc/document_html.html  |   3 +
 ietf/templates/status/latest.html      |  18 ++++
 ietf/templates/status/status.html      |  15 ++++
 ietf/urls.py                           |   1 +
 playwright/tests/status/status.spec.js |  61 +++++++++++++
 26 files changed, 574 insertions(+), 8 deletions(-)
 create mode 100644 client/components/Status.vue
 create mode 100644 client/shared/json-wrapper.js
 create mode 100644 client/shared/local-storage-wrapper.js
 create mode 100644 client/shared/status-common.js
 create mode 100644 ietf/status/__init__.py
 create mode 100644 ietf/status/admin.py
 create mode 100644 ietf/status/apps.py
 create mode 100644 ietf/status/migrations/0001_initial.py
 create mode 100644 ietf/status/migrations/__init__.py
 create mode 100644 ietf/status/models.py
 create mode 100644 ietf/status/tests.py
 create mode 100644 ietf/status/urls.py
 create mode 100644 ietf/status/views.py
 create mode 100644 ietf/templates/base/status.html
 create mode 100644 ietf/templates/status/latest.html
 create mode 100644 ietf/templates/status/status.html
 create mode 100644 playwright/tests/status/status.spec.js

diff --git a/client/Embedded.vue b/client/Embedded.vue
index a0f0d2831e..80b105dc15 100644
--- a/client/Embedded.vue
+++ b/client/Embedded.vue
@@ -1,12 +1,13 @@
 <template lang="pug">
 n-theme
-  n-message-provider
-    component(:is='currentComponent', :component-id='props.componentId')
+  n-notification-provider
+    n-message-provider
+      component(:is='currentComponent', :component-id='props.componentId')
 </template>
 
 <script setup>
 import { defineAsyncComponent, markRaw, onMounted, ref } from 'vue'
-import { NMessageProvider } from 'naive-ui'
+import { NMessageProvider, NNotificationProvider } from 'naive-ui'
 
 import NTheme from './components/n-theme.vue'
 
@@ -15,6 +16,7 @@ import NTheme from './components/n-theme.vue'
 const availableComponents = {
   ChatLog: defineAsyncComponent(() => import('./components/ChatLog.vue')),
   Polls: defineAsyncComponent(() => import('./components/Polls.vue')),
+  Status: defineAsyncComponent(() => import('./components/Status.vue'))
 }
 
 // PROPS
diff --git a/client/components/Status.vue b/client/components/Status.vue
new file mode 100644
index 0000000000..4fded5bbe4
--- /dev/null
+++ b/client/components/Status.vue
@@ -0,0 +1,80 @@
+<script setup>
+  import { h, onMounted } from 'vue'
+  import { useNotification } from 'naive-ui'
+  import { localStorageWrapper } from '../shared/local-storage-wrapper'
+  import { JSONWrapper } from '../shared/json-wrapper'
+  import { STATUS_STORAGE_KEY, generateStatusTestId } from '../shared/status-common'
+
+  const getDismissedStatuses = () => {
+    const jsonString = localStorageWrapper.getItem(STATUS_STORAGE_KEY)
+    const jsonValue = JSONWrapper.parse(jsonString, [])
+    if(Array.isArray(jsonValue)) {
+      return jsonValue
+    }
+    return []
+  }
+
+  const dismissStatus = (id) => {
+    const dissmissed = [id, ...getDismissedStatuses()]
+    localStorageWrapper.setItem(STATUS_STORAGE_KEY, JSONWrapper.stringify(dissmissed))
+    return true
+  }
+
+  let notificationInstances = {} // keyed by status.id
+  let notification
+
+  const pollStatusAPI = () => {
+    fetch('/status/latest.json')
+        .then(resp => resp.json())
+        .then(status => {
+            if(status === null || status.hasMessage === false) {
+                console.debug("No status message")
+                return
+            }
+            const dismissedStatuses = getDismissedStatuses()
+            if(dismissedStatuses.includes(status.id)) {
+              console.debug(`Not showing site status ${status.id} because it was already dismissed. Dismissed Ids:`, dismissedStatuses)
+              return
+            }
+
+            const isSameStatusPage = Boolean(document.querySelector(`[data-status-id="${status.id}"]`))
+
+            if(isSameStatusPage) {
+              console.debug(`Not showing site status ${status.id} because we're on the target page`)
+              return
+            }
+
+            if(notificationInstances[status.id]) {
+              console.debug(`Not showing site status ${status.id} because it's already been displayed`)
+              return
+            }
+
+            notificationInstances[status.id] = notification.create({
+              title: status.title,
+              content: status.body,
+              meta: `${status.date}`,
+              action: () =>
+                h(
+                  'a',
+                  {
+                    'data-testid': generateStatusTestId(status.id),
+                    href: status.url,
+                    'aria-label': `Read more about ${status.title}`
+                  },
+                  "Read more"
+                ),
+              onClose: () => {
+                return dismissStatus(status.id)
+              }
+            })
+        })
+        .catch(e => {
+            console.error(e)
+        })
+  }
+  
+  onMounted(() => {
+      notification = useNotification()
+      pollStatusAPI(notification)
+  })
+</script>
diff --git a/client/index.html b/client/index.html
index 740c994329..75d6f77727 100644
--- a/client/index.html
+++ b/client/index.html
@@ -12,6 +12,7 @@
     <link href="https://static.ietf.org/fonts/noto-sans-mono/import.css" rel="stylesheet">
   </head>
   <body>
+    <div class="vue-embed" data-component="Status"></div>
     <div class="pt-3 container-fluid">
       <div class="row">
         <div class="col mx-lg-3" id="content">
@@ -20,5 +21,6 @@
       </div>
     </div>
     <script type="module" src="./main.js"></script>
+    <script type="module" src="./embedded.js"></script>
   </body>
 </html>
diff --git a/client/shared/json-wrapper.js b/client/shared/json-wrapper.js
new file mode 100644
index 0000000000..e080b5a479
--- /dev/null
+++ b/client/shared/json-wrapper.js
@@ -0,0 +1,20 @@
+export const JSONWrapper = {
+  parse(jsonString, defaultValue) {
+    if(typeof jsonString !== "string") {
+      return defaultValue
+    }
+    try {
+      return JSON.parse(jsonString);
+    } catch (e) {
+      console.error(e);
+    }
+    return defaultValue
+  },
+  stringify(data) {
+    try {
+      return JSON.stringify(data);
+    } catch (e) {
+      console.error(e)
+    }
+  },
+}
diff --git a/client/shared/local-storage-wrapper.js b/client/shared/local-storage-wrapper.js
new file mode 100644
index 0000000000..88cd3dc589
--- /dev/null
+++ b/client/shared/local-storage-wrapper.js
@@ -0,0 +1,42 @@
+
+/*
+ *  DEVELOPER NOTE
+ *
+ *  Some browsers can block storage (localStorage, sessionStorage)
+ *  access for privacy reasons, and all browsers can have storage
+ *  that's full, and then they throw exceptions.
+ *
+ *  See https://michalzalecki.com/why-using-localStorage-directly-is-a-bad-idea/
+ *
+ *  Exceptions can even be thrown when testing if localStorage
+ *  even exists. This can throw:
+ * 
+ *      if (window.localStorage)
+ *
+ *  Also localStorage/sessionStorage can be enabled after DOMContentLoaded
+ *  so we handle it gracefully.
+ *
+ *  1) we need to wrap all usage in try/catch
+ *  2) we need to defer actual usage of these until
+ *     necessary,
+ *    
+ */
+
+export const localStorageWrapper = {
+  getItem: (key) => {
+    try {
+      return localStorage.getItem(key)
+    } catch (e) {
+      console.error(e); 
+    }
+    return null;
+  },
+  setItem: (key, value) => {
+    try {
+      return localStorage.setItem(key, value)
+    } catch (e) {
+      console.error(e); 
+    }
+    return;
+  },
+}
diff --git a/client/shared/status-common.js b/client/shared/status-common.js
new file mode 100644
index 0000000000..6503bfbf63
--- /dev/null
+++ b/client/shared/status-common.js
@@ -0,0 +1,5 @@
+// Used in Playwright Status and components
+
+export const STATUS_STORAGE_KEY = "status-dismissed"
+
+export const generateStatusTestId = (id) => `status-${id}`
diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py
index 81f370121f..9fadab8e6f 100644
--- a/ietf/api/__init__.py
+++ b/ietf/api/__init__.py
@@ -21,14 +21,14 @@
 from tastypie.exceptions import ApiFieldError
 from tastypie.fields import ApiField
 
-
 _api_list = []
 
+OMITTED_APPS_APIS = ["ietf.status"]
 
 def populate_api_list():
+    _module_dict = globals()
     for app_config in django_apps.get_app_configs():
-        _module_dict = globals()
-        if '.' in app_config.name:
+        if '.' in app_config.name and app_config.name not in OMITTED_APPS_APIS:
             _root, _name = app_config.name.split('.', 1)
             if _root == 'ietf':
                 if not '.' in _name:
diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index fd8eb52cd6..20c3e2cb44 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -48,6 +48,7 @@
     'ietf.secr.meetings',
     'ietf.secr.proceedings',
     'ietf.ipr',
+    'ietf.status',
 )
 
 class CustomApiTests(TestCase):
diff --git a/ietf/settings.py b/ietf/settings.py
index 7572b15213..db53efe0a5 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -479,6 +479,7 @@ def skip_unreadable_post(record):
     'ietf.release',
     'ietf.review',
     'ietf.stats',
+    'ietf.status',
     'ietf.submit',
     'ietf.sync',
     'ietf.utils',
diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss
index 5bd520f041..e2d5cb3959 100644
--- a/ietf/static/css/ietf.scss
+++ b/ietf/static/css/ietf.scss
@@ -1189,6 +1189,13 @@ blockquote {
     border-left: solid 1px var(--bs-body-color);
 }
 
+iframe.status {
+    background-color:transparent;
+    border:none;
+    width:100%;
+    height:3.5em;
+}
+
 .overflow-shadows {
     transition: box-shadow 0.5s;
 }
diff --git a/ietf/status/__init__.py b/ietf/status/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/ietf/status/admin.py b/ietf/status/admin.py
new file mode 100644
index 0000000000..f9c4e891a7
--- /dev/null
+++ b/ietf/status/admin.py
@@ -0,0 +1,19 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from datetime import datetime
+from django.contrib import admin
+from django.template.defaultfilters import slugify
+from .models import Status
+
+class StatusAdmin(admin.ModelAdmin):
+    list_display = ['title', 'body', 'active', 'date', 'by', 'page']
+    raw_id_fields = ['by']
+
+    def get_changeform_initial_data(self, request):
+        date = datetime.now()
+        return {
+            "slug": slugify(f"{date.year}-{date.month}-{date.day}-"),
+        }
+    
+admin.site.register(Status, StatusAdmin)
diff --git a/ietf/status/apps.py b/ietf/status/apps.py
new file mode 100644
index 0000000000..ba64a41afc
--- /dev/null
+++ b/ietf/status/apps.py
@@ -0,0 +1,9 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from django.apps import AppConfig
+
+
+class StatusConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "ietf.status"
diff --git a/ietf/status/migrations/0001_initial.py b/ietf/status/migrations/0001_initial.py
new file mode 100644
index 0000000000..5185189496
--- /dev/null
+++ b/ietf/status/migrations/0001_initial.py
@@ -0,0 +1,75 @@
+# Generated by Django 4.2.13 on 2024-07-21 22:47
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ("person", "0002_alter_historicalperson_ascii_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Status",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("date", models.DateTimeField(default=django.utils.timezone.now)),
+                ("slug", models.SlugField(unique=True)),
+                (
+                    "title",
+                    models.CharField(
+                        help_text="Your site status notification title.",
+                        max_length=255,
+                        verbose_name="Status title",
+                    ),
+                ),
+                (
+                    "body",
+                    models.CharField(
+                        help_text="Your site status notification body.",
+                        max_length=255,
+                        verbose_name="Status body",
+                    ),
+                ),
+                (
+                    "active",
+                    models.BooleanField(
+                        default=True,
+                        help_text="Only active messages will be shown.",
+                        verbose_name="Active?",
+                    ),
+                ),
+                (
+                    "page",
+                    models.TextField(
+                        blank=True,
+                        help_text="More detail shown after people click 'Read more'. If empty no 'read more' will be shown",
+                        null=True,
+                        verbose_name="More detail (markdown)",
+                    ),
+                ),
+                (
+                    "by",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE, to="person.person"
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name_plural": "statuses",
+            },
+        ),
+    ]
diff --git a/ietf/status/migrations/__init__.py b/ietf/status/migrations/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/ietf/status/models.py b/ietf/status/models.py
new file mode 100644
index 0000000000..b3f97d989e
--- /dev/null
+++ b/ietf/status/models.py
@@ -0,0 +1,24 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from django.utils import timezone
+from django.db import models
+from django.db.models import ForeignKey
+
+import debug                            # pyflakes:ignore
+
+class Status(models.Model):
+    name = 'Status'
+    
+    date = models.DateTimeField(default=timezone.now)
+    slug = models.SlugField(blank=False, null=False, unique=True)
+    title = models.CharField(max_length=255, verbose_name="Status title", help_text="Your site status notification title.")
+    body = models.CharField(max_length=255, verbose_name="Status body", help_text="Your site status notification body.", unique=False)    
+    active = models.BooleanField(default=True, verbose_name="Active?", help_text="Only active messages will be shown.")
+    by = ForeignKey('person.Person', on_delete=models.CASCADE)
+    page = models.TextField(blank=True, null=True, verbose_name="More detail (markdown)", help_text="More detail shown after people click 'Read more'. If empty no 'read more' will be shown")
+
+    def __str__(self):
+        return "{} {} {} {}".format(self.date, self.active, self.by, self.title)
+    class Meta:
+        verbose_name_plural = "statuses"
diff --git a/ietf/status/tests.py b/ietf/status/tests.py
new file mode 100644
index 0000000000..9c0dd9114e
--- /dev/null
+++ b/ietf/status/tests.py
@@ -0,0 +1,120 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+import debug    # pyflakes:ignore
+
+from django.urls import reverse as urlreverse
+from ietf.utils.test_utils import TestCase
+from ietf.person.models import Person
+from ietf.status.models import Status
+
+class StatusTests(TestCase):
+    def test_status_latest_html(self):
+        status = Status.objects.create(
+            title = "my title 1",
+            body = "my body 1",
+            active = True,
+            by = Person.objects.get(user__username='ad'),
+            slug = "2024-1-1-my-title-1"
+        )
+        status.save()
+
+        url = urlreverse('ietf.status.views.status_latest_html')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, 'my title 1')
+        self.assertContains(r, 'my body 1')
+
+        status.delete()
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertNotContains(r, 'my title 1')
+        self.assertNotContains(r, 'my body 1')
+
+    def test_status_latest_json(self):
+        url = urlreverse('ietf.status.views.status_latest_json')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        data = r.json()
+        self.assertFalse(data["hasMessage"])
+
+        status = Status.objects.create(
+            title = "my title 1",
+            body = "my body 1",
+            active = True,
+            by = Person.objects.get(user__username='ad'),
+            slug = "2024-1-1-my-title-1"
+        )
+        status.save()
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        data = r.json()
+        self.assertTrue(data["hasMessage"])
+        self.assertEqual(data["title"], "my title 1")
+        self.assertEqual(data["body"], "my body 1")
+        self.assertEqual(data["slug"], '2024-1-1-my-title-1')
+        self.assertEqual(data["url"], '/status/2024-1-1-my-title-1')
+
+        status.delete()
+
+    def test_status_latest_redirect(self):
+        url = urlreverse('ietf.status.views.status_latest_redirect')
+        r = self.client.get(url)
+        # without a Status it should return Not Found
+        self.assertEqual(r.status_code, 404)
+
+        status = Status.objects.create(
+            title = "my title 1",
+            body = "my body 1",
+            active = True,
+            by = Person.objects.get(user__username='ad'),
+            slug = "2024-1-1-my-title-1"
+        )
+        status.save()
+
+        r = self.client.get(url)
+        # with a Status it should redirect
+        self.assertEqual(r.status_code, 302)
+        self.assertEqual(r.headers["Location"], "/status/2024-1-1-my-title-1")
+        
+        status.delete()
+
+    def test_status_page(self):
+        slug = "2024-1-1-my-unique-slug"
+        r = self.client.get(f'/status/{slug}/')
+        # without a Status it should return Not Found
+        self.assertEqual(r.status_code, 404)
+
+        # status without `page` markdown should still 200
+        status = Status.objects.create(
+            title = "my title 1",
+            body = "my body 1",
+            active = True,
+            by = Person.objects.get(user__username='ad'),
+            slug = slug
+        )
+        status.save()
+
+        r = self.client.get(f'/status/{slug}/')
+        self.assertEqual(r.status_code, 200)
+        
+        status.delete()
+
+        test_string = 'a string that'
+        status = Status.objects.create(
+            title = "my title 1",
+            body = "my body 1",
+            active = True,
+            by = Person.objects.get(user__username='ad'),
+            slug = slug,
+            page = f"# {test_string}"
+        )
+        status.save()
+
+        r = self.client.get(f'/status/{slug}/')
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, test_string)
+        
+        status.delete()
diff --git a/ietf/status/urls.py b/ietf/status/urls.py
new file mode 100644
index 0000000000..060c0257e5
--- /dev/null
+++ b/ietf/status/urls.py
@@ -0,0 +1,12 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from ietf.status import views
+from ietf.utils.urls import url
+
+urlpatterns = [
+    url(r"^$", views.status_latest_redirect),
+    url(r"^latest$", views.status_latest_html),
+    url(r"^latest.json$", views.status_latest_json),
+    url(r"(?P<slug>.*)", views.status_page)
+]
diff --git a/ietf/status/views.py b/ietf/status/views.py
new file mode 100644
index 0000000000..9037d01dc2
--- /dev/null
+++ b/ietf/status/views.py
@@ -0,0 +1,46 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from django.urls import reverse as urlreverse
+from django.http import HttpResponseRedirect, HttpResponseNotFound, JsonResponse
+from ietf.utils import markdown
+from django.shortcuts import render, get_object_or_404
+from ietf.status.models import Status
+
+import debug                            # pyflakes:ignore
+
+def get_last_active_status():
+    status = Status.objects.filter(active=True).order_by("-date").first()
+    if status is None:
+        return { "hasMessage": False }
+
+    context = {
+        "hasMessage": True,
+        "id": status.id,
+        "slug": status.slug,
+        "title": status.title,
+        "body": status.body,
+        "url": urlreverse("ietf.status.views.status_page", kwargs={ "slug": status.slug }),
+        "date": status.date.isoformat()
+    }
+    return context
+
+def status_latest_html(request):
+    return render(request, "status/latest.html", context=get_last_active_status())
+
+def status_page(request, slug):
+    sanitised_slug = slug.rstrip("/")
+    status = get_object_or_404(Status, slug=sanitised_slug)
+    return render(request, "status/status.html", context={
+        'status': status,
+        'status_page_html': markdown.markdown(status.page or ""),
+    })
+
+def status_latest_json(request):
+    return JsonResponse(get_last_active_status())
+
+def status_latest_redirect(request):
+    context = get_last_active_status()
+    if context["hasMessage"] == True:
+        return HttpResponseRedirect(context["url"])
+    return HttpResponseNotFound()
diff --git a/ietf/templates/base.html b/ietf/templates/base.html
index f426d361ce..ceb1d2df08 100644
--- a/ietf/templates/base.html
+++ b/ietf/templates/base.html
@@ -34,6 +34,7 @@
     <body {% block bodyAttrs %}{% endblock %} class="navbar-offset position-relative"
           data-group-menu-data-url="{% url 'ietf.group.views.group_menu_data' %}">
         {% analytical_body_top %}
+        {% include "base/status.html" %}
         <a class="visually-hidden visually-hidden-focusable" href="#content">Skip to main content</a>
         <nav class="navbar navbar-expand-lg fixed-top {% if server_mode and server_mode != "production" %}bg-danger-subtle{% else %}bg-secondary-subtle{% endif %}">
             <div class="container-fluid">
@@ -85,7 +86,7 @@
             </div>
         </nav>
         {% block precontent %}{% endblock %}
-        <div class="pt-3 container-fluid">
+        <main class="pt-3 container-fluid" id="main">
             <div class="row">
                 {% if request.COOKIES.left_menu == "on" and not hide_menu %}
                     <div class="d-none d-md-block bg-light-subtle py-3 leftmenu small">
@@ -114,7 +115,7 @@
                     {% block content_end %}{% endblock %}
                 </div>
             </div>
-        </div>
+        </main>
         {% block footer %}
             <footer class="col-md-12 col-sm-12 border-top mt-5 py-5 bg-light-subtle text-center position-sticky">
                 <a href="https://www.ietf.org/" class="p-3">IETF</a>
diff --git a/ietf/templates/base/status.html b/ietf/templates/base/status.html
new file mode 100644
index 0000000000..33e1abf699
--- /dev/null
+++ b/ietf/templates/base/status.html
@@ -0,0 +1,2 @@
+<noscript><iframe class="status" title="Site status" src="/status/latest"></iframe></noscript>
+<div class="vue-embed" data-component="Status"></div>
\ No newline at end of file
diff --git a/ietf/templates/doc/document_html.html b/ietf/templates/doc/document_html.html
index 40223d0fd1..3eabe2ebb7 100644
--- a/ietf/templates/doc/document_html.html
+++ b/ietf/templates/doc/document_html.html
@@ -4,6 +4,7 @@
 {% load origin %}
 {% load static %}
 {% load ietf_filters textfilters %}
+{% load django_vite %}
 {% origin %}
 <html data-bs-theme="auto" lang="en">
     <head>
@@ -28,6 +29,7 @@
             {% if html %}
                 <link rel="stylesheet" href="{% static 'ietf/css/document_html_txt.css' %}">
             {% endif %}
+            {% vite_asset 'client/embedded.js' %}
             <script src="{% static 'ietf/js/document_html.js' %}"></script>
             <script src="{% static 'ietf/js/theme.js' %}"></script>
         {% endif %}
@@ -51,6 +53,7 @@
     </head>
     <body>
         {% analytical_body_top %}
+        {% include "base/status.html" %}
         <div class="btn-toolbar sidebar-toolbar position-fixed top-0 end-0 m-2 m-lg-3 d-print-none">
             <div class="dropdown">
                 <button class="btn btn-outline-secondary btn-sm me-1 dropdown-toggle d-flex align-items-center"
diff --git a/ietf/templates/status/latest.html b/ietf/templates/status/latest.html
new file mode 100644
index 0000000000..3ad54c9dc3
--- /dev/null
+++ b/ietf/templates/status/latest.html
@@ -0,0 +1,18 @@
+{% load origin %}
+{% load ietf_filters static %}
+{% origin %}
+<meta name="color-scheme" content="light dark">
+<style type="text/css">
+    {# this template doesn't inherit from base.html so it has its own styles #}
+    body {background:transparent;font-family:sans-serif}
+    h1{font-size:18px;display:inline}
+    p{font-size:14px;display:inline}
+    .unimportant{opacity:0.6}
+</style>
+<!-- This page is intended to be iframed, and is only for non-JavaScript browsers. -->
+{% if title %}
+    <h1>{{ title }}</h1>
+    <p>{{ body }} <a href="{{ url }}" target="_top" aria-label="read more about {{title}}">read more</a><br /><span class="unimportant">{{ date }}</span></p>
+{% else %}
+    <p class="unimportant">No site status message.</p>
+{% endif %}
diff --git a/ietf/templates/status/status.html b/ietf/templates/status/status.html
new file mode 100644
index 0000000000..18f64c49a8
--- /dev/null
+++ b/ietf/templates/status/status.html
@@ -0,0 +1,15 @@
+{% extends "base.html" %}
+{% load origin %}
+{% load ietf_filters static %}
+{% block content %}
+    {% origin %}
+    <h1 data-status-id="{{ status.id }}">
+        {% block title %} {{ status.title }} {% endblock %}
+        {% if status.active == False %}
+            <span class="badge bg-secondary">inactive</span>
+        {% endif %}
+    </h1>
+    <div>
+        {{ status_page_html }}
+    </div>
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/urls.py b/ietf/urls.py
index 90b161b530..e822b2042e 100644
--- a/ietf/urls.py
+++ b/ietf/urls.py
@@ -61,6 +61,7 @@
     url(r'^sitemap-(?P<section>.+).xml$', sitemap_views.sitemap, {'sitemaps': sitemaps}),
     url(r'^sitemap.xml$', sitemap_views.index, { 'sitemaps': sitemaps}),
     url(r'^stats/', include('ietf.stats.urls')),
+    url(r'^status/', include('ietf.status.urls')),
     url(r'^stream/', include(stream_urls)),
     url(r'^submit/', include('ietf.submit.urls')),
     url(r'^sync/', include('ietf.sync.urls')),
diff --git a/playwright/tests/status/status.spec.js b/playwright/tests/status/status.spec.js
new file mode 100644
index 0000000000..7b3b90bfa4
--- /dev/null
+++ b/playwright/tests/status/status.spec.js
@@ -0,0 +1,61 @@
+const {
+  test,
+  expect
+} = require('@playwright/test')
+const { STATUS_STORAGE_KEY, generateStatusTestId } = require('../../../client/shared/status-common.js')
+
+test.describe('site status', () => {
+  const noStatus = {
+    hasMessage: false
+  }
+
+  const status1 = {
+    hasMessage: true,
+    id: 1,
+    slug: '2024-7-9fdfdf-sdfsdf',
+    title: 'My status title',
+    body: 'My status body',
+    url: '/status/2024-7-9fdfdf-sdfsdf',
+    date: '2024-07-09T07:05:13+00:00',
+    by: 'Exile is a cool Amiga game'
+  }
+
+  test('Renders server status as Notification', async ({ page }) => {
+    await page.route('/status/latest.json', route => {
+      route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify(status1)
+      })
+    })
+    await page.goto('/')
+    await expect(page.getByTestId(generateStatusTestId(status1.id)), 'should have status').toHaveCount(1)
+  })
+
+  test("Doesn't render dismissed server statuses", async ({ page }) => {
+    await page.route('/status/latest.json', route => {
+      route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify(status1)
+      })
+    })
+    await page.goto('/')
+    await page.evaluate(({ key, value }) => localStorage.setItem(key, value), { key: STATUS_STORAGE_KEY, value: JSON.stringify([status1.id]) })
+    await expect(page.getByTestId(generateStatusTestId(status1.id)), 'should have status').toHaveCount(0)
+  })
+
+  test('Handles no server status', async ({ page }) => {
+    await page.route('/status/latest.json', route => {
+      route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify(noStatus)
+      })
+    })
+
+    await page.goto('/')
+
+    await expect(page.getByTestId(generateStatusTestId(status1.id)), 'should have status').toHaveCount(0)
+  })
+})

From 1252cd2ac28ac4c6791731df0878f31e497885f1 Mon Sep 17 00:00:00 2001
From: Warren Kumari <wkumari@google.com>
Date: Thu, 8 Aug 2024 13:10:54 -0400
Subject: [PATCH 042/656] fix: badgeify "Serious Issues", "On the right track"
 review summaries. (#7809)

Addresses #7808
---
 ietf/doc/templatetags/ietf_filters.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py
index 35f9f91b43..5a8afd9956 100644
--- a/ietf/doc/templatetags/ietf_filters.py
+++ b/ietf/doc/templatetags/ietf_filters.py
@@ -856,10 +856,10 @@ def badgeify(blob):
     Add an appropriate bootstrap badge around "text", based on its contents.
     """
     config = [
-        (r"rejected|not ready", "danger", "x-lg"),
+        (r"rejected|not ready|serious issues", "danger", "x-lg"),
         (r"complete|accepted|ready", "success", ""),
         (r"has nits|almost ready", "info", "info-lg"),
-        (r"has issues", "warning", "exclamation-lg"),
+        (r"has issues|on the right track", "warning", "exclamation-lg"),
         (r"assigned", "info", "person-plus-fill"),
         (r"will not review|overtaken by events|withdrawn", "secondary", "dash-lg"),
         (r"no response", "warning", "question-lg"),

From ff8898186ba06072d0e226a6c88bb6e371cc3dee Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Sat, 10 Aug 2024 02:24:04 +1200
Subject: [PATCH 043/656] fix: Ballot return to url via url params rather than
 session (#7788)

* fix: #7287 ballot return params

* fix: Moving Ballot edit position ballot_edit_return_point from session to query param

* fix: tests for return_to_path param

* chore: removing Playwright tests until we can figure out a plan

* feat: return_to path utility refactoring

* fix: throw HTTP 400 in view rather than bubbling up a 500

* fix: return http400 rather than raising
---
 ietf/doc/tests_ballot.py                      | 30 +++++++++++++
 ietf/doc/views_ballot.py                      | 45 +++++++++++++------
 ietf/doc/views_doc.py                         |  5 ---
 ietf/iesg/views.py                            |  3 +-
 ietf/templates/doc/ballot_popup.html          |  2 +-
 .../doc/document_ballot_content.html          |  2 +-
 ietf/utils/http.py                            | 26 ++++++++++-
 playwright/package.json                       |  1 +
 8 files changed, 91 insertions(+), 23 deletions(-)

diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py
index 034ba6f4b2..2403b6ef5c 100644
--- a/ietf/doc/tests_ballot.py
+++ b/ietf/doc/tests_ballot.py
@@ -20,6 +20,7 @@
                                 BallotPositionDocEventFactory, BallotDocEventFactory, IRSGBallotDocEventFactory)
 from ietf.doc.templatetags.ietf_filters import can_defer
 from ietf.doc.utils import create_ballot_if_not_open
+from ietf.doc.views_ballot import parse_ballot_edit_return_point
 from ietf.doc.views_doc import document_ballot_content
 from ietf.group.models import Group, Role
 from ietf.group.factories import GroupFactory, RoleFactory, ReviewTeamFactory
@@ -1451,3 +1452,32 @@ def test_document_ballot_content_without_send_email_values(self):
         self._assertBallotMessage(q, balloters[0], 'No discuss send log available')
         self._assertBallotMessage(q, balloters[1], 'No comment send log available')
         self._assertBallotMessage(q, old_balloter, 'No ballot position send log available')
+
+class ReturnToUrlTests(TestCase):
+    def test_invalid_return_to_url(self):
+        self.assertRaises(
+            Exception,
+            lambda: parse_ballot_edit_return_point('/doc/', 'draft-ietf-opsawg-ipfix-tcpo-v6eh', '998718'),
+        )
+        self.assertRaises(
+            Exception,
+            lambda: parse_ballot_edit_return_point('/a-route-that-does-not-exist/', 'draft-ietf-opsawg-ipfix-tcpo-v6eh', '998718'),
+        )
+        self.assertRaises(
+            Exception,
+            lambda: parse_ballot_edit_return_point('https://example.com/phishing', 'draft-ietf-opsawg-ipfix-tcpo-v6eh', '998718'),
+        )
+
+    def test_valid_default_return_to_url(self):
+        self.assertEqual(parse_ballot_edit_return_point(
+            None,
+            'draft-ietf-opsawg-ipfix-tcpo-v6eh',
+            '998718'
+        ), '/doc/draft-ietf-opsawg-ipfix-tcpo-v6eh/ballot/998718/')
+        
+    def test_valid_return_to_url(self):
+        self.assertEqual(parse_ballot_edit_return_point(
+            '/doc/draft-ietf-opsawg-ipfix-tcpo-v6eh/ballot/998718/',
+            'draft-ietf-opsawg-ipfix-tcpo-v6eh',
+            '998718'
+        ), '/doc/draft-ietf-opsawg-ipfix-tcpo-v6eh/ballot/998718/')
diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py
index 83ccb07be6..357bd2fa2f 100644
--- a/ietf/doc/views_ballot.py
+++ b/ietf/doc/views_ballot.py
@@ -8,13 +8,14 @@
 
 from django import forms
 from django.conf import settings
-from django.http import HttpResponse, HttpResponseRedirect, Http404
+from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseBadRequest
 from django.shortcuts import render, get_object_or_404, redirect
 from django.template.defaultfilters import striptags
 from django.template.loader import render_to_string
 from django.urls import reverse as urlreverse
 from django.views.decorators.csrf import csrf_exempt
 from django.utils.html import escape
+from urllib.parse import urlencode as urllib_urlencode
 
 import debug                            # pyflakes:ignore
 
@@ -39,6 +40,7 @@
 from ietf.name.models import BallotPositionName, DocTypeName
 from ietf.person.models import Person
 from ietf.utils.fields import ModelMultipleChoiceField
+from ietf.utils.http import validate_return_to_path
 from ietf.utils.mail import send_mail_text, send_mail_preformatted
 from ietf.utils.decorators import require_api_key
 from ietf.utils.response import permission_denied
@@ -185,11 +187,11 @@ def edit_position(request, name, ballot_id):
 
     balloter = login = request.user.person
 
-    if 'ballot_edit_return_point' in request.session:
-        return_to_url = request.session['ballot_edit_return_point']
-    else:
-        return_to_url = urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name, ballot_id=ballot_id))
-
+    try:
+        return_to_url = parse_ballot_edit_return_point(request.GET.get('ballot_edit_return_point'), doc.name, ballot_id)
+    except ValueError:
+        return HttpResponseBadRequest('ballot_edit_return_point is invalid')
+    
     # if we're in the Secretariat, we can select a balloter to act as stand-in for
     if has_role(request.user, "Secretariat"):
         balloter_id = request.GET.get('balloter')
@@ -209,9 +211,14 @@ def edit_position(request, name, ballot_id):
             save_position(form, doc, ballot, balloter, login, send_mail)
 
             if send_mail:
-                qstr=""
+                query = {}
                 if request.GET.get('balloter'):
-                    qstr += "?balloter=%s" % request.GET.get('balloter')
+                    query['balloter'] = request.GET.get('balloter')
+                if request.GET.get('ballot_edit_return_point'):
+                    query['ballot_edit_return_point'] = request.GET.get('ballot_edit_return_point')
+                qstr = ""
+                if len(query) > 0:
+                    qstr = "?" + urllib_urlencode(query, safe='/')
                 return HttpResponseRedirect(urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=doc.name, ballot_id=ballot_id)) + qstr)
             elif request.POST.get("Defer") and doc.stream.slug != "irtf":
                 return redirect('ietf.doc.views_ballot.defer_ballot', name=doc)
@@ -337,11 +344,11 @@ def send_ballot_comment(request, name, ballot_id):
 
     balloter = request.user.person
 
-    if 'ballot_edit_return_point' in request.session:
-        return_to_url = request.session['ballot_edit_return_point']
-    else:
-        return_to_url = urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name, ballot_id=ballot_id))
-
+    try:
+        return_to_url = parse_ballot_edit_return_point(request.GET.get('ballot_edit_return_point'), doc.name, ballot_id)
+    except ValueError:
+        return HttpResponseBadRequest('ballot_edit_return_point is invalid')
+    
     if 'HTTP_REFERER' in request.META:
         back_url = request.META['HTTP_REFERER']
     else:
@@ -1302,3 +1309,15 @@ def rsab_ballot_status(request):
     # Possible TODO: add a menu item to show this? Maybe only if you're in rsab or an rswg chair?
     # There will be so few of these that the general community would follow them from the rswg docs page.
     # Maybe the view isn't actually needed at all...
+
+
+def parse_ballot_edit_return_point(path, doc_name, ballot_id):
+    get_default_path = lambda: urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc_name, ballot_id=ballot_id))
+    allowed_path_handlers = {
+        "ietf.doc.views_doc.document_ballot",
+        "ietf.doc.views_doc.document_irsg_ballot",
+        "ietf.doc.views_doc.document_rsab_ballot",
+        "ietf.iesg.views.agenda",
+        "ietf.iesg.views.agenda_documents",
+    }
+    return validate_return_to_path(path, get_default_path, allowed_path_handlers)
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index bd49275082..0ef132c215 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -1538,7 +1538,6 @@ def document_ballot(request, name, ballot_id=None):
     top = render_document_top(request, doc, ballot_tab, name)
 
     c = document_ballot_content(request, doc, ballot.id, editable=True)
-    request.session['ballot_edit_return_point'] = request.path_info
 
     return render(request, "doc/document_ballot.html",
                               dict(doc=doc,
@@ -1556,8 +1555,6 @@ def document_irsg_ballot(request, name, ballot_id=None):
 
     c = document_ballot_content(request, doc, ballot_id, editable=True)
 
-    request.session['ballot_edit_return_point'] = request.path_info
-
     return render(request, "doc/document_ballot.html",
                               dict(doc=doc,
                                    top=top,
@@ -1575,8 +1572,6 @@ def document_rsab_ballot(request, name, ballot_id=None):
 
     c = document_ballot_content(request, doc, ballot_id, editable=True)
 
-    request.session['ballot_edit_return_point'] = request.path_info
-
     return render(
         request,
         "doc/document_ballot.html",
diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py
index b67ef04a03..a92d617ac5 100644
--- a/ietf/iesg/views.py
+++ b/ietf/iesg/views.py
@@ -209,7 +209,6 @@ def agenda(request, date=None):
                 urlreverse("ietf.iesg.views.telechat_agenda_content_view", kwargs={"section": "minutes"})
             ))
 
-    request.session['ballot_edit_return_point'] = request.path_info
     return render(request, "iesg/agenda.html", {
             "date": data["date"],
             "sections": sorted(data["sections"].items(), key=lambda x:[int(p) for p in x[0].split('.')]),
@@ -398,7 +397,7 @@ def agenda_documents(request):
                 "sections": sorted((num, section) for num, section in sections.items()
                                    if "2" <= num < "5")
                 })
-    request.session['ballot_edit_return_point'] = request.path_info
+    
     return render(request, 'iesg/agenda_documents.html', { 'telechats': telechats })
 
 def past_documents(request):
diff --git a/ietf/templates/doc/ballot_popup.html b/ietf/templates/doc/ballot_popup.html
index 2a04ffab69..2c0863ab6c 100644
--- a/ietf/templates/doc/ballot_popup.html
+++ b/ietf/templates/doc/ballot_popup.html
@@ -27,7 +27,7 @@
             {% if editable and user|has_role:"Area Director,Secretariat,IRSG Member,RSAB Member" %}
                 {% if user|can_ballot:doc %}
                     <a class="btn btn-primary"
-                       href="{% url "ietf.doc.views_ballot.edit_position" name=doc.name ballot_id=ballot_id %}">
+                       href="{% url "ietf.doc.views_ballot.edit_position" name=doc.name ballot_id=ballot_id %}?ballot_edit_return_point={{ request.path|urlencode }}">
                         Edit position
                     </a>
                 {% endif %}
diff --git a/ietf/templates/doc/document_ballot_content.html b/ietf/templates/doc/document_ballot_content.html
index 803ed84a36..e0feb78bc7 100644
--- a/ietf/templates/doc/document_ballot_content.html
+++ b/ietf/templates/doc/document_ballot_content.html
@@ -60,7 +60,7 @@
             </a>
             {% if user|can_ballot:doc %}
                 <a class="btn btn-primary"
-                   href="{% url "ietf.doc.views_ballot.edit_position" name=doc.name ballot_id=ballot.pk %}">
+                   href="{% url "ietf.doc.views_ballot.edit_position" name=doc.name ballot_id=ballot.pk %}?ballot_edit_return_point={{ request.path|urlencode }}">
                     Edit position
                 </a>
             {% endif %}
diff --git a/ietf/utils/http.py b/ietf/utils/http.py
index 6e6409e31f..cda51680ab 100644
--- a/ietf/utils/http.py
+++ b/ietf/utils/http.py
@@ -1,6 +1,8 @@
-# Copyright The IETF Trust 2023, All Rights Reserved
+# Copyright The IETF Trust 2023-2024, All Rights Reserved
 # -*- coding: utf-8 -*-
 
+from django.urls import resolve as urlresolve, Resolver404
+
 def is_ajax(request):
     """Checks whether a request was an AJAX call
 
@@ -8,3 +10,25 @@ def is_ajax(request):
     exact reproduction of the deprecated method suggested there.
     """
     return request.headers.get("x-requested-with") == "XMLHttpRequest"
+
+def validate_return_to_path(path, get_default_path, allowed_path_handlers):
+    if path is None:
+        path = get_default_path()
+
+    # we need to ensure the path isn't used for attacks (eg phishing).
+    # `path` can be used in HttpResponseRedirect() which could redirect to Datatracker or offsite.
+    # Eg http://datatracker.ietf.org/...?ballot_edit_return_point=https://example.com/phish
+    # offsite links could be phishing attempts so let's reject them all, and require valid Datatracker
+    # routes
+    try:
+        # urlresolve will throw if the url doesn't match a route known to Django
+        match = urlresolve(path)                
+        # further restrict by whether it's in the list of valid routes to prevent
+        # (eg) redirecting to logout
+        if match.url_name not in allowed_path_handlers:
+            raise ValueError("Invalid return to path not among valid matches")
+        pass
+    except Resolver404:
+        raise ValueError("Invalid return to path doesn't match a route")
+
+    return path
diff --git a/playwright/package.json b/playwright/package.json
index 874faa824e..5e3ba94782 100644
--- a/playwright/package.json
+++ b/playwright/package.json
@@ -3,6 +3,7 @@
     "install-deps": "playwright install --with-deps",
     "test": "playwright test",
     "test:legacy": "playwright test -c playwright-legacy.config.js",
+    "test:legacy:visual": "playwright test -c playwright-legacy.config.js --headed --workers=1",
     "test:visual": "playwright test --headed --workers=1",
     "test:debug": "playwright test --debug"
   },

From 00d61432934a61e1a1c204fadb218874237d382c Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 14 Aug 2024 13:43:21 -0300
Subject: [PATCH 044/656] fix: guard against missing ipr revisions (#7826)

* fix: guard against missing "revisions"

* test: add tests for DraftForm

Only testing the part relevant to this issue.

* test: clarify intentions

* chore: "document" -> "draft" in comments
---
 ietf/ipr/forms.py |  2 +-
 ietf/ipr/tests.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/ietf/ipr/forms.py b/ietf/ipr/forms.py
index 8ea179789b..62d3f9c216 100644
--- a/ietf/ipr/forms.py
+++ b/ietf/ipr/forms.py
@@ -112,7 +112,7 @@ def clean(self):
         if not document:
             self.add_error("document", "Identifying the Internet-Draft or RFC for this disclosure is required.")
         elif not document.name.startswith("rfc"):
-            if revisions.strip() == "":
+            if revisions is None or revisions.strip() == "":
                 self.add_error("revisions", "Revisions of this Internet-Draft for which this disclosure is relevant must be specified.")
         return cleaned_data
 
diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py
index 3a91c9dfb7..5b6a5d5d58 100644
--- a/ietf/ipr/tests.py
+++ b/ietf/ipr/tests.py
@@ -28,9 +28,11 @@
 from ietf.ipr.factories import (
     HolderIprDisclosureFactory,
     GenericIprDisclosureFactory,
+    IprDisclosureBaseFactory,
     IprDocRelFactory,
     IprEventFactory
 )
+from ietf.ipr.forms import DraftForm
 from ietf.ipr.mail import (process_response_email, get_reply_to, get_update_submitter_emails,
     get_pseudo_submitter, get_holders, get_update_cc_addrs)
 from ietf.ipr.models import (IprDisclosureBase,GenericIprDisclosure,HolderIprDisclosure,
@@ -935,3 +937,57 @@ def test_no_revisions_message(self):
             no_revisions_message(iprdocrel),
             "No revisions for this Internet-Draft were specified in this disclosure. However, there is only one revision of this Internet-Draft."
         )
+
+
+class DraftFormTests(TestCase):
+    def setUp(self):
+        super().setUp()
+        self.disclosure = IprDisclosureBaseFactory()
+        self.draft = WgDraftFactory()
+        self.rfc = RfcFactory()
+
+    def test_revisions_valid(self):
+        post_data = {
+            "document": str(self.draft.pk),
+            "disclosure": str(self.disclosure.pk),
+        }
+        # The revisions field is just a char field that allows descriptions of the applicable
+        # document revisions. It's usually just a rev or "00-02", but the form allows anything
+        # not empty. The secretariat will review the value before the disclosure is posted so
+        # minimal validation is ok here.
+        self.assertTrue(DraftForm(post_data | {"revisions": "00"}).is_valid())
+        self.assertTrue(DraftForm(post_data | {"revisions": "00-02"}).is_valid())
+        self.assertTrue(DraftForm(post_data | {"revisions": "01,03, 05"}).is_valid())
+        self.assertTrue(DraftForm(post_data | {"revisions": "all but 01"}).is_valid())
+        # RFC instead of draft - allow empty / missing revisions
+        post_data["document"] = str(self.rfc.pk)
+        self.assertTrue(DraftForm(post_data).is_valid())
+        self.assertTrue(DraftForm(post_data | {"revisions": ""}).is_valid())
+
+    def test_revisions_invalid(self):
+        missing_rev_error_msg = (
+            "Revisions of this Internet-Draft for which this disclosure is relevant must be specified."
+        )
+        null_char_error_msg = "Null characters are not allowed."
+        
+        post_data = {
+            "document": str(self.draft.pk),
+            "disclosure": str(self.disclosure.pk),
+        }
+        self.assertFormError(
+            DraftForm(post_data), "revisions", missing_rev_error_msg
+        )
+        self.assertFormError(
+            DraftForm(post_data | {"revisions": ""}), "revisions", missing_rev_error_msg
+        )
+        self.assertFormError(
+            DraftForm(post_data | {"revisions": "1\x00"}),
+            "revisions",
+            [null_char_error_msg, missing_rev_error_msg],
+        )
+        # RFC instead of draft still validates the revisions field
+        self.assertFormError(
+            DraftForm(post_data | {"document": str(self.rfc.pk), "revisions": "1\x00"}),
+            "revisions",
+            null_char_error_msg,
+        )

From bb0b3427167b91d434a2b751718df7ad7fa976f8 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 15 Aug 2024 18:33:14 -0300
Subject: [PATCH 045/656] test: fix DraftFormTest setup (#7828)

* test: fix DraftFormTest setup

Tests were failing when the doc factories started
producing pks >= 10 because of an incorrect test
data structure.

* chore: improve comments
---
 ietf/ipr/tests.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py
index 5b6a5d5d58..922ae272a9 100644
--- a/ietf/ipr/tests.py
+++ b/ietf/ipr/tests.py
@@ -943,12 +943,14 @@ class DraftFormTests(TestCase):
     def setUp(self):
         super().setUp()
         self.disclosure = IprDisclosureBaseFactory()
-        self.draft = WgDraftFactory()
+        self.draft = WgDraftFactory.create_batch(10)[-1]
         self.rfc = RfcFactory()
 
     def test_revisions_valid(self):
         post_data = {
-            "document": str(self.draft.pk),
+            # n.b., "document" is a SearchableDocumentField, which is a multiple choice field limited
+            # to a single choice. Its value must be an array of pks with one element.
+            "document": [str(self.draft.pk)],
             "disclosure": str(self.disclosure.pk),
         }
         # The revisions field is just a char field that allows descriptions of the applicable
@@ -960,7 +962,7 @@ def test_revisions_valid(self):
         self.assertTrue(DraftForm(post_data | {"revisions": "01,03, 05"}).is_valid())
         self.assertTrue(DraftForm(post_data | {"revisions": "all but 01"}).is_valid())
         # RFC instead of draft - allow empty / missing revisions
-        post_data["document"] = str(self.rfc.pk)
+        post_data["document"] = [str(self.rfc.pk)]
         self.assertTrue(DraftForm(post_data).is_valid())
         self.assertTrue(DraftForm(post_data | {"revisions": ""}).is_valid())
 
@@ -971,7 +973,9 @@ def test_revisions_invalid(self):
         null_char_error_msg = "Null characters are not allowed."
         
         post_data = {
-            "document": str(self.draft.pk),
+            # n.b., "document" is a SearchableDocumentField, which is a multiple choice field limited
+            # to a single choice. Its value must be an array of pks with one element.
+            "document": [str(self.draft.pk)],
             "disclosure": str(self.disclosure.pk),
         }
         self.assertFormError(
@@ -987,7 +991,7 @@ def test_revisions_invalid(self):
         )
         # RFC instead of draft still validates the revisions field
         self.assertFormError(
-            DraftForm(post_data | {"document": str(self.rfc.pk), "revisions": "1\x00"}),
+            DraftForm(post_data | {"document": [str(self.rfc.pk)], "revisions": "1\x00"}),
             "revisions",
             null_char_error_msg,
         )

From f53d1cae65e3462af485ac10a55e0ba7cf0a6828 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 21 Aug 2024 08:29:09 -0500
Subject: [PATCH 046/656] fix: remove unused notes field from
 SchedSessTimeAssignment model (#7832)

---
 ...0008_remove_schedtimesessassignment_notes.py | 17 +++++++++++++++++
 ietf/meeting/models.py                          |  3 +--
 ietf/meeting/resources.py                       |  1 -
 3 files changed, 18 insertions(+), 3 deletions(-)
 create mode 100644 ietf/meeting/migrations/0008_remove_schedtimesessassignment_notes.py

diff --git a/ietf/meeting/migrations/0008_remove_schedtimesessassignment_notes.py b/ietf/meeting/migrations/0008_remove_schedtimesessassignment_notes.py
new file mode 100644
index 0000000000..3c0b85fc22
--- /dev/null
+++ b/ietf/meeting/migrations/0008_remove_schedtimesessassignment_notes.py
@@ -0,0 +1,17 @@
+# Generated by Django 4.2.15 on 2024-08-16 13:49
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("meeting", "0007_attended_origin_attended_time"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="schedtimesessassignment",
+            name="notes",
+        ),
+    ]
diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index d8a069ef3c..693cb99dfd 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -788,7 +788,6 @@ class SchedTimeSessAssignment(models.Model):
     schedule = ForeignKey('Schedule', null=False, blank=False, related_name='assignments')
     extendedfrom = ForeignKey('self', null=True, default=None, help_text="Timeslot this session is an extension of.")
     modified = models.DateTimeField(auto_now=True)
-    notes    = models.TextField(blank=True)
     badness  = models.IntegerField(default=0, blank=True, null=True)
     pinned   = models.BooleanField(default=False, help_text="Do not move session during automatic placement.")
 
@@ -1423,7 +1422,7 @@ class MeetingHost(models.Model):
                 validate_file_extension,
                 settings.MEETING_VALID_UPLOAD_EXTENSIONS['meetinghostlogo'],
             ),
-            WrappedValidator(
+   WrappedValidator(
                 validate_mime_type,
                 settings.MEETING_VALID_UPLOAD_MIME_TYPES['meetinghostlogo'],
                 True,
diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py
index dc273c04cf..de9ca01476 100644
--- a/ietf/meeting/resources.py
+++ b/ietf/meeting/resources.py
@@ -269,7 +269,6 @@ class Meta:
         filtering = { 
             "id": ALL,
             "modified": ALL,
-            "notes": ALL,
             "badness": ALL,
             "pinned": ALL,
             "timeslot": ALL_WITH_RELATIONS,

From c323cdde036593244ddcec218c0ffcedda56e792 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 21 Aug 2024 08:32:06 -0500
Subject: [PATCH 047/656] fix: remove unused internal_comments from the
 DocumentInfo abstract model (#7833)

---
 ...e_dochistory_internal_comments_and_more.py | 21 +++++++++++++++++++
 ietf/doc/models.py                            |  1 -
 ietf/doc/resources.py                         |  2 --
 ietf/secr/static/css/custom.css               |  7 +------
 4 files changed, 22 insertions(+), 9 deletions(-)
 create mode 100644 ietf/doc/migrations/0022_remove_dochistory_internal_comments_and_more.py

diff --git a/ietf/doc/migrations/0022_remove_dochistory_internal_comments_and_more.py b/ietf/doc/migrations/0022_remove_dochistory_internal_comments_and_more.py
new file mode 100644
index 0000000000..ad27793a83
--- /dev/null
+++ b/ietf/doc/migrations/0022_remove_dochistory_internal_comments_and_more.py
@@ -0,0 +1,21 @@
+# Generated by Django 4.2.15 on 2024-08-16 16:43
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("doc", "0021_narrativeminutes"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="dochistory",
+            name="internal_comments",
+        ),
+        migrations.RemoveField(
+            model_name="document",
+            name="internal_comments",
+        ),
+    ]
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index a103fca645..639e6ca857 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -122,7 +122,6 @@ class DocumentInfo(models.Model):
     external_url = models.URLField(blank=True)
     uploaded_filename = models.TextField(blank=True)
     note = models.TextField(blank=True)
-    internal_comments = models.TextField(blank=True)
     rfc_number = models.PositiveIntegerField(blank=True, null=True)  # only valid for type="rfc"
 
     def file_extension(self):
diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py
index 6bb6ffa281..bba57013b9 100644
--- a/ietf/doc/resources.py
+++ b/ietf/doc/resources.py
@@ -130,7 +130,6 @@ class Meta:
             "external_url": ALL,
             "uploaded_filename": ALL,
             "note": ALL,
-            "internal_comments": ALL,
             "name": ALL,
             "type": ALL_WITH_RELATIONS,
             "stream": ALL_WITH_RELATIONS,
@@ -247,7 +246,6 @@ class Meta:
             "external_url": ALL,
             "uploaded_filename": ALL,
             "note": ALL,
-            "internal_comments": ALL,
             "name": ALL,
             "type": ALL_WITH_RELATIONS,
             "stream": ALL_WITH_RELATIONS,
diff --git a/ietf/secr/static/css/custom.css b/ietf/secr/static/css/custom.css
index 8816b3f13d..8a622cba5d 100644
--- a/ietf/secr/static/css/custom.css
+++ b/ietf/secr/static/css/custom.css
@@ -319,11 +319,6 @@ input.draft-file-input {
     width: 4em;
 }
 
-.draft-container #id_internal_comments {
-    height: 4em;
-    width: 40em;
-}
-
 .draft-container #id_abstract {
     height: 15em;
     width: 40em;
@@ -842,4 +837,4 @@ td, th, li, h2 {
 
 thead th {
     font-size: 12px;
-}
\ No newline at end of file
+}

From 9b4671c076d5393117abda5faf6256dd8dcdc62a Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Fri, 23 Aug 2024 02:25:03 +1200
Subject: [PATCH 048/656] fix: Install weasyprint as xml2rfc extra (#7845)

Since xml2rfc v3.23.0, WeasyPrint version is pinned.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 8187c1cebf..88b78e665b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -71,5 +71,5 @@ tqdm>=4.64.0
 Unidecode>=1.3.4
 urllib3>=2
 weasyprint>=59
-xml2rfc>=3.12.4
+xml2rfc[pdf]>=3.23.0
 xym>=0.6,<1.0

From 2230242b9b7c6f3c64a0e32d96ce6ac004dffe54 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 22 Aug 2024 12:34:52 -0300
Subject: [PATCH 049/656] refactor: speed up finalizing proceedings (#7846)

* chore: increase nginx proxy_read_timeout

* refactor: speed up bluesheet_data

The affiliation helper was extremely slow. Using
queryset annotation speeds it up by almost an order
of magnitude.

* chore: delint
---
 ietf/meeting/utils.py      | 31 ++++++++++++++++++++-----------
 k8s/nginx-auth.conf        |  4 ++++
 k8s/nginx-datatracker.conf |  4 ++++
 3 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index e3d8c830ee..b68a311f5d 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -12,7 +12,8 @@
 
 from django.conf import settings
 from django.contrib import messages
-from django.db.models import Q
+from django.db.models import OuterRef, Subquery, TextField, Q, Value
+from django.db.models.functions import Coalesce
 from django.template.loader import render_to_string
 from django.utils import timezone
 from django.utils.encoding import smart_str
@@ -149,19 +150,27 @@ def create_proceedings_templates(meeting):
 
 
 def bluesheet_data(session):
-    def affiliation(meeting, person):
-        # from OidcExtraScopeClaims.scope_registration()
-        email_list = person.email_set.values_list("address")
-        q = Q(person=person, meeting=meeting) | Q(email__in=email_list, meeting=meeting)
-        reg = MeetingRegistration.objects.filter(q).exclude(affiliation="").first()
-        return reg.affiliation if reg else ""
-
-    attendance = Attended.objects.filter(session=session).order_by("time")
-    meeting = session.meeting
+    attendance = (
+        Attended.objects.filter(session=session)
+        .annotate(
+            affiliation=Coalesce(
+                Subquery(
+                    MeetingRegistration.objects.filter(
+                        Q(meeting=session.meeting),
+                        Q(person=OuterRef("person")) | Q(email=OuterRef("person__email")),
+                    ).values("affiliation")[:1]
+                ),
+                Value(""),
+                output_field=TextField(),
+            )
+        ).distinct()
+        .order_by("time")
+    )
+
     return [
         {
             "name": attended.person.plain_name(),
-            "affiliation": affiliation(meeting, attended.person),
+            "affiliation": attended.affiliation,
         }
         for attended in attendance
     ]
diff --git a/k8s/nginx-auth.conf b/k8s/nginx-auth.conf
index 6dd5d6ed56..a38b8f50c7 100644
--- a/k8s/nginx-auth.conf
+++ b/k8s/nginx-auth.conf
@@ -34,5 +34,9 @@ server {
         proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
         proxy_set_header X-Real-IP $${keepempty}remote_addr;
         proxy_pass http://localhost:8000;
+        # Set timeouts longer than Cloudflare proxy limits
+        proxy_connect_timeout 60;  # nginx default (Cf = 15)
+        proxy_read_timeout 120;  # nginx default = 60 (Cf = 100) 
+        proxy_send_timeout 60;  # nginx default = 60 (Cf = 30)
     }
 }
diff --git a/k8s/nginx-datatracker.conf b/k8s/nginx-datatracker.conf
index 5cbc22e6c7..7c0dc85fd0 100644
--- a/k8s/nginx-datatracker.conf
+++ b/k8s/nginx-datatracker.conf
@@ -23,6 +23,10 @@ server {
         proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
         proxy_set_header X-Real-IP $${keepempty}remote_addr;
         proxy_pass http://localhost:8000;
+        # Set timeouts longer than Cloudflare proxy limits
+        proxy_connect_timeout 60;  # nginx default (Cf = 15)
+        proxy_read_timeout 120;  # nginx default = 60 (Cf = 100) 
+        proxy_send_timeout 60;  # nginx default = 60 (Cf = 30)
         client_max_body_size 0;  # disable size check
     }
 }

From af21347b67a5663b46c41d9c91f7e0ef4ef0b920 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Sat, 24 Aug 2024 10:50:31 +1200
Subject: [PATCH 050/656] fix: Dark mode bug, pinia store reset across Vite
 entry points (#7851)

---
 client/embedded.js                      | 4 ++--
 client/main.js                          | 4 ++--
 client/shared/create-pinia-singleton.js | 6 ++++++
 3 files changed, 10 insertions(+), 4 deletions(-)
 create mode 100644 client/shared/create-pinia-singleton.js

diff --git a/client/embedded.js b/client/embedded.js
index 3b104650cf..0509c0aecf 100644
--- a/client/embedded.js
+++ b/client/embedded.js
@@ -1,11 +1,11 @@
 import { createApp } from 'vue'
-import { createPinia } from 'pinia'
 import piniaPersist from 'pinia-plugin-persist'
 import Embedded from './Embedded.vue'
+import { createPiniaSingleton } from './shared/create-pinia-singleton'
 
 // Initialize store (Pinia)
 
-const pinia = createPinia()
+const pinia = createPiniaSingleton()
 pinia.use(piniaPersist)
 
 // Mount App
diff --git a/client/main.js b/client/main.js
index 0dc5cf32e0..3fbad907b1 100644
--- a/client/main.js
+++ b/client/main.js
@@ -1,14 +1,14 @@
 import { createApp } from 'vue'
-import { createPinia } from 'pinia'
 import piniaPersist from 'pinia-plugin-persist'
 import App from './App.vue'
 import router from './router'
+import { createPiniaSingleton } from './shared/create-pinia-singleton'
 
 const app = createApp(App, {})
 
 // Initialize store (Pinia)
 
-const pinia = createPinia()
+const pinia = createPiniaSingleton()
 pinia.use(piniaPersist)
 app.use(pinia)
 
diff --git a/client/shared/create-pinia-singleton.js b/client/shared/create-pinia-singleton.js
new file mode 100644
index 0000000000..f0013245a1
--- /dev/null
+++ b/client/shared/create-pinia-singleton.js
@@ -0,0 +1,6 @@
+import { createPinia } from 'pinia'
+
+export function createPiniaSingleton(){
+  window.pinia = window.pinia ?? createPinia()
+  return window.pinia
+}

From bece8fd71bb58e214acf1c02ba1ccbe000d23dd6 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Tue, 27 Aug 2024 01:03:15 +1200
Subject: [PATCH 051/656] feat: Agenda URL #hash scrolls to 'now' or specific
 day (#7772)

* feat: Agenda URL #now scroll to current event

* fix: agendaData date offset for dev purposes

* feat: scroll to day

* fix: Showing correct hostname in agenda modify log

* feat: mobile menu scroll to hash and general bug fixes

* fix: agenda mobile menu formatting and Playwright selectors

* fix: removing spurious ? mark

* chore: removing redundant agenda time setter in favour of agenda settings panel for debugging

* style: Update AgendaMobileBar.vue

* style: Update AgendaScheduleList.vue

---------

Co-authored-by: Nicolas Giard <github@ngpixel.com>
---
 client/agenda/Agenda.vue                |  2 +-
 client/agenda/AgendaMobileBar.vue       | 54 +++++++++++++++++++++----
 client/agenda/AgendaQuickAccess.vue     | 10 ++---
 client/agenda/AgendaScheduleList.vue    | 29 ++++++++++++-
 client/agenda/store.js                  |  7 +++-
 playwright/tests/meeting/agenda.spec.js |  2 +-
 6 files changed, 85 insertions(+), 19 deletions(-)

diff --git a/client/agenda/Agenda.vue b/client/agenda/Agenda.vue
index 40dae008a4..99a46f4fe9 100644
--- a/client/agenda/Agenda.vue
+++ b/client/agenda/Agenda.vue
@@ -434,7 +434,7 @@ function reconnectScrollObservers () {
   scrollObserver.disconnect()
   visibleDays.length = 0
   for (const mDay of agendaStore.meetingDays) {
-    const el = document.getElementById(`agenda-day-${mDay.slug}`)
+    const el = document.getElementById(mDay.slug)
     el.dataset.dayId = mDay.slug.toString()
     el.dataset.dayTs = mDay.ts
     scrollObserver.observe(el)
diff --git a/client/agenda/AgendaMobileBar.vue b/client/agenda/AgendaMobileBar.vue
index 5d15e9e692..78af96e255 100644
--- a/client/agenda/AgendaMobileBar.vue
+++ b/client/agenda/AgendaMobileBar.vue
@@ -29,7 +29,6 @@
 
 <script setup>
 import { computed, h } from 'vue'
-
 import {
   NBadge,
   NDropdown,
@@ -51,21 +50,48 @@ const siteStore = useSiteStore()
 
 // Meeting Days
 
+function optionToLink(opts){
+  const { key, label, icon } = opts
+
+  return {
+    ...opts,
+    type: 'render',
+    render: () => h(
+      'a',
+      {
+        class: 'dropdown-link',
+        'data-testid': 'mobile-link',
+        href: `#${key}`
+      },
+      [
+        h(
+          'span',
+          icon()
+        ),
+        h(
+          'span',
+          label
+        )
+      ]
+    )
+  }
+}
+
 const jumpToDayOptions = computed(() => {
   const days = []
   if (agendaStore.isMeetingLive) {
-    days.push({
+    days.push(optionToLink({
       label: 'Jump to Now',
       key: 'now',
       icon: () => h('i', { class: 'bi bi-arrow-down-right-square text-red' })
-    })
+    }))
   }
   for (const day of agendaStore.meetingDays) {
-    days.push({
+    days.push(optionToLink({
       label: `Jump to ${day.label}`,
       key: day.slug,
       icon: () => h('i', { class: 'bi bi-arrow-down-right-square' })
-    })
+    }))
   }
   return days
 })
@@ -90,14 +116,13 @@ const downloadIcsOptions = [
 function jumpToDay (dayId) {
   if (dayId === 'now') {
     const lastEventId = agendaStore.findCurrentEventId()
-
     if (lastEventId) {
       document.getElementById(`agenda-rowid-${lastEventId}`)?.scrollIntoView(true)
     } else {
       message.warning('There is no event happening right now.')
     }
   } else {
-    document.getElementById(`agenda-day-${dayId}`)?.scrollIntoView(true)
+    document.getElementById(dayId)?.scrollIntoView(true)
   }
 }
 
@@ -162,4 +187,19 @@ function downloadIcs (key) {
     }
   }
 }
+
+.dropdown-link {
+  display: flex;
+  text-decoration:none;
+  gap: 0.2rem 0.5rem;
+  padding: 0.5em;
+  color: var(--bs-body-color);
+  
+  &:hover,
+  &:focus {
+    background-color: var(--bs-dark-bg-subtle);
+    text-decoration: underline;
+  }
+}
+
 </style>
diff --git a/client/agenda/AgendaQuickAccess.vue b/client/agenda/AgendaQuickAccess.vue
index 99adc1027e..b226d09c60 100644
--- a/client/agenda/AgendaQuickAccess.vue
+++ b/client/agenda/AgendaQuickAccess.vue
@@ -99,7 +99,7 @@
             li.nav-item(v-for='day of agendaStore.meetingDays')
               a.nav-link(
                 :class='agendaStore.dayIntersectId === day.slug ? `active` : ``'
-                :href='`#slot-` + day.slug'
+                :href='`#${day.slug}`'
                 @click='scrollToDay(day.slug, $event)'
                 )
                 i.bi.bi-arrow-right-short.d-none.d-xxl-inline.me-2
@@ -109,7 +109,6 @@
 <script setup>
 import { computed, h } from 'vue'
 import { useRoute, useRouter } from 'vue-router'
-import { DateTime } from 'luxon'
 import {
   NAffix,
   NBadge,
@@ -200,14 +199,11 @@ function pickerDiscard () {
   }
 }
 
-function scrollToDay (dayId, ev) {
-  ev.preventDefault()
-  document.getElementById(`agenda-day-${dayId}`)?.scrollIntoView(true)
+function scrollToDay (daySlug, ev) {
+  document.getElementById(daySlug)?.scrollIntoView(true)
 }
 
 function scrollToNow (ev) {
-  ev.preventDefault()
-
   const lastEventId = agendaStore.findCurrentEventId()
 
   if (lastEventId) {
diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index a6d2b9a5aa..2869992579 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -24,7 +24,7 @@
           )
           //- ROW - DAY HEADING -----------------------
           template(v-if='item.displayType === `day`')
-            td(:id='`agenda-day-` + item.id', :colspan='pickerModeActive ? 6 : 5') {{item.date}}
+            td(:id='item.slug', :colspan='pickerModeActive ? 6 : 5') {{item.date}}
           //- ROW - SESSION HEADING -------------------
           template(v-else-if='item.displayType === `session-head`')
             td.agenda-table-cell-check(v-if='pickerModeActive') &nbsp;
@@ -200,7 +200,7 @@ import {
 
 import AgendaDetailsModal from './AgendaDetailsModal.vue'
 
-import { useAgendaStore } from './store'
+import { useAgendaStore, daySlugPrefix, daySlug } from './store'
 import { useSiteStore } from '../shared/store'
 import { getUrl } from '../shared/urls'
 
@@ -248,6 +248,7 @@ const meetingEvents = computed(() => {
     if (itemDate.toISODate() !== acc.lastDate) {
       acc.result.push({
         id: item.id,
+        slug: daySlug(item),
         key: `day-${itemDate.toISODate()}`,
         displayType: 'day',
         date: itemDate.toLocaleString(DateTime.DATE_HUGE),
@@ -575,6 +576,30 @@ function recalculateRedLine () {
   }
 }
 
+/**
+ * On page load when browser location hash contains '#now' or '#agenda-day-*' then scroll accordingly
+ */
+;(function scrollToHashInit() {
+  if (!window.location.hash) {
+    return
+  }
+  if (!(window.location.hash === "#now" || window.location.hash.startsWith(`#${daySlugPrefix}`))) {
+    return
+  }
+  const unsubscribe = agendaStore.$subscribe((_mutation, agendaStoreState) => {
+    if (agendaStoreState.schedule.length === 0) {
+      return
+    }
+    unsubscribe() // we only need to scroll once, so unsubscribe from future updates
+    if(window.location.hash === "#now") {
+      const lastEventId = agendaStore.findCurrentEventId()
+      document.getElementById(`agenda-rowid-${lastEventId}`)?.scrollIntoView(true)
+    } else if(window.location.hash.startsWith(`#${daySlugPrefix}`)) {
+      document.getElementById(window.location.hash.substring(1))?.scrollIntoView(true)
+    }
+  })
+})()
+
 // MOUNTED
 
 onMounted(() => {
diff --git a/client/agenda/store.js b/client/agenda/store.js
index 18c3e8c657..71c1219725 100644
--- a/client/agenda/store.js
+++ b/client/agenda/store.js
@@ -121,7 +121,7 @@ export const useAgendaStore = defineStore('agenda', {
     meetingDays () {
       const siteStore = useSiteStore()
       return uniqBy(this.scheduleAdjusted, 'adjustedStartDate').sort().map(s => ({
-        slug: s.id.toString(),
+        slug: daySlug(s),
         ts: s.adjustedStartDate,
         label: siteStore.viewport < 1350 ? DateTime.fromISO(s.adjustedStartDate).toFormat('ccc LLL d') : DateTime.fromISO(s.adjustedStartDate).toLocaleString(DateTime.DATE_HUGE)
       }))
@@ -292,3 +292,8 @@ function findFirstConferenceUrl (txt) {
   } catch (err) { }
   return null
 }
+
+export const daySlugPrefix = 'agenda-day-'
+export function daySlug(s) {
+  return `${daySlugPrefix}${s.adjustedStartDate}` // eg 'agenda-day-2024-08-13'
+}
diff --git a/playwright/tests/meeting/agenda.spec.js b/playwright/tests/meeting/agenda.spec.js
index 4bd870ebc9..109e1b3b81 100644
--- a/playwright/tests/meeting/agenda.spec.js
+++ b/playwright/tests/meeting/agenda.spec.js
@@ -1431,7 +1431,7 @@ test.describe('past - small screens', () => {
 
         // can open the jump to day dropdown
         await barBtnLocator.first().click()
-        const jumpDayDdnLocator = page.locator('.n-dropdown-menu > .n-dropdown-option')
+        const jumpDayDdnLocator = page.locator('.n-dropdown-menu [data-testid=mobile-link]')
         await expect(jumpDayDdnLocator).toHaveCount(7)
         for (let idx = 0; idx < 7; idx++) {
           const localDateTime = DateTime.fromISO(meetingData.meeting.startDate, { zone: meetingData.meeting.timezone })

From 715edafcdca16ea89797628d9cea2c3d2ee5ac36 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Thu, 29 Aug 2024 00:51:05 +1200
Subject: [PATCH 052/656] fix: return to ballotpopup (#7858)

* fix: return to ballotpopup

* fix: ballot position return to in modals

* fix: ballot return to test

* fix: ballot return to additional allow handlers

* fix: ballot return to handler syntax error in test
---
 ietf/doc/templatetags/ballot_icon.py | 13 +++++++++++--
 ietf/doc/tests_ballot.py             | 20 ++++++++------------
 ietf/doc/views_ballot.py             | 13 +++++++++++++
 ietf/doc/views_doc.py                | 10 +++++++++-
 ietf/templates/doc/ballot_popup.html |  2 +-
 5 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/ietf/doc/templatetags/ballot_icon.py b/ietf/doc/templatetags/ballot_icon.py
index ebcc605cd5..5f59822c7b 100644
--- a/ietf/doc/templatetags/ballot_icon.py
+++ b/ietf/doc/templatetags/ballot_icon.py
@@ -96,9 +96,14 @@ def sort_key(t):
     positions = list(ballot.active_balloter_positions().items())
     positions.sort(key=sort_key)
 
+    request = context.get("request")
+    ballot_edit_return_point_param = f"ballot_edit_return_point={request.path}"
+
     right_click_string = ''
     if has_role(user, "Area Director"):
-        right_click_string = 'oncontextmenu="window.location.href=\'%s\';return false;"' %  urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=doc.name, ballot_id=ballot.pk))
+        right_click_string = 'oncontextmenu="window.location.href=\'{}?{}\';return false;"'.format(
+            urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=doc.name, ballot_id=ballot.pk)),
+            ballot_edit_return_point_param)
 
     my_blocking = False
     for i, (balloter, pos) in enumerate(positions):
@@ -113,10 +118,14 @@ def sort_key(t):
         typename = "RSAB"
     else:
         typename = "IESG"
+    
+    modal_url = "{}?{}".format(
+        urlreverse("ietf.doc.views_doc.ballot_popup", kwargs=dict(name=doc.name, ballot_id=ballot.pk)),
+        ballot_edit_return_point_param)
 
     res = ['<a %s href="%s" data-bs-toggle="modal" data-bs-target="#modal-%d" aria-label="%s positions" title="%s positions (click to show more)" class="ballot-icon"><table' % (
             right_click_string,
-            urlreverse("ietf.doc.views_doc.ballot_popup", kwargs=dict(name=doc.name, ballot_id=ballot.pk)),
+            modal_url,
             ballot.pk,
             typename,
             typename,)]
diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py
index 2403b6ef5c..d77e2ddc98 100644
--- a/ietf/doc/tests_ballot.py
+++ b/ietf/doc/tests_ballot.py
@@ -1455,18 +1455,14 @@ def test_document_ballot_content_without_send_email_values(self):
 
 class ReturnToUrlTests(TestCase):
     def test_invalid_return_to_url(self):
-        self.assertRaises(
-            Exception,
-            lambda: parse_ballot_edit_return_point('/doc/', 'draft-ietf-opsawg-ipfix-tcpo-v6eh', '998718'),
-        )
-        self.assertRaises(
-            Exception,
-            lambda: parse_ballot_edit_return_point('/a-route-that-does-not-exist/', 'draft-ietf-opsawg-ipfix-tcpo-v6eh', '998718'),
-        )
-        self.assertRaises(
-            Exception,
-            lambda: parse_ballot_edit_return_point('https://example.com/phishing', 'draft-ietf-opsawg-ipfix-tcpo-v6eh', '998718'),
-        )
+        with self.assertRaises(ValueError):
+            parse_ballot_edit_return_point('/', 'draft-ietf-opsawg-ipfix-tcpo-v6eh', '998718')
+
+        with self.assertRaises(ValueError):
+            parse_ballot_edit_return_point('/a-route-that-does-not-exist/', 'draft-ietf-opsawg-ipfix-tcpo-v6eh', '998718')
+
+        with self.assertRaises(ValueError):
+            parse_ballot_edit_return_point('https://example.com/phishing', 'draft-ietf-opsawg-ipfix-tcpo-v6eh', '998718')
 
     def test_valid_default_return_to_url(self):
         self.assertEqual(parse_ballot_edit_return_point(
diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py
index 357bd2fa2f..8d9ab0b740 100644
--- a/ietf/doc/views_ballot.py
+++ b/ietf/doc/views_ballot.py
@@ -1314,10 +1314,23 @@ def rsab_ballot_status(request):
 def parse_ballot_edit_return_point(path, doc_name, ballot_id):
     get_default_path = lambda: urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc_name, ballot_id=ballot_id))
     allowed_path_handlers = {
+        "ietf.community.views.view_list",
         "ietf.doc.views_doc.document_ballot",
         "ietf.doc.views_doc.document_irsg_ballot",
         "ietf.doc.views_doc.document_rsab_ballot",
+        "ietf.doc.views_ballot.irsg_ballot_status",
+        "ietf.doc.views_ballot.rsab_ballot_status",
+        "ietf.doc.views_search.search",
+        "ietf.doc.views_search.docs_for_ad",
+        "ietf.doc.views_search.drafts_in_last_call",
+        "ietf.doc.views_search.recent_drafts",
+        "ietf.group.views.chartering_groups",
+        "ietf.group.views.group_documents",
+        "ietf.group.views.stream_documents",
         "ietf.iesg.views.agenda",
         "ietf.iesg.views.agenda_documents",
+        "ietf.iesg.views.discusses",
+        "ietf.iesg.views.past_documents",
     }
     return validate_return_to_path(path, get_default_path, allowed_path_handlers)
+
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 0ef132c215..d12dd04181 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -43,7 +43,7 @@
 
 from django.core.cache import caches
 from django.db.models import Max
-from django.http import HttpResponse, Http404
+from django.http import HttpResponse, Http404, HttpResponseBadRequest
 from django.shortcuts import render, get_object_or_404, redirect
 from django.template.loader import render_to_string
 from django.urls import reverse as urlreverse
@@ -73,6 +73,7 @@
     role_required, is_individual_draft_author, can_request_rfc_publication)
 from ietf.name.models import StreamName, BallotPositionName
 from ietf.utils.history import find_history_active_at
+from ietf.doc.views_ballot import parse_ballot_edit_return_point
 from ietf.doc.forms import InvestigateForm, TelechatForm, NotifyForm, ActionHoldersForm, DocAuthorForm, DocAuthorChangeBasisForm
 from ietf.doc.mails import email_comment, email_remind_action_holders
 from ietf.mailtrigger.utils import gather_relevant_expansions
@@ -1586,11 +1587,18 @@ def ballot_popup(request, name, ballot_id):
     doc = get_object_or_404(Document, name=name)
     c = document_ballot_content(request, doc, ballot_id=ballot_id, editable=False)
     ballot = get_object_or_404(BallotDocEvent,id=ballot_id)
+    
+    try:
+        return_to_url = parse_ballot_edit_return_point(request.GET.get('ballot_edit_return_point'), name, ballot_id)
+    except ValueError:
+        return HttpResponseBadRequest('ballot_edit_return_point is invalid')
+    
     return render(request, "doc/ballot_popup.html",
                               dict(doc=doc,
                                    ballot_content=c,
                                    ballot_id=ballot_id,
                                    ballot_type_slug=ballot.ballot_type.slug,
+                                   ballot_edit_return_point=return_to_url,
                                    editable=True,
                                    ))
 
diff --git a/ietf/templates/doc/ballot_popup.html b/ietf/templates/doc/ballot_popup.html
index 2c0863ab6c..d2589cd54c 100644
--- a/ietf/templates/doc/ballot_popup.html
+++ b/ietf/templates/doc/ballot_popup.html
@@ -27,7 +27,7 @@
             {% if editable and user|has_role:"Area Director,Secretariat,IRSG Member,RSAB Member" %}
                 {% if user|can_ballot:doc %}
                     <a class="btn btn-primary"
-                       href="{% url "ietf.doc.views_ballot.edit_position" name=doc.name ballot_id=ballot_id %}?ballot_edit_return_point={{ request.path|urlencode }}">
+                       href="{% url "ietf.doc.views_ballot.edit_position" name=doc.name ballot_id=ballot_id %}?ballot_edit_return_point={{ ballot_edit_return_point|urlencode }}">
                         Edit position
                     </a>
                 {% endif %}

From 1f6db0fef1e92b38fbd756e71b2faad7e1595998 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 28 Aug 2024 14:17:14 -0500
Subject: [PATCH 053/656] feat: mark bofreqs as spam (#7869)

---
 ietf/doc/migrations/0023_bofreqspamstate.py | 30 +++++++++
 ietf/doc/tests_bofreq.py                    | 11 +++-
 ietf/name/fixtures/names.json               | 13 ++++
 ietf/templates/doc/bofreq/bof_requests.html | 68 +++++++++++----------
 4 files changed, 87 insertions(+), 35 deletions(-)
 create mode 100644 ietf/doc/migrations/0023_bofreqspamstate.py

diff --git a/ietf/doc/migrations/0023_bofreqspamstate.py b/ietf/doc/migrations/0023_bofreqspamstate.py
new file mode 100644
index 0000000000..dbbaf996e9
--- /dev/null
+++ b/ietf/doc/migrations/0023_bofreqspamstate.py
@@ -0,0 +1,30 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    State = apps.get_model("doc", "State")
+    State.objects.get_or_create(
+        type_id="bofreq",
+        slug="spam",
+        defaults={"name": "Spam", "desc": "The BOF request is spam", "order": 5},
+    )
+
+
+def reverse(apps, schema_editor):
+    State = apps.get_model("doc", "State")
+    Document = apps.get_model("doc", "Document")
+    assert not Document.objects.filter(
+        states__type="bofreq", states__slug="spam"
+    ).exists()
+    State.objects.filter(type_id="bofreq", slug="spam").delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("doc", "0022_remove_dochistory_internal_comments_and_more"),
+    ]
+
+    operations = [migrations.RunPython(forward, reverse)]
diff --git a/ietf/doc/tests_bofreq.py b/ietf/doc/tests_bofreq.py
index 319ab778d6..2e27efd627 100644
--- a/ietf/doc/tests_bofreq.py
+++ b/ietf/doc/tests_bofreq.py
@@ -54,8 +54,8 @@ def test_show_bof_requests(self):
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         for state in states:
-            self.assertEqual(len(q(f'#bofreqs-{state.slug}')), 1)
-            self.assertEqual(len(q(f'#bofreqs-{state.slug} tbody tr')), 3)
+            self.assertEqual(len(q(f'#bofreqs-{state.slug}')), 1 if state.slug!="spam" else 0)
+            self.assertEqual(len(q(f'#bofreqs-{state.slug} tbody tr')), 3 if state.slug!="spam" else 0)
         self.assertFalse(q('#start_button'))
         PersonFactory(user__username='nobody')
         self.client.login(username='nobody', password='nobody+password')
@@ -63,6 +63,13 @@ def test_show_bof_requests(self):
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertTrue(q('#start_button'))
+        self.client.logout()
+        self.client.login(username='secretary', password='secretary+password')
+        r = self.client.get(url)
+        q = PyQuery(r.content)
+        for state in states:
+            self.assertEqual(len(q(f'#bofreqs-{state.slug}')), 1)
+            self.assertEqual(len(q(f'#bofreqs-{state.slug} tbody tr')), 3)
 
 
     def test_bofreq_main_page(self):
diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json
index 59b367deb8..401767657d 100644
--- a/ietf/name/fixtures/names.json
+++ b/ietf/name/fixtures/names.json
@@ -2617,6 +2617,19 @@
     "model": "doc.state",
     "pk": 180
   },
+  {
+    "fields": {
+      "desc": "The BOF request is spam",
+      "name": "Spam",
+      "next_states": [],
+      "order": 5,
+      "slug": "spam",
+      "type": "bofreq",
+      "used": true
+    },
+    "model": "doc.state",
+    "pk": 182
+  },
   {
     "fields": {
       "label": "State"
diff --git a/ietf/templates/doc/bofreq/bof_requests.html b/ietf/templates/doc/bofreq/bof_requests.html
index 2a806b18a5..19c2e5dd13 100644
--- a/ietf/templates/doc/bofreq/bof_requests.html
+++ b/ietf/templates/doc/bofreq/bof_requests.html
@@ -1,7 +1,7 @@
 {% extends "base.html" %}
 {# Copyright The IETF Trust 2021 All Rights Reserved #}
 {% load origin %}
-{% load person_filters %}
+{% load person_filters ietf_filters %}
 {% load static %}
 {% block pagehead %}
     <link rel="stylesheet" href="{% static "ietf/css/list.css" %}">
@@ -26,40 +26,42 @@ <h1>BOF Requests</h1>
     {% else %}
         {% regroup reqs by get_state_slug as grouped_reqs %}
         {% for req_group in grouped_reqs %}
-            <h2 class="mt-5">{{ req_group.grouper|capfirst }} BOF Requests</h2>
-            <table id="bofreqs-{{ req_group.grouper }}"
-                   class="table table-sm table-striped tablesorter">
-                <thead>
-                    <tr>
-                        <th scope="col" data-sort="name">Name</th>
-                        <th scope="col" class="d-none d-sm-table-cell" data-sort="date">Date</th>
-                        <th scope="col" data-sort="title">Title</th>
-                        <th scope="col" data-sort="responsible">Responsible</th>
-                        <th scope="col" data-sort="editors">Editors</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    {% for req in req_group.list %}
+            {% if req_group.grouper != "spam" or request.user|has_role:"Secretariat" %}
+                <h2 class="mt-5">{{ req_group.grouper|capfirst }} BOF Requests</h2>
+                <table id="bofreqs-{{ req_group.grouper }}"
+                    class="table table-sm table-striped tablesorter">
+                    <thead>
                         <tr>
-                            <td>
-                                <a href="{% url 'ietf.doc.views_doc.document_main' name=req.name %}">{{ req.name }}-{{ req.rev }}</a>
-                            </td>
-                            <td class="d-none d-sm-table-cell">{{ req.latest_revision_event.time|date:"Y-m-d" }}</td>
-                            <td>{{ req.title }}</td>
-                            <td>
-                                {% for person in req.responsible %}
-                                    {% person_link person %}{% if not forloop.last %},{% endif %}
-                                {% endfor %}
-                            </td>
-                            <td>
-                                {% for person in req.editors %}
-                                    {% person_link person %}{% if not forloop.last %},{% endif %}
-                                {% endfor %}
-                            </td>
+                            <th scope="col" data-sort="name">Name</th>
+                            <th scope="col" class="d-none d-sm-table-cell" data-sort="date">Date</th>
+                            <th scope="col" data-sort="title">Title</th>
+                            <th scope="col" data-sort="responsible">Responsible</th>
+                            <th scope="col" data-sort="editors">Editors</th>
                         </tr>
-                    {% endfor %}
-                </tbody>
-            </table>
+                    </thead>
+                    <tbody>
+                        {% for req in req_group.list %}
+                            <tr>
+                                <td>
+                                    <a href="{% url 'ietf.doc.views_doc.document_main' name=req.name %}">{{ req.name }}-{{ req.rev }}</a>
+                                </td>
+                                <td class="d-none d-sm-table-cell">{{ req.latest_revision_event.time|date:"Y-m-d" }}</td>
+                                <td>{{ req.title }}</td>
+                                <td>
+                                    {% for person in req.responsible %}
+                                        {% person_link person %}{% if not forloop.last %},{% endif %}
+                                    {% endfor %}
+                                </td>
+                                <td>
+                                    {% for person in req.editors %}
+                                        {% person_link person %}{% if not forloop.last %},{% endif %}
+                                    {% endfor %}
+                                </td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            {% endif %}
         {% endfor %}
     {% endif %}
 {% endblock %}

From 73968dca5f0ee2450080b585d6da162bdc870955 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 28 Aug 2024 14:17:36 -0500
Subject: [PATCH 054/656] fix: use rsab mailtrigger for To in ballot position
 email (#7868)

---
 ietf/doc/tests_ballot.py | 3 +++
 ietf/doc/views_ballot.py | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py
index d77e2ddc98..0c99ed281a 100644
--- a/ietf/doc/tests_ballot.py
+++ b/ietf/doc/tests_ballot.py
@@ -230,6 +230,9 @@ def test_cannot_edit_position_as_pre_ad(self):
         r = self.client.post(url, dict(position="discuss", discuss="Test discuss text"))
         self.assertEqual(r.status_code, 403)
         
+    # N.B. This test needs to be rewritten to exercise all types of ballots (iesg, irsg, rsab)
+    # and test against the output of the mailtriggers instead of looking for hardcoded values
+    # in the To and CC results. See #7864
     def test_send_ballot_comment(self):
         ad = Person.objects.get(user__username="ad")
         draft = WgDraftFactory(ad=ad,group__acronym='mars')
diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py
index 8d9ab0b740..2b7faac92b 100644
--- a/ietf/doc/views_ballot.py
+++ b/ietf/doc/views_ballot.py
@@ -323,6 +323,8 @@ def build_position_email(balloter, doc, pos):
 
     if doc.stream_id == "irtf":
         addrs = gather_address_lists('irsg_ballot_saved',doc=doc)
+    elif doc.stream_id == "editorial":
+        addrs = gather_address_lists('rsab_ballot_saved',doc=doc)
     else:
         addrs = gather_address_lists('iesg_ballot_saved',doc=doc)
 

From 17bd3124686c33c1089a179e32a2d57f4856a54a Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Thu, 29 Aug 2024 07:32:58 +1200
Subject: [PATCH 055/656] fix: Document type badges (#7760)

* fix: document type badge#7475

* chore: PR feedback (naming)

* fix: use type_id rather than str(type) for conditionals in templatetag

* fix: remove debugging CSS inline style dashed border

* fix: document type badge template tag passing context variables

* fix: context vars to document_type_badge

* fix: badge error logging

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/doc/templatetags/document_type_badge.py  | 29 +++++++++++++++++++
 ietf/static/css/document_html.scss            |  5 ++++
 ietf/templates/doc/badge/doc-badge-draft.html | 16 ++++++++++
 ietf/templates/doc/badge/doc-badge-rfc.html   | 13 +++++++++
 ietf/templates/doc/document_html.html         | 10 ++-----
 ietf/templates/doc/document_info.html         | 19 ++----------
 6 files changed, 68 insertions(+), 24 deletions(-)
 create mode 100644 ietf/doc/templatetags/document_type_badge.py
 create mode 100644 ietf/templates/doc/badge/doc-badge-draft.html
 create mode 100644 ietf/templates/doc/badge/doc-badge-rfc.html

diff --git a/ietf/doc/templatetags/document_type_badge.py b/ietf/doc/templatetags/document_type_badge.py
new file mode 100644
index 0000000000..a82c606ff9
--- /dev/null
+++ b/ietf/doc/templatetags/document_type_badge.py
@@ -0,0 +1,29 @@
+# Copyright The IETF Trust 2015-2020, All Rights Reserved
+from django import template
+from django.conf import settings
+from django.template.loader import render_to_string
+from ietf.utils.log import log
+
+register = template.Library()
+
+
+@register.simple_tag
+def document_type_badge(doc, snapshot, submission, resurrected_by):
+    context = {"doc": doc, "snapshot": snapshot, "submission": submission, "resurrected_by": resurrected_by}
+    if doc.type_id == "rfc":
+        return render_to_string(
+            "doc/badge/doc-badge-rfc.html",
+            context,
+        )
+    elif doc.type_id == "draft":
+        return render_to_string(
+            "doc/badge/doc-badge-draft.html",
+            context,
+        )
+    else:
+        error_message = f"Unsupported document type {doc.type_id}."
+        if settings.SERVER_MODE != 'production':
+            raise ValueError(error_message)
+        else:
+            log(error_message)
+        return ""
diff --git a/ietf/static/css/document_html.scss b/ietf/static/css/document_html.scss
index 0b5a70f124..47ef8d64b4 100644
--- a/ietf/static/css/document_html.scss
+++ b/ietf/static/css/document_html.scss
@@ -320,6 +320,11 @@ tbody.meta tr {
     background-color: $danger;
 }
 
+.badge-generic {
+    color: white;
+    background-color: $danger;
+}
+
 #toc-nav {
     width: inherit;
     overscroll-behavior-y: none; // Prevent overscrolling from scrolling the main content
diff --git a/ietf/templates/doc/badge/doc-badge-draft.html b/ietf/templates/doc/badge/doc-badge-draft.html
new file mode 100644
index 0000000000..f7f66b6c5e
--- /dev/null
+++ b/ietf/templates/doc/badge/doc-badge-draft.html
@@ -0,0 +1,16 @@
+{% load origin %}
+{% load static %}
+{% load ietf_filters %}
+{% load person_filters %}
+{% origin %}
+{# Non-RFC #}
+
+{% if doc.became_rfc %} 
+    <div{% if document_html %} class="alert alert-warning small"{% endif %}>This is an older version of an Internet-Draft that was ultimately published as <a href="{% if document_html %}{% url 'ietf.doc.views_doc.document_html' name=doc.became_rfc.name %}{% else %}{% url 'ietf.doc.views_doc.document_main' name=doc.became_rfc.name %}{% endif %}">{{doc.became_rfc.name|prettystdname}}</a>.</div>
+{% elif snapshot and doc.rev != latest_rev  %}
+    <div{% if document_html %} class="alert alert-warning small p-2 mt-2"{% endif %}>This is an older version of an Internet-Draft whose latest revision state is "{{ doc.doc.get_state }}".</div>
+{% else %}
+    <span class="{% if doc.get_state_slug == 'active' %}text-success{% elif doc.get_state_slug == 'expired' or doc.get_state_slug == 'repl' %}text-danger{% endif %}">{% if snapshot and doc.rev == latest_rev %}{{ doc.doc.get_state }}{% else %}{{ doc.get_state }}{% endif %} Internet-Draft</span>
+    {% if submission %}({{ submission|safe }}){% endif %}
+    {% if resurrected_by %}- resurrect requested by {{ resurrected_by }}{% endif %}
+{% endif %}
\ No newline at end of file
diff --git a/ietf/templates/doc/badge/doc-badge-rfc.html b/ietf/templates/doc/badge/doc-badge-rfc.html
new file mode 100644
index 0000000000..780f14a54f
--- /dev/null
+++ b/ietf/templates/doc/badge/doc-badge-rfc.html
@@ -0,0 +1,13 @@
+{% load origin %}
+{% load static %}
+{% load ietf_filters %}
+{% load person_filters %}
+{% origin %}
+
+<span class="text-success">RFC
+    {% if not document_html %}
+    - {{ doc.std_level }}
+    {% else %}
+        <span class="badge rounded-pill badge-{% if not snapshot %}{{ doc|std_level_to_label_format }}{% else %}generic{% endif %}">{{ doc.std_level }}</span>
+    {% endif %}
+</span>
diff --git a/ietf/templates/doc/document_html.html b/ietf/templates/doc/document_html.html
index 3eabe2ebb7..a85b1f5310 100644
--- a/ietf/templates/doc/document_html.html
+++ b/ietf/templates/doc/document_html.html
@@ -4,6 +4,7 @@
 {% load origin %}
 {% load static %}
 {% load ietf_filters textfilters %}
+{% load document_type_badge %}
 {% load django_vite %}
 {% origin %}
 <html data-bs-theme="auto" lang="en">
@@ -107,6 +108,7 @@
                     {{ doc.name }}-{{ doc.rev }}
                 {% endif %}
                 <br class="d-sm-none">
+
                 <span class="ms-sm-3 badge rounded-pill badge-{% if not snapshot %}{{ doc|std_level_to_label_format }}{% else %}draft{% endif %}">
                     {% if not snapshot %}
                         {{ doc.std_level }}
@@ -185,13 +187,7 @@
                                     {{ doc.name }}-{{ doc.rev }}
                                 {% endif %}
                                 <br>
-                                <span class="badge rounded-pill badge-{% if not snapshot %}{{ doc|std_level_to_label_format }}{% else %}draft{% endif %}">
-                                    {% if not snapshot %}
-                                        {{ doc.std_level }}
-                                    {% else %}
-                                        Internet-Draft
-                                    {% endif %}
-                                </span>
+                                {% document_type_badge doc snapshot submission resurrected_by %}
                             </p>
                         </div>
                         {% if request.COOKIES.htmlconf and request.COOKIES.htmlconf != 'html' and html %}
diff --git a/ietf/templates/doc/document_info.html b/ietf/templates/doc/document_info.html
index 97cd49cf7e..71050f9d41 100644
--- a/ietf/templates/doc/document_info.html
+++ b/ietf/templates/doc/document_info.html
@@ -3,6 +3,7 @@
 {% load static %}
 {% load ietf_filters %}
 {% load person_filters %}
+{% load document_type_badge %}
 {% origin %}
 
 <tbody class="meta align-top {% if not document_html %} border-top{% endif %}">
@@ -11,14 +12,8 @@
         <th scope="row">{% if document_html %}Document type{% else %}Type{% endif %}</th>
         <td class="edit"></td>
         <td>
+            {% document_type_badge doc snapshot submission resurrected_by %}
             {% if doc.type_id == "rfc" %}
-                <span class="text-success">RFC
-                    {% if not document_html %}
-                    - {{ doc.std_level }}
-                    {% else %}
-                        <span class="badge rounded-pill badge-{% if not snapshot %}{{ doc|std_level_to_label_format }}{% else %}draft{% endif %}">{{ doc.std_level }}</span>
-                    {% endif %}
-                </span>
                 {% if doc.pub_date %}
                     {% if document_html %}<br>{% else %}({% endif %}{{ doc.pub_date|date:"F Y" }}{% if not document_html %}){% endif %}
                 {% else %}
@@ -59,16 +54,6 @@
                         {% if submission %}({{ submission|safe }}){% endif %}
                     </div>
                 {% endif %}
-            {% else %}
-                {% if doc.became_rfc %} 
-                   <div{% if document_html %} class="alert alert-warning small"{% endif %}>This is an older version of an Internet-Draft that was ultimately published as <a href="{% if document_html %}{% url 'ietf.doc.views_doc.document_html' name=doc.became_rfc.name %}{% else %}{% url 'ietf.doc.views_doc.document_main' name=doc.became_rfc.name %}{% endif %}">{{doc.became_rfc.name|prettystdname}}</a>.</div>
-                {% elif snapshot and doc.rev != latest_rev  %}
-                    <div{% if document_html %} class="alert alert-warning small p-2 mt-2"{% endif %}>This is an older version of an Internet-Draft whose latest revision state is "{{ doc.doc.get_state }}".</div>
-                 {% else %}
-                    <span class="{% if doc.get_state_slug == 'active' %}text-success{% elif doc.get_state_slug == 'expired' or doc.get_state_slug == 'repl' %}text-danger{% endif %}">{% if snapshot and doc.rev == latest_rev %}{{ doc.doc.get_state }}{% else %}{{ doc.get_state }}{% endif %} Internet-Draft</span>
-                    {% if submission %}({{ submission|safe }}){% endif %}
-                    {% if resurrected_by %}- resurrect requested by {{ resurrected_by }}{% endif %}
-                {% endif %}
             {% endif %}
             {% if doc.get_state_slug != "active" and doc.get_state_slug != "rfc" and doc.type_id != "rfc" %}
                 <div class="badge rounded-pill text-bg-warning{% if not document_html %} float-end{% endif %}">

From 23e5307d2e097ae32749f74788ad1eff59cd6042 Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Wed, 4 Sep 2024 01:20:36 +1200
Subject: [PATCH 056/656] feat: Use meetecho-player.ietf.org for session
 recording (#7873)

---
 ietf/meeting/models.py                        | 15 ++++---
 ietf/meeting/tests_models.py                  | 44 +++++++++++++++++++
 ietf/settings.py                              |  2 +-
 .../meeting/interim_session_buttons.html      |  9 ++--
 4 files changed, 60 insertions(+), 10 deletions(-)

diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index 693cb99dfd..3470679327 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -1321,11 +1321,16 @@ def onsite_tool_url(self):
         return None
 
     def session_recording_url(self):
-        url = getattr(settings, "MEETECHO_SESSION_RECORDING_URL", "")
-        if self.meeting.type.slug == "ietf" and self.has_onsite_tool and url:
-            self.group.acronym_upper = self.group.acronym.upper()
-            return url.format(session=self)
-        return None
+        url_formatter = getattr(settings, "MEETECHO_SESSION_RECORDING_URL", "")
+        url = None
+        if url_formatter and self.video_stream_url:
+            if self.meeting.type.slug == "ietf" and self.has_onsite_tool:
+                session_label = f"IETF{self.meeting.number}-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}"
+            else:
+                session_label = f"IETF-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}"
+            url = url_formatter.format(session_label=session_label)
+
+        return url
 
 
 class SchedulingEvent(models.Model):
diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py
index 0ccd462715..ff874100dc 100644
--- a/ietf/meeting/tests_models.py
+++ b/ietf/meeting/tests_models.py
@@ -146,3 +146,47 @@ def test_chat_room_name(self):
         self.assertEqual(session.chat_room_name(), 'plenary')
         session.chat_room = 'fnord'
         self.assertEqual(session.chat_room_name(), 'fnord')
+
+    def test_session_recording_url(self):
+        group_acronym = "foobar"
+        meeting_date = datetime.date.today()
+        meeting_number = 123
+
+        # IETF meeting
+        session = SessionFactory(
+            meeting__type_id='ietf',
+            meeting__date=meeting_date,
+            group__acronym=group_acronym,
+            meeting__number=meeting_number,
+        )
+        with override_settings():
+            if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"):
+                del settings.MEETECHO_SESSION_RECORDING_URL
+            self.assertIsNone(session.session_recording_url())
+
+            settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com"
+            self.assertEqual(session.session_recording_url(), "http://player.example.com")
+
+            settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}"
+            self.assertIn(f"IETF{meeting_number}-{group_acronym.upper()}", session.session_recording_url())
+            self.assertIn(f"{meeting_date.strftime('%Y%m%d')}", session.session_recording_url())
+            self.assertTrue(session.session_recording_url().startswith("http://player.example.com"))
+
+        # interim meeting
+        session = SessionFactory(
+            meeting__type_id='interim',
+            meeting__date=meeting_date,
+            group__acronym=group_acronym,
+        )
+        with override_settings():
+            if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"):
+                del settings.MEETECHO_SESSION_RECORDING_URL
+            self.assertIsNone(session.session_recording_url())
+
+            settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com"
+            self.assertEqual(session.session_recording_url(), "http://player.example.com")
+
+            settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}"
+            self.assertIn(f"IETF-{group_acronym.upper()}", session.session_recording_url())
+            self.assertIn(f"{meeting_date.strftime('%Y%m%d')}", session.session_recording_url())
+            self.assertTrue(session.session_recording_url().startswith("http://player.example.com"))
diff --git a/ietf/settings.py b/ietf/settings.py
index db53efe0a5..1bb7a122c3 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -1174,7 +1174,7 @@ def skip_unreadable_post(record):
 MEETECHO_ONSITE_TOOL_URL = "https://meetings.conf.meetecho.com/onsite{session.meeting.number}/?session={session.pk}"
 MEETECHO_VIDEO_STREAM_URL = "https://meetings.conf.meetecho.com/ietf{session.meeting.number}/?session={session.pk}"
 MEETECHO_AUDIO_STREAM_URL = "https://mp3.conf.meetecho.com/ietf{session.meeting.number}/{session.pk}.m3u"
-MEETECHO_SESSION_RECORDING_URL = "https://www.meetecho.com/ietf{session.meeting.number}/recordings#{session.group.acronym_upper}"
+MEETECHO_SESSION_RECORDING_URL = "https://meetecho-player.ietf.org/playout/?session={session_label}"
 
 # Put the production SECRET_KEY in settings_local.py, and also any other
 # sensitive or site-specific changes.  DO NOT commit settings_local.py to svn.
diff --git a/ietf/templates/meeting/interim_session_buttons.html b/ietf/templates/meeting/interim_session_buttons.html
index a32f4345c9..23263b9859 100644
--- a/ietf/templates/meeting/interim_session_buttons.html
+++ b/ietf/templates/meeting/interim_session_buttons.html
@@ -146,17 +146,18 @@
                             {% endif %}
                         {% endwith %}
                     {% endfor %}
-                {% elif session.video_stream_url %}
+                {% elif show_empty  %}
+                    {# <i class="bi"></i> #}
+                {% endif %}
+                {% if session.session_recording_url %}
                     <a class="btn btn-outline-primary"
                        href="{{ session.session_recording_url }}"
                        aria-label="Meetecho session recording"
                        title="Meetecho session recording">
                         <i class="bi bi-file-slides"></i>
                     </a>
-                {% elif show_empty  %}
-                    {# <i class="bi"></i> #}
                 {% endif %}
             {% endwith %}
         {% endif %}
     </div>
-{% endwith %}
\ No newline at end of file
+{% endwith %}

From 061c89f3b559829d0d1e20ef6d81c33542906ff7 Mon Sep 17 00:00:00 2001
From: Seonghyeon Cho <seonghyeoncho96@gmail.com>
Date: Wed, 4 Sep 2024 00:46:34 +0900
Subject: [PATCH 057/656] fix: Missing button text for PostScript in RFC
 (#7889)

Resolves #7879

Signed-off-by: Seonghyeon Cho <seonghyeoncho96@gmail.com>
---
 ietf/doc/utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index a98b46cb50..74000e598b 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -1046,6 +1046,8 @@ def build_file_urls(doc: Union[Document, DocHistory]):
 
         file_urls = []
         for t in found_types:
+            if t == "ps": # Postscript might have been submitted but should not be displayed in the list of URLs
+                continue
             label = "plain text" if t == "txt" else t
             file_urls.append((label, base + doc.name + "." + t))
 

From b6f8ede98a7c3f18210cc72907b7a5de2a68ef51 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 3 Sep 2024 19:24:26 -0300
Subject: [PATCH 058/656] feat: is_authenticated request logging + cleanup
 (#7893)

* chore: nginx log is s, not ms

* chore: log seconds from gunicorn too

* chore: drop X-Real-IP header / log

* style: Black

* style: single -> double quotes

* feat: add is-authenticated header

* feat: log is-authenticated header

* chore: update nginx-auth.conf to match
---
 ietf/middleware.py         | 46 ++++++++++++++++++++++++++++++--------
 ietf/settings.py           | 35 +++++++++++++++--------------
 ietf/utils/jsonlogger.py   |  4 ++--
 k8s/nginx-auth.conf        |  2 +-
 k8s/nginx-datatracker.conf |  2 +-
 k8s/nginx-logging.conf     |  2 +-
 6 files changed, 60 insertions(+), 31 deletions(-)

diff --git a/ietf/middleware.py b/ietf/middleware.py
index 48146abf5e..a4b7a0d24c 100644
--- a/ietf/middleware.py
+++ b/ietf/middleware.py
@@ -17,45 +17,61 @@ def sql_log_middleware(get_response):
     def sql_log(request):
         response = get_response(request)
         for q in connection.queries:
-            if re.match('(update|insert)', q['sql'], re.IGNORECASE):
-                log(q['sql'])
+            if re.match("(update|insert)", q["sql"], re.IGNORECASE):
+                log(q["sql"])
         return response
+
     return sql_log
 
+
 class SMTPExceptionMiddleware(object):
     def __init__(self, get_response):
         self.get_response = get_response
+
     def __call__(self, request):
         return self.get_response(request)
+
     def process_exception(self, request, exception):
         if isinstance(exception, smtplib.SMTPException):
             (extype, value, tb) = log_smtp_exception(exception)
-            return render(request, 'email_failed.html',
-                          {'exception': extype, 'args': value, 'traceback': "".join(tb)} )
+            return render(
+                request,
+                "email_failed.html",
+                {"exception": extype, "args": value, "traceback": "".join(tb)},
+            )
         return None
 
+
 class Utf8ExceptionMiddleware(object):
     def __init__(self, get_response):
         self.get_response = get_response
+
     def __call__(self, request):
         return self.get_response(request)
+
     def process_exception(self, request, exception):
         if isinstance(exception, OperationalError):
             extype, e, tb = exc_parts()
             if e.args[0] == 1366:
                 log("Database 4-byte utf8 exception: %s: %s" % (extype, e))
-                return render(request, 'utf8_4byte_failed.html',
-                              {'exception': extype, 'args': e.args, 'traceback': "".join(tb)} )
+                return render(
+                    request,
+                    "utf8_4byte_failed.html",
+                    {"exception": extype, "args": e.args, "traceback": "".join(tb)},
+                )
         return None
 
+
 def redirect_trailing_period_middleware(get_response):
     def redirect_trailing_period(request):
         response = get_response(request)
         if response.status_code == 404 and request.path.endswith("."):
             return HttpResponsePermanentRedirect(request.path.rstrip("."))
         return response
+
     return redirect_trailing_period
 
+
 def unicode_nfkc_normalization_middleware(get_response):
     def unicode_nfkc_normalization(request):
         """Do Unicode NFKC normalization to turn ligatures into individual characters.
@@ -65,9 +81,21 @@ def unicode_nfkc_normalization(request):
         There are probably other elements of a request which may need this normalization
         too, but let's put that in as it comes up, rather than guess ahead.
         """
-        request.META["PATH_INFO"] = unicodedata.normalize('NFKC', request.META["PATH_INFO"])
-        request.path_info = unicodedata.normalize('NFKC', request.path_info)
+        request.META["PATH_INFO"] = unicodedata.normalize(
+            "NFKC", request.META["PATH_INFO"]
+        )
+        request.path_info = unicodedata.normalize("NFKC", request.path_info)
         response = get_response(request)
         return response
+
     return unicode_nfkc_normalization
-        
+
+
+def is_authenticated_header_middleware(get_response):
+    """Middleware to add an is-authenticated header to the response"""
+    def add_header(request):
+        response = get_response(request)
+        response["X-Datatracker-Is-Authenticated"] = "yes" if request.user.is_authenticated else "no"
+        return response
+
+    return add_header
diff --git a/ietf/settings.py b/ietf/settings.py
index 1bb7a122c3..a1a7fee102 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -401,24 +401,25 @@ def skip_unreadable_post(record):
 
 
 MIDDLEWARE = [
-    'django.middleware.csrf.CsrfViewMiddleware',
-    'corsheaders.middleware.CorsMiddleware', # see docs on CORS_REPLACE_HTTPS_REFERER before using it
-    'django.middleware.common.CommonMiddleware',
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.messages.middleware.MessageMiddleware',
-    'django.middleware.http.ConditionalGetMiddleware',
-    'simple_history.middleware.HistoryRequestMiddleware',
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "corsheaders.middleware.CorsMiddleware", # see docs on CORS_REPLACE_HTTPS_REFERER before using it
+    "django.middleware.common.CommonMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.http.ConditionalGetMiddleware",
+    "simple_history.middleware.HistoryRequestMiddleware",
     # comment in this to get logging of SQL insert and update statements:
-    #'ietf.middleware.sql_log_middleware',
-    'ietf.middleware.SMTPExceptionMiddleware',
-    'ietf.middleware.Utf8ExceptionMiddleware',
-    'ietf.middleware.redirect_trailing_period_middleware',
-    'django_referrer_policy.middleware.ReferrerPolicyMiddleware',
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
-    'django.middleware.security.SecurityMiddleware',
- #   'csp.middleware.CSPMiddleware',
-    'ietf.middleware.unicode_nfkc_normalization_middleware',
+    #"ietf.middleware.sql_log_middleware",
+    "ietf.middleware.SMTPExceptionMiddleware",
+    "ietf.middleware.Utf8ExceptionMiddleware",
+    "ietf.middleware.redirect_trailing_period_middleware",
+    "django_referrer_policy.middleware.ReferrerPolicyMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "django.middleware.security.SecurityMiddleware",
+    #"csp.middleware.CSPMiddleware",
+    "ietf.middleware.unicode_nfkc_normalization_middleware",
+    "ietf.middleware.is_authenticated_header_middleware",
 ]
 
 ROOT_URLCONF = 'ietf.urls'
diff --git a/ietf/utils/jsonlogger.py b/ietf/utils/jsonlogger.py
index 9c7949fd58..c383ba310f 100644
--- a/ietf/utils/jsonlogger.py
+++ b/ietf/utils/jsonlogger.py
@@ -23,12 +23,12 @@ def add_fields(self, log_record, record, message_dict):
         log_record.setdefault("referer", record.args["f"])
         log_record.setdefault("user_agent", record.args["a"])
         log_record.setdefault("len_bytes", record.args["B"])
-        log_record.setdefault("duration_ms", record.args["M"])
+        log_record.setdefault("duration_s", record.args["L"])  # decimal seconds
         log_record.setdefault("host", record.args["{host}i"])
         log_record.setdefault("x_request_start", record.args["{x-request-start}i"])
-        log_record.setdefault("x_real_ip", record.args["{x-real-ip}i"])
         log_record.setdefault("x_forwarded_for", record.args["{x-forwarded-for}i"])
         log_record.setdefault("x_forwarded_proto", record.args["{x-forwarded-proto}i"])
         log_record.setdefault("cf_connecting_ip", record.args["{cf-connecting-ip}i"])
         log_record.setdefault("cf_connecting_ipv6", record.args["{cf-connecting-ipv6}i"])
         log_record.setdefault("cf_ray", record.args["{cf-ray}i"])
+        log_record.setdefault("is_authenticated", record.args["{x-datatracker-is-authenticated}i"])
diff --git a/k8s/nginx-auth.conf b/k8s/nginx-auth.conf
index a38b8f50c7..95aa838064 100644
--- a/k8s/nginx-auth.conf
+++ b/k8s/nginx-auth.conf
@@ -32,7 +32,7 @@ server {
         proxy_set_header Connection close;
         proxy_set_header X-Request-Start "t=$${keepempty}msec";
         proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
-        proxy_set_header X-Real-IP $${keepempty}remote_addr;
+        proxy_hide_header X-Datatracker-Is-Authenticated; # hide this from the outside world
         proxy_pass http://localhost:8000;
         # Set timeouts longer than Cloudflare proxy limits
         proxy_connect_timeout 60;  # nginx default (Cf = 15)
diff --git a/k8s/nginx-datatracker.conf b/k8s/nginx-datatracker.conf
index 7c0dc85fd0..882d7563c2 100644
--- a/k8s/nginx-datatracker.conf
+++ b/k8s/nginx-datatracker.conf
@@ -21,7 +21,7 @@ server {
         proxy_set_header Connection close;
         proxy_set_header X-Request-Start "t=$${keepempty}msec";
         proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
-        proxy_set_header X-Real-IP $${keepempty}remote_addr;
+        proxy_hide_header X-Datatracker-Is-Authenticated; # hide this from the outside world
         proxy_pass http://localhost:8000;
         # Set timeouts longer than Cloudflare proxy limits
         proxy_connect_timeout 60;  # nginx default (Cf = 15)
diff --git a/k8s/nginx-logging.conf b/k8s/nginx-logging.conf
index 0938b0530e..3c4ade4614 100644
--- a/k8s/nginx-logging.conf
+++ b/k8s/nginx-logging.conf
@@ -9,7 +9,7 @@ log_format  ietfjson  escape=json
     '"method":"$${keepempty}request_method",'
     '"status":"$${keepempty}status",'
     '"len_bytes":"$${keepempty}body_bytes_sent",'
-    '"duration_ms":"$${keepempty}request_time",'
+    '"duration_s":"$${keepempty}request_time",'
     '"referer":"$${keepempty}http_referer",'
     '"user_agent":"$${keepempty}http_user_agent",'
     '"x_forwarded_for":"$${keepempty}http_x_forwarded_for",'

From 36847428d5991c75cbc96799f68ff327af7b89f8 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 4 Sep 2024 17:39:03 -0300
Subject: [PATCH 059/656] fix: label > 26 sessions per group (#7599)

* fix: label > 26 sessions correctly

* test: test new helper

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/meeting/models.py       | 21 ++++++++++++++++-----
 ietf/meeting/tests_models.py |  9 +++++++++
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index 3470679327..01b695bcea 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -1205,19 +1205,30 @@ def special_request_token(self):
         else:
             return ""
 
+    @staticmethod
+    def _alpha_str(n: int):
+        """Convert integer to string of a-z characters (a, b, c, ..., aa, ab, ...)"""
+        chars = []
+        while True:
+            chars.append(string.ascii_lowercase[n % 26])
+            n //= 26
+            # for 2nd letter and beyond, 0 means end the string
+            if n == 0:
+                break
+            # beyond the first letter, no need to represent a 0, so decrement
+            n -= 1
+        return "".join(chars[::-1])
+
     def docname_token(self):
         sess_mtg = Session.objects.filter(meeting=self.meeting, group=self.group).order_by('pk')
         index = list(sess_mtg).index(self)
-        return 'sess%s' % (string.ascii_lowercase[index])
+        return f"sess{self._alpha_str(index)}"
 
     def docname_token_only_for_multiple(self):
         sess_mtg = Session.objects.filter(meeting=self.meeting, group=self.group).order_by('pk')
         if len(list(sess_mtg)) > 1:
             index = list(sess_mtg).index(self)
-            if index < 26:
-                token = 'sess%s' % (string.ascii_lowercase[index])
-            else:
-                token = 'sess%s%s' % (string.ascii_lowercase[index//26],string.ascii_lowercase[index%26])
+            token = f"sess{self._alpha_str(index)}"
             return token
         return None
         
diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py
index ff874100dc..0a089ee9e8 100644
--- a/ietf/meeting/tests_models.py
+++ b/ietf/meeting/tests_models.py
@@ -10,6 +10,7 @@
 
 from ietf.group.factories import GroupFactory, GroupHistoryFactory
 from ietf.meeting.factories import MeetingFactory, SessionFactory, AttendedFactory, SessionPresentationFactory
+from ietf.meeting.models import Session
 from ietf.stats.factories import MeetingRegistrationFactory
 from ietf.utils.test_utils import TestCase
 from ietf.utils.timezone import date_today, datetime_today
@@ -147,6 +148,14 @@ def test_chat_room_name(self):
         session.chat_room = 'fnord'
         self.assertEqual(session.chat_room_name(), 'fnord')
 
+    def test_alpha_str(self):
+        self.assertEqual(Session._alpha_str(0), "a")
+        self.assertEqual(Session._alpha_str(1), "b")
+        self.assertEqual(Session._alpha_str(25), "z")
+        self.assertEqual(Session._alpha_str(26), "aa")
+        self.assertEqual(Session._alpha_str(27 * 26 - 1), "zz")
+        self.assertEqual(Session._alpha_str(27 * 26), "aaa")
+
     def test_session_recording_url(self):
         group_acronym = "foobar"
         meeting_date = datetime.date.today()

From 2a6fd3e1969f52c65089c024e1188c8a07bcb4a3 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 5 Sep 2024 12:39:35 -0300
Subject: [PATCH 060/656] ci: remove auth on port 8080 (#7903)

It's now on port 80
---
 k8s/auth.yaml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/k8s/auth.yaml b/k8s/auth.yaml
index c35cdc8ac2..c92ed05163 100644
--- a/k8s/auth.yaml
+++ b/k8s/auth.yaml
@@ -144,9 +144,5 @@ spec:
       targetPort: http
       protocol: TCP
       name: http
-    - port: 8080
-      targetPort: http
-      protocol: TCP
-      name: http-old
   selector:
     app: auth

From cb25831a2a33a97d0b5cf0ee2a0ba29af51c887b Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 5 Sep 2024 10:43:43 -0500
Subject: [PATCH 061/656] feat: total ids, pre-pubreq counts and pages left to
 ballot on on the AD dashboard (#7813)

* feat: Total ids on IESG dashboard

* IESG I-D code comments

* Using Robert's query forIESG dashboard total_ids

* Hiding columns in later IESG Dashboard tables

* Changing IESG dashboard var name to match column table

* Updating IESG pre_pubreqquery

* IESG dashboard prepub req safeParser and graphs

* IESG dashboard fixing Playwright API usage

* IESG dashboard fixing Playwright API usage (2)

* Updating .gitignore for /geckodriver.log

* IESG ad test title

* feat: pages left to ballot on [WIP]

* Adding geckodriver.log to gitignore

* [WIP] pages left to ballot on

* integrating pages left to ballot on WIP

* Tests for ad pages remaining

* Setting states to test ballot items

* refactor ad_pages_left_to_ballot_on count logic

* WIP tests for pages left to ballot on

* chore: remove whitespace change

* fix: look into the BallotPositionDocEventObject

* chore: remove prints

* fix: restructure test

* style: fix js code styling

* fix: only show graph for ADs/Secretariat

---------

Co-authored-by: Matthew Holloway <Matthew Holloway>
Co-authored-by: holloway <matthew@holloway.co.nz>
Co-authored-by: Nicolas Giard <github@ngpixel.com>
Co-authored-by: Matthew Holloway <matthew@staff.ietf.org>
---
 .gitignore                                |   1 +
 ietf/doc/views_search.py                  |  23 +++
 ietf/iesg/tests.py                        |  68 +++++-
 ietf/iesg/utils.py                        |  22 +-
 ietf/iesg/views.py                        |   8 +-
 ietf/templates/doc/ad_list.html           | 241 +++++++++++++++-------
 ietf/templates/iesg/agenda_documents.html |   7 +-
 playwright/tests-legacy/docs/ad.spec.js   |  26 +++
 8 files changed, 307 insertions(+), 89 deletions(-)
 create mode 100644 playwright/tests-legacy/docs/ad.spec.js

diff --git a/.gitignore b/.gitignore
index c25e6b5bfe..84bc800e3b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,7 @@ datatracker.sublime-workspace
 /docker/docker-compose.extend-custom.yml
 /env
 /ghostdriver.log
+/geckodriver.log
 /htmlcov
 /ietf/static/dist-neue
 /latest-coverage.json
diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py
index 2ef4ee83e6..528fb05a22 100644
--- a/ietf/doc/views_search.py
+++ b/ietf/doc/views_search.py
@@ -485,6 +485,29 @@ def _state_to_doc_type(state):
         )
         ad.buckets = copy.deepcopy(bucket_template)
 
+        # https://github.com/ietf-tools/datatracker/issues/4577
+        docs_via_group_ad = Document.objects.exclude(
+            group__acronym="none"
+        ).filter(
+            group__role__name="ad",
+            group__role__person=ad
+        ).filter(
+            states__type="draft-stream-ietf",
+            states__slug__in=["wg-doc","wg-lc","waiting-for-implementation","chair-w","writeupw"]
+        )
+
+        doc_for_ad = Document.objects.filter(ad=ad)
+
+        ad.pre_pubreq = (docs_via_group_ad | doc_for_ad).filter(
+            type="draft"
+        ).filter(
+            states__type="draft",
+            states__slug="active"
+        ).filter(
+            states__type="draft-iesg",
+            states__slug="idexists"
+        ).distinct().count()
+
         for doc in Document.objects.exclude(type_id="rfc").filter(ad=ad):
             dt = doc_type(doc)
             state = doc_state(doc)
diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py
index 4579316f22..8438cb44dd 100644
--- a/ietf/iesg/tests.py
+++ b/ietf/iesg/tests.py
@@ -18,7 +18,7 @@
 
 from ietf.doc.models import DocEvent, BallotPositionDocEvent, TelechatDocEvent
 from ietf.doc.models import Document, State, RelatedDocument
-from ietf.doc.factories import WgDraftFactory, IndividualDraftFactory, ConflictReviewFactory, BaseDocumentFactory, CharterFactory, WgRfcFactory, IndividualRfcFactory
+from ietf.doc.factories import BallotDocEventFactory, BallotPositionDocEventFactory, TelechatDocEventFactory, WgDraftFactory, IndividualDraftFactory, ConflictReviewFactory, BaseDocumentFactory, CharterFactory, WgRfcFactory, IndividualRfcFactory
 from ietf.doc.utils import create_ballot_if_not_open
 from ietf.group.factories import RoleFactory, GroupFactory, DatedGroupMilestoneFactory, DatelessGroupMilestoneFactory
 from ietf.group.models import Group, GroupMilestone, Role
@@ -30,7 +30,6 @@
 from ietf.iesg.factories import IESGMgmtItemFactory, TelechatAgendaContentFactory
 from ietf.utils.timezone import date_today, DEADLINE_TZINFO
 
-
 class IESGTests(TestCase):
     def test_feed(self):
         draft = WgDraftFactory(states=[('draft','active'),('draft-iesg','iesg-eva')],ad=Person.objects.get(user__username='ad'))
@@ -509,12 +508,13 @@ def test_agenda_documents_txt(self):
     def test_agenda_documents(self):
         url = urlreverse("ietf.iesg.views.agenda_documents")
         r = self.client.get(url)
+
         self.assertEqual(r.status_code, 200)
 
         for k, d in self.telechat_docs.items():
             self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name, ))
-            self.assertContains(r, d.title, msg_prefix="%s '%s' title not in response" % (k, d.title, ))
-
+            self.assertContains(r, d.title, msg_prefix="%s '%s' not in response" % (k, d.title, ))
+    
     def test_past_documents(self):
         url = urlreverse("ietf.iesg.views.past_documents")
         # We haven't put any documents on past telechats, so this should be empty
@@ -589,6 +589,66 @@ def test_admin_change(self):
         draft = Document.objects.get(name="draft-ietf-mars-test")
         self.assertEqual(draft.telechat_date(),today)
 
+class IESGAgendaTelechatPagesTests(TestCase):
+    def setUp(self):
+        super().setUp()
+        # make_immutable_test_data made a set of future telechats - only need one
+        # We'll take the "next" one
+        self.telechat_date = get_agenda_date()
+        # make_immutable_test_data made and area with only one ad - give it another
+        ad = Person.objects.get(user__username="ad")
+        adrole = Role.objects.get(person=ad, name="ad")
+        ad2 = RoleFactory(group=adrole.group, name_id="ad").person
+        self.ads=[ad,ad2]
+        
+        # Make some drafts
+        docs = [
+            WgDraftFactory(pages=2, states=[('draft-iesg','iesg-eva'),]),
+            IndividualDraftFactory(pages=20, states=[('draft-iesg','iesg-eva'),]),
+            WgDraftFactory(pages=200, states=[('draft-iesg','iesg-eva'),]),
+        ]
+        # Put them on the telechat
+        for doc in docs:
+            TelechatDocEventFactory(doc=doc, telechat_date=self.telechat_date)
+        # Give them ballots
+        ballots = [BallotDocEventFactory(doc=doc) for doc in docs]
+
+        # Give the "ad" Area-Director a discuss on one 
+        BallotPositionDocEventFactory(balloter=ad, doc=docs[0], pos_id="discuss", ballot=ballots[0])
+        # and a "norecord" position on another
+        BallotPositionDocEventFactory(balloter=ad, doc=docs[1], pos_id="norecord", ballot=ballots[1])
+        # Now "ad" should have 220 pages left to ballot on.
+        # Every other ad should have 222 pages left to ballot on.
+
+    def test_ad_pages_left_to_ballot_on(self):
+        url = urlreverse("ietf.iesg.views.agenda_documents")
+
+        # A non-AD user won't get "pages left"
+        response = self.client.get(url)
+        telechat = response.context["telechats"][0]
+        self.assertEqual(telechat["date"], self.telechat_date)
+        self.assertEqual(telechat["ad_pages_left_to_ballot_on"],0)
+        self.assertNotContains(response,"pages left to ballot on")
+
+        username=self.ads[0].user.username
+        self.assertTrue(self.client.login(username=username, password=f"{username}+password"))
+
+        response = self.client.get(url)
+        telechat = response.context["telechats"][0]
+        self.assertEqual(telechat["ad_pages_left_to_ballot_on"],220)
+        self.assertContains(response,"220 pages left to ballot on")
+
+        self.client.logout()
+        username=self.ads[1].user.username
+        self.assertTrue(self.client.login(username=username, password=f"{username}+password"))
+
+        response = self.client.get(url)
+        telechat = response.context["telechats"][0]
+        self.assertEqual(telechat["ad_pages_left_to_ballot_on"],222)
+
+
+
+
 class RescheduleOnAgendaTests(TestCase):
     def test_reschedule(self):
         draft = WgDraftFactory()
diff --git a/ietf/iesg/utils.py b/ietf/iesg/utils.py
index 3f4883798f..a56fa72cee 100644
--- a/ietf/iesg/utils.py
+++ b/ietf/iesg/utils.py
@@ -7,11 +7,11 @@
 from ietf.iesg.agenda import get_doc_section
 
 
-TelechatPageCount = namedtuple('TelechatPageCount',['for_approval','for_action','related'])
+TelechatPageCount = namedtuple('TelechatPageCount',['for_approval','for_action','related','ad_pages_left_to_ballot_on'])
 
-def telechat_page_count(date=None, docs=None):
+def telechat_page_count(date=None, docs=None, ad=None):
     if not date and not docs:
-        return TelechatPageCount(0, 0, 0)
+        return TelechatPageCount(0, 0, 0, 0)
 
     if not docs:
         candidates = Document.objects.filter(docevent__telechatdocevent__telechat_date=date).distinct() 
@@ -24,7 +24,18 @@ def telechat_page_count(date=None, docs=None):
 
     drafts = [d for d in for_approval if d.type_id == 'draft']
 
-    pages_for_approval = sum([d.pages or 0 for d in drafts])
+    ad_pages_left_to_ballot_on = 0
+    pages_for_approval = 0
+    
+    for draft in drafts:
+        pages_for_approval += draft.pages or 0
+        if ad:
+            ballot = draft.active_ballot()
+            if ballot:
+                positions = ballot.active_balloter_positions()
+                ad_position = positions[ad]
+                if ad_position is None or ad_position.pos_id == "norecord":
+                    ad_pages_left_to_ballot_on += draft.pages or 0
 
     pages_for_action = 0
     for d in for_action:
@@ -53,4 +64,5 @@ def telechat_page_count(date=None, docs=None):
     
     return TelechatPageCount(for_approval=pages_for_approval,
                              for_action=pages_for_action,
-                             related=related_pages)
+                             related=related_pages,
+                             ad_pages_left_to_ballot_on=ad_pages_left_to_ballot_on)
diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py
index a92d617ac5..df02754f2e 100644
--- a/ietf/iesg/views.py
+++ b/ietf/iesg/views.py
@@ -360,6 +360,8 @@ def handle_reschedule_form(request, doc, dates, status):
     return form
 
 def agenda_documents(request):
+    ad = request.user.person if has_role(request.user, "Area Director") else None
+
     dates = list(TelechatDate.objects.active().order_by('date').values_list("date", flat=True)[:4])
 
     docs_by_date = dict((d, []) for d in dates)
@@ -389,11 +391,13 @@ def agenda_documents(request):
         # the search_result_row view to display them (which expects them)
         fill_in_document_table_attributes(docs_by_date[date], have_telechat_date=True)
         fill_in_agenda_docs(date, sections, docs_by_date[date])
-        pages = telechat_page_count(docs=docs_by_date[date]).for_approval
-
+        page_count = telechat_page_count(docs=docs_by_date[date], ad=ad)
+        pages = page_count.for_approval
+        
         telechats.append({
                 "date":     date,
                 "pages":    pages,
+                "ad_pages_left_to_ballot_on": page_count.ad_pages_left_to_ballot_on,
                 "sections": sorted((num, section) for num, section in sections.items()
                                    if "2" <= num < "5")
                 })
diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html
index cfc8830e50..a73264c0f3 100644
--- a/ietf/templates/doc/ad_list.html
+++ b/ietf/templates/doc/ad_list.html
@@ -35,9 +35,12 @@ <h2 class="mt-5" id="{{ dt.type.0 }}">{{ dt.type.1 }} State Counts</h2>
             <thead class="wrap-anywhere">
                 <tr>
                     <th scope="col" data-sort="name">Area Director</th>
+                    {% if dt.type.1 == "Internet-Draft" %}
+                        <th scope="col" data-sort="pre-pubreq">Pre pubreq</th>
+                    {% endif %}                    
                     {% for state, state_name in dt.states %}
-                        <th scope="col" class="col-1" title=""
-                            data-sort="{{ state }}-num">
+                        <th scope="col" class="col-1" data-sort="{{ state }}-num"
+                        >
                             <a href="{% url 'ietf.doc.views_help.state_help' type='draft-iesg' %}#{{ state }}">
                                 {{ state_name|split:'/'|join:'/<wbr>' }}
                             </a>
@@ -51,6 +54,17 @@ <h2 class="mt-5" id="{{ dt.type.0 }}">{{ dt.type.1 }} State Counts</h2>
                         <td>
                             <a href="{{ ad.dashboard }}">{{ ad.name }}</a>
                         </td>
+                        {% if dt.type.1 == "Internet-Draft" %}
+                            <td
+                                class="col-1 align-bottom"
+                                data-sum="pre-pubreq"
+                                {% if user|has_role:"Area Director,Secretariat" %}
+                                    data-series-graph
+                                {% endif %}
+                            >
+                                {{ ad.pre_pubreq }}
+                            </td>
+                        {% endif %}            
                         {% for state, state_name in dt.states %}
                         <td class="col-1 align-bottom"
                             id="{{ dt.type.0 }}-{{ ad|slugify }}-{{ state }}">
@@ -63,6 +77,16 @@ <h2 class="mt-5" id="{{ dt.type.0 }}">{{ dt.type.1 }} State Counts</h2>
             <tfoot class="table-group-divider">
                 <tr>
                     <th scope="row">Sum</th>
+                    {% if dt.type.1 == "Internet-Draft" %}
+                        <td class="align-bottom">
+                            <div
+                                data-sum-result="pre-pubreq"
+                                {% if user|has_role:"Area Director,Secretariat" %}
+                                    data-series-graph
+                                {% endif %}
+                            ></div>
+                        </td>
+                    {% endif %}
                     {% for state, state_name in dt.states %}
                         <td class="align-bottom">
                             <div id="chart-{{ dt.type.0 }}-sum-{{ state }}"></div>
@@ -87,37 +111,151 @@ <h2 class="mt-5" id="{{ dt.type.0 }}">{{ dt.type.1 }} State Counts</h2>
     </script>
     <script src="{% static "ietf/js/highcharts.js" %}"></script>
     {{ data|json_script:"data" }}
+
     <script>
-        const data = JSON.parse(document.getElementById("data").textContent);
+        function highchartsConfigFactory({ element, ymax, series }){
+            return {
+                title: { text: undefined },
+                chart: {
+                    type: "line",
+                    animation: false,
+                    renderTo: element,
+                    panning: { enabled: false },
+                    spacing: [4, 0, 5, 0],
+                    height: "45%"
+                },
+                scrollbar: { enabled: false },
+                tooltip: { enabled: false },
+                navigator: { enabled: false },
+                exporting: { enabled: false },
+                legend: { enabled: false },
+                credits: { enabled: false },
+                xAxis: {
+                    title: { text: undefined},
+                    labels: { enabled: false },
+                    zoomEnabled: false,
+                    tickLength: 0
+                },
+                yAxis: {
+                    title: { text: undefined},
+                    zoomEnabled: false,
+                    tickLength: 0,
+                    labels: { x: -3 },
+                    min: 0,
+                    max: ymax,
+                    tickInterval: ymax
+                },
+                plotOptions: {
+                    series: {
+                        animation: false,
+                        dataLabels: {
+                            enabled: true,
+                            inside: true,
+                            padding: 0,
+                            formatter: function() {
+                                // only label the last
+                                // (= current-day) point in
+                                // the curve with today's
+                                // value
+                                if (this.point.index + 1 ==
+                                    this.series.points.length) {
+                                    return this.y
+                                }
+                                return undefined
+                            }
+                        }
+                    }
+                },
+                series: [{
+                    name: undefined,
+                    data: series,
+                    enableMouseTracking: false
+                }]
+            }
+        }
+    </script>
+
+    <script>
+        const GRAPH_BUFFER = 2;
+
+        function safeParseFloat(text) {
+            const trimNumber = text.trim()
+            if(!trimNumber.match(/^[0-9.]+$/)) {
+                console.warn(`Unable to parse "${trimNumber}" as a number.`)
+                return Number.NaN
+            }
+            return parseFloat(text)
+        }
+
+        Array.from(document.querySelectorAll("table"))
+            .filter(table => Boolean(table.querySelector("[data-sum]")))
+            .forEach(table => {
+                const sums = Array.from(table.querySelectorAll("[data-sum]")).reduce((
+                    sumsAccumulator,
+                    cell
+                ) => {                    
+                    const key = cell.dataset.sum
+                    const value = safeParseFloat(cell.textContent)
+                    if(key && !isNaN(value)) {
+                        sumsAccumulator[key] = (sumsAccumulator[key] || 0) + (value || 0)
+                    }
+                    return sumsAccumulator
+                }, {})
+
+                Array.from(table.querySelectorAll('[data-sum-result]')).forEach(result => {
+                    const key = result.dataset.sumResult
+                    const value = sums[key]
+
+                    if(value) {
+                        result.innerText = value
+                    }
+                })
+
+                Array.from(table.querySelectorAll('[data-series-graph]')).forEach(element => {
+                    const endValue = safeParseFloat(element.innerText)
+                    if(isNaN(endValue)) throw Error("Can't render Highcharts chart with non-numerical " + element.innerText)
+
+                    const ymax = Math.max(1, endValue + GRAPH_BUFFER)
+
+                    Highcharts.chart(
+                        highchartsConfigFactory({
+                            element,
+                            ymax,
+                            series: [endValue]
+                        })
+                    )
+                })
+            })
+    </script>
+    <script>
+        const data = JSON.parse(document.getElementById("data").textContent)
 
         Object.entries(data).forEach(([dt, ads]) => {
-            max = {};
+            max = {}
             Object.entries(ads).forEach(([ad, states]) => {
                 Object.entries(states).forEach(([state, buckets]) => {
-                    buckets.series = buckets.map((x) => x.length);
+                    buckets.series = buckets.map((x) => x.length)
                     if (ad != "sum") {
-                        max[state] = Math.max(...buckets.series,
-                                              max[state] ? max[state] : 0);
+                        max[state] = Math.max(...buckets.series, max[state] ? max[state] : 0)
                     }
-                });
-            });
+                })
+            })
 
             Object.entries(ads).forEach(([ad, states]) => {
                 Object.entries(states).forEach(([state, buckets]) => {
-                    const cell = `chart-${dt}-${ad}-${state}`;
+                    const cell = `chart-${dt}-${ad}-${state}`
 
                     // if there is only a single datapoint in the
                     // bucket, display it without a graph
                     if (buckets.series.length == 1) {
-                        document.getElementById(cell).innerHTML =
-                            buckets.series[0];
-                        return;
+                        document.getElementById(cell).innerHTML = buckets.series[0]
+                        return
                     }
 
                     // if a bucket has all zeroes, fake a Highcharts
                     // plot with HTML, to reduce the number of plot
                     // objects on the page
-                    if (buckets.series.every((x) => x == 0)) {
+                    if (buckets.series.every((x) => x === 0)) {
                         // document.getElementById(cell).innerHTML = `
                         //     <div class="position-relative">
                         //         <div class="position-absolute bottom-0 start-0">
@@ -128,71 +266,20 @@ <h2 class="mt-5" id="{{ dt.type.0 }}">{{ dt.type.1 }} State Counts</h2>
                         //         </div>
                         //     </div>
                         // `;
-                        return;
+                        return
                     }
 
                     // else actually create a graph
-                    const ymax = Math.max(1, ad != "sum" ? max[state] : Math.max(...buckets.series));
-                    Highcharts.chart({
-                        title: { text: undefined },
-                        chart: {
-                            type: "line",
-                            animation: false,
-                            renderTo: cell,
-                            panning: { enabled: false },
-                            spacing: [4, 0, 5, 0],
-                            height: "45%",
-                        },
-                        scrollbar: { enabled: false },
-                        tooltip: { enabled: false },
-                        navigator: { enabled: false },
-                        exporting: { enabled: false },
-                        legend: { enabled: false },
-                        credits: { enabled: false },
-                        xAxis: {
-                            title: { text: undefined},
-                            labels: { enabled: false },
-                            zoomEnabled: false,
-                            tickLength: 0,
-                        },
-                        yAxis: {
-                            title: { text: undefined},
-                            zoomEnabled: false,
-                            tickLength: 0,
-                            labels: { x: -3 },
-                            min: 0,
-                            max: ymax,
-                            tickInterval: ymax,
-                        },
-                        plotOptions: {
-                            series: {
-                                animation: false,
-                                dataLabels: {
-                                    enabled: true,
-                                    inside: true,
-                                    padding: 0,
-                                    formatter: function() {
-                                        // only label the last
-                                        // (= current-day) point in
-                                        // the curve with today's
-                                        // value
-                                        if (this.point.index + 1 ==
-                                            this.series.points.length) {
-                                            return this.y;
-                                        }
-                                        return undefined;
-                                    }
-                                }
-                            }
-                        },
-                        series: [{
-                            name: undefined,
-                            data: buckets.series,
-                            enableMouseTracking: false,
-                        }],
-                    });
-                });
-            });
-        });
+                    const ymax = Math.max(1, ad !== "sum" ? max[state] : Math.max(...buckets.series))
+                    Highcharts.chart(
+                        highchartsConfigFactory({
+                            element: cell,
+                            ymax,
+                            series: buckets.series
+                        })
+                    )
+                })
+            })
+        })
     </script>
 {% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/iesg/agenda_documents.html b/ietf/templates/iesg/agenda_documents.html
index 80dd9956fa..f732672df0 100644
--- a/ietf/templates/iesg/agenda_documents.html
+++ b/ietf/templates/iesg/agenda_documents.html
@@ -21,7 +21,12 @@ <h1>Documents on future IESG telechat agendas</h1>
             <h2>
                 IESG telechat {{ t.date }}
                 <br>
-                <small class="text-body-secondary">{{ t.pages }} page{{ t.pages|pluralize }}</small>
+                <small class="text-body-secondary">
+                    {{ t.pages }} page{{ t.pages|pluralize }}
+                    {% if t.ad_pages_left_to_ballot_on %}
+                        ({{ t.ad_pages_left_to_ballot_on }} pages left to ballot on)
+                    {% endif %}
+                </small>
             </h2>
             <div class="buttonlist">
                 <a class="btn btn-primary" role="button" href="{% url 'ietf.iesg.views.agenda' %}">
diff --git a/playwright/tests-legacy/docs/ad.spec.js b/playwright/tests-legacy/docs/ad.spec.js
new file mode 100644
index 0000000000..80b8b27cda
--- /dev/null
+++ b/playwright/tests-legacy/docs/ad.spec.js
@@ -0,0 +1,26 @@
+const { test, expect } = require('@playwright/test')
+const viewports = require('../../helpers/viewports')
+
+// ====================================================================
+// IESG Dashboard
+// ====================================================================
+
+test.describe('/doc/ad/', () => {
+  test.beforeEach(async ({ page }) => {
+    await page.setViewportSize({
+      width: viewports.desktop[0],
+      height: viewports.desktop[1]
+    })
+
+    await page.goto('/doc/ad/')
+  })
+
+  test('Pre pubreq', async ({ page }) => {
+    const tablesLocator = page.locator('table')
+    const tablesCount = await tablesLocator.count()
+    expect(tablesCount).toBeGreaterThan(0)
+    const firstTable = tablesLocator.nth(0)
+    const theadTexts = await firstTable.locator('thead').allInnerTexts()
+    expect(theadTexts.join('')).toContain('Pre pubreq')
+  })
+})

From d8d52eedbf6508d219fbabbfb0e9f36bc6ef0160 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 9 Sep 2024 12:00:54 -0300
Subject: [PATCH 062/656] feat: email ingestor api test endpoint (#7915)

* feat: email ingestor api test endpoint

* ci: add ingestion test token for sandbox

* chore: fix comments
---
 dev/deploy-to-container/settings_local.py |  6 ++
 ietf/api/tests.py                         | 97 ++++++++++++++++++++++-
 ietf/api/urls.py                          |  2 +
 ietf/api/views.py                         | 41 ++++++++--
 4 files changed, 138 insertions(+), 8 deletions(-)

diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py
index 25eacc3004..ae698e20b6 100644
--- a/dev/deploy-to-container/settings_local.py
+++ b/dev/deploy-to-container/settings_local.py
@@ -71,5 +71,11 @@
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
 
+# No real secrets here, these are public testing values _only_
+APP_API_TOKENS = {
+  "ietf.api.views.ingest_email_test": ["ingestion-test-token"]
+}
+
+
 # OIDC configuration
 SITE_URL = 'https://__HOSTNAME__'
diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 20c3e2cb44..4f2a7f7d3c 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -1022,7 +1022,9 @@ def test_role_holder_addresses(self):
             sorted(e.address for e in emails),
         )
 
-    @override_settings(APP_API_TOKENS={"ietf.api.views.ingest_email": "valid-token"})
+    @override_settings(
+        APP_API_TOKENS={"ietf.api.views.ingest_email": "valid-token", "ietf.api.views.ingest_email_test": "test-token"}
+    )
     @mock.patch("ietf.api.views.iana_ingest_review_email")
     @mock.patch("ietf.api.views.ipr_ingest_response_email")
     @mock.patch("ietf.api.views.nomcom_ingest_feedback_email")
@@ -1032,29 +1034,47 @@ def test_ingest_email(
         mocks = {mock_nomcom_ingest, mock_ipr_ingest, mock_iana_ingest}
         empty_outbox()        
         url = urlreverse("ietf.api.views.ingest_email")
+        test_mode_url = urlreverse("ietf.api.views.ingest_email_test")
 
         # test various bad calls
         r = self.client.get(url)
         self.assertEqual(r.status_code, 403)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.get(test_mode_url)
+        self.assertEqual(r.status_code, 403)
+        self.assertFalse(any(m.called for m in mocks))
 
         r = self.client.post(url)
         self.assertEqual(r.status_code, 403)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.post(test_mode_url)
+        self.assertEqual(r.status_code, 403)
+        self.assertFalse(any(m.called for m in mocks))
 
         r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
         self.assertEqual(r.status_code, 405)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.get(test_mode_url, headers={"X-Api-Key": "test-token"})
+        self.assertEqual(r.status_code, 405)
+        self.assertFalse(any(m.called for m in mocks))
 
         r = self.client.post(url, headers={"X-Api-Key": "valid-token"})
         self.assertEqual(r.status_code, 415)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.post(test_mode_url, headers={"X-Api-Key": "test-token"})
+        self.assertEqual(r.status_code, 415)
+        self.assertFalse(any(m.called for m in mocks))
 
         r = self.client.post(
             url, content_type="application/json", headers={"X-Api-Key": "valid-token"}
         )
         self.assertEqual(r.status_code, 400)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.post(
+            test_mode_url, content_type="application/json", headers={"X-Api-Key": "test-token"}
+        )
+        self.assertEqual(r.status_code, 400)
+        self.assertFalse(any(m.called for m in mocks))
 
         r = self.client.post(
             url,
@@ -1064,6 +1084,14 @@ def test_ingest_email(
         )
         self.assertEqual(r.status_code, 400)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.post(
+            test_mode_url,
+            "this is not JSON!",
+            content_type="application/json",
+            headers={"X-Api-Key": "test-token"},
+        )
+        self.assertEqual(r.status_code, 400)
+        self.assertFalse(any(m.called for m in mocks))
 
         r = self.client.post(
             url,
@@ -1073,6 +1101,14 @@ def test_ingest_email(
         )
         self.assertEqual(r.status_code, 400)
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.post(
+            test_mode_url,
+            {"json": "yes", "valid_schema": False},
+            content_type="application/json",
+            headers={"X-Api-Key": "test-token"},
+        )
+        self.assertEqual(r.status_code, 400)
+        self.assertFalse(any(m.called for m in mocks))
 
         # bad destination
         message_b64 = base64.b64encode(b"This is a message").decode()
@@ -1086,6 +1122,16 @@ def test_ingest_email(
         self.assertEqual(r.headers["Content-Type"], "application/json")
         self.assertEqual(json.loads(r.content), {"result": "bad_dest"})
         self.assertFalse(any(m.called for m in mocks))
+        r = self.client.post(
+            test_mode_url,
+            {"dest": "not-a-destination", "message": message_b64},
+            content_type="application/json",
+            headers={"X-Api-Key": "test-token"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "bad_dest"})
+        self.assertFalse(any(m.called for m in mocks))
 
         # test that valid requests call handlers appropriately
         r = self.client.post(
@@ -1102,6 +1148,19 @@ def test_ingest_email(
         self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
         mock_iana_ingest.reset_mock()
         
+        # the test mode endpoint should _not_ call the handler
+        r = self.client.post(
+            test_mode_url,
+            {"dest": "iana-review", "message": message_b64},
+            content_type="application/json",
+            headers={"X-Api-Key": "test-token"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "ok"})
+        self.assertFalse(any(m.called for m in mocks))
+        mock_iana_ingest.reset_mock()
+        
         r = self.client.post(
             url,
             {"dest": "ipr-response", "message": message_b64},
@@ -1116,6 +1175,19 @@ def test_ingest_email(
         self.assertFalse(any(m.called for m in (mocks - {mock_ipr_ingest})))
         mock_ipr_ingest.reset_mock()
 
+        # the test mode endpoint should _not_ call the handler
+        r = self.client.post(
+            test_mode_url,
+            {"dest": "ipr-response", "message": message_b64},
+            content_type="application/json",
+            headers={"X-Api-Key": "test-token"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "ok"})
+        self.assertFalse(any(m.called for m in mocks))
+        mock_ipr_ingest.reset_mock()
+
         # bad nomcom-feedback dest
         for bad_nomcom_dest in [
             "nomcom-feedback",  # no suffix
@@ -1133,6 +1205,16 @@ def test_ingest_email(
             self.assertEqual(r.headers["Content-Type"], "application/json")
             self.assertEqual(json.loads(r.content), {"result": "bad_dest"})
             self.assertFalse(any(m.called for m in mocks))
+            r = self.client.post(
+                test_mode_url,
+                {"dest": bad_nomcom_dest, "message": message_b64},
+                content_type="application/json",
+                headers={"X-Api-Key": "test-token"},
+            )
+            self.assertEqual(r.status_code, 200)
+            self.assertEqual(r.headers["Content-Type"], "application/json")
+            self.assertEqual(json.loads(r.content), {"result": "bad_dest"})
+            self.assertFalse(any(m.called for m in mocks))
 
         # good nomcom-feedback dest
         random_year = randrange(100000)
@@ -1150,6 +1232,19 @@ def test_ingest_email(
         self.assertFalse(any(m.called for m in (mocks - {mock_nomcom_ingest})))
         mock_nomcom_ingest.reset_mock()
 
+        # the test mode endpoint should _not_ call the handler
+        r = self.client.post(
+            test_mode_url,
+            {"dest": f"nomcom-feedback-{random_year}", "message": message_b64},
+            content_type="application/json",
+            headers={"X-Api-Key": "test-token"},
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        self.assertEqual(json.loads(r.content), {"result": "ok"})
+        self.assertFalse(any(m.called for m in mocks))
+        mock_nomcom_ingest.reset_mock()
+
         # test that exceptions lead to email being sent - assumes that iana-review handling is representative
         mock_iana_ingest.side_effect = EmailIngestionError("Error: don't send email")
         r = self.client.post(
diff --git a/ietf/api/urls.py b/ietf/api/urls.py
index 3c0fb872c9..396b3813d6 100644
--- a/ietf/api/urls.py
+++ b/ietf/api/urls.py
@@ -27,6 +27,8 @@
     url(r'^doc/draft-aliases/$', api_views.draft_aliases),
     # email ingestor
     url(r'email/$', api_views.ingest_email),
+    # email ingestor
+    url(r'email/test/$', api_views.ingest_email_test),
     # GDPR: export of personal information for the logged-in person
     url(r'^export/personal-information/$', api_views.PersonalInformationExportView.as_view()),
     # Email alias information for groups
diff --git a/ietf/api/views.py b/ietf/api/views.py
index 62857bff54..f8662f9a0e 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -614,14 +614,16 @@ def as_emailmessage(self) -> Optional[EmailMessage]:
         return msg
 
 
-@requires_api_token
-@csrf_exempt
-def ingest_email(request):
-    """Ingest incoming email
+def ingest_email_handler(request, test_mode=False):
+    """Ingest incoming email - handler
     
     Returns a 4xx or 5xx status code if the HTTP request was invalid or something went
     wrong while processing it. If the request was valid, returns a 200. This may or may
     not indicate that the message was accepted.
+    
+    If test_mode is true, actual processing of a valid message will be skipped. In this
+    mode, a valid request with a valid destination will be treated as accepted. The
+    "bad_dest" error may still be returned.
     """
 
     def _http_err(code, text):
@@ -657,15 +659,18 @@ def _api_response(result):
     try:
         if dest == "iana-review":
             valid_dest = True
-            iana_ingest_review_email(message)
+            if not test_mode:
+                iana_ingest_review_email(message)
         elif dest == "ipr-response":
             valid_dest = True
-            ipr_ingest_response_email(message)
+            if not test_mode:
+                ipr_ingest_response_email(message)
         elif dest.startswith("nomcom-feedback-"):
             maybe_year = dest[len("nomcom-feedback-"):]
             if maybe_year.isdecimal():
                 valid_dest = True
-                nomcom_ingest_feedback_email(message, int(maybe_year))
+                if not test_mode:
+                    nomcom_ingest_feedback_email(message, int(maybe_year))
     except EmailIngestionError as err:
         error_email = err.as_emailmessage()
         if error_email is not None:
@@ -677,3 +682,25 @@ def _api_response(result):
         return _api_response("bad_dest")
 
     return _api_response("ok")
+
+
+@requires_api_token
+@csrf_exempt
+def ingest_email(request):
+    """Ingest incoming email
+
+    Hands off to ingest_email_handler() with test_mode=False. This allows @requires_api_token to
+    give the test endpoint a distinct token from the real one.
+    """
+    return ingest_email_handler(request, test_mode=False)
+
+
+@requires_api_token
+@csrf_exempt
+def ingest_email_test(request):
+    """Ingest incoming email test endpoint
+    
+    Hands off to ingest_email_handler() with test_mode=True. This allows @requires_api_token to
+    give the test endpoint a distinct token from the real one.
+    """
+    return ingest_email_handler(request, test_mode=True)

From 80599f29f0dacb9069f2963aa0714f91889d10db Mon Sep 17 00:00:00 2001
From: Sangho Na <sangho@nsh.nz>
Date: Wed, 11 Sep 2024 10:22:45 +1200
Subject: [PATCH 063/656] fix: Include missing related drafts in IPR searches
 (#7836)

* fix: Include missing related drafts in IPR searches

* refactor: extract drafts, sort docs

* chore: indent loop and conditionals to improve readability

* test: handle whitespaces added to IPR search result page

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/ipr/tests.py                         | 33 ++++++++++++++++++++
 ietf/ipr/views.py                         | 38 ++++++++++++++++++++---
 ietf/templates/ipr/search_doc_result.html | 19 +++++++++---
 3 files changed, 82 insertions(+), 8 deletions(-)

diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py
index 922ae272a9..a65b0d6c6c 100644
--- a/ietf/ipr/tests.py
+++ b/ietf/ipr/tests.py
@@ -24,6 +24,7 @@
     RfcFactory,
     NewRevisionDocEventFactory
 )
+from ietf.doc.utils import prettify_std_name
 from ietf.group.factories import RoleFactory
 from ietf.ipr.factories import (
     HolderIprDisclosureFactory,
@@ -192,6 +193,38 @@ def test_search(self):
         r = self.client.get(url + "?submit=rfc&rfc=321")
         self.assertContains(r, ipr.title)
 
+        rfc_new = RfcFactory(rfc_number=322)
+        rfc_new.relateddocument_set.create(relationship_id="obs", target=rfc)
+
+        # find RFC 322 which obsoletes RFC 321 whose draft has IPR
+        r = self.client.get(url + "?submit=rfc&rfc=322")
+        self.assertContains(r, ipr.title)
+        self.assertContains(r, "Total number of IPR disclosures found: <b>1</b>")
+        self.assertContains(r, "Total number of documents searched: <b>3</b>.")
+        self.assertContains(
+            r,
+            f"""Results for <a href="/doc/{rfc_new.name}/">{prettify_std_name(rfc_new.name)}</a>
+                        ("{rfc_new.title}")""",
+        )
+        self.assertContains(
+            r,
+            f"""Results for <a href="/doc/{rfc.name}/">{prettify_std_name(rfc.name)}</a>
+                        ("{rfc.title}"), which
+                            
+                                was obsoleted by
+                                <a href="/doc/{rfc_new.name}/">{prettify_std_name(rfc_new.name)}</a>
+                                ("{rfc_new.title}")""",
+        )
+        self.assertContains(
+            r,
+            f"""Results for <a href="/doc/{draft.name}/">{prettify_std_name(draft.name)}</a>
+                        ("{draft.title}"), which
+                            
+                                became rfc
+                                <a href="/doc/{rfc.name}/">{prettify_std_name(rfc.name)}</a>
+                                ("{rfc.title}")""",
+        )
+
         # find by patent owner
         r = self.client.get(url + "?submit=holder&holder=%s" % ipr.holder_legal_name)
         self.assertContains(r, ipr.title)
diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py
index a061232b8f..45fad9a2cc 100644
--- a/ietf/ipr/views.py
+++ b/ietf/ipr/views.py
@@ -689,11 +689,41 @@ def search(request):
                 if len(start) == 1:
                     first = start[0]
                     doc = first
-                    docs = related_docs(first)
-                    iprs = iprs_from_docs(docs,states=states)
+                    docs = set([first])
+                    docs.update(
+                        related_docs(
+                            first, relationship=("replaces", "obs"), reverse_relationship=()
+                        )
+                    )
+                    docs.update(
+                        set(
+                            [
+                                draft
+                                for drafts in [
+                                    related_docs(
+                                        d, relationship=(), reverse_relationship=("became_rfc",)
+                                    )
+                                    for d in docs
+                                ]
+                                for draft in drafts
+                            ]
+                        )
+                    )
+                    docs.discard(None)
+                    docs = sorted(
+                        docs,
+                        key=lambda d: (
+                            d.rfc_number if d.rfc_number is not None else 0,
+                            d.became_rfc().rfc_number if d.became_rfc() else 0,
+                        ),
+                        reverse=True,
+                    )
+                    iprs = iprs_from_docs(docs, states=states)
                     template = "ipr/search_doc_result.html"
-                    updated_docs = related_docs(first, ('updates',))
-                    related_iprs = list(set(iprs_from_docs(updated_docs, states=states)) - set(iprs))
+                    updated_docs = related_docs(first, ("updates",))
+                    related_iprs = list(
+                        set(iprs_from_docs(updated_docs, states=states)) - set(iprs)
+                    )
                 # multiple matches, select just one
                 elif start:
                     docs = start
diff --git a/ietf/templates/ipr/search_doc_result.html b/ietf/templates/ipr/search_doc_result.html
index dc7d8b95b5..b003a32108 100644
--- a/ietf/templates/ipr/search_doc_result.html
+++ b/ietf/templates/ipr/search_doc_result.html
@@ -54,16 +54,27 @@
             </tr>
         </thead>
 
-        {% for doc in docs %}
+        {% for d in docs %}
             <tbody>
                 <tr>
                     <th scope="col" class="table-info" colspan="3">
-                        Results for {{ doc.name|prettystdname|urlize_ietf_docs }} ("{{ doc.title }}"){% if not forloop.first %}{% if doc.related %}, which was {{ doc.relation|lower }} {{ doc.related.source|prettystdname|urlize_ietf_docs  }} ("{{ doc.related.source.title }}"){% endif %}{% endif %}
+                        Results for {{ d.name|prettystdname|urlize_ietf_docs }}
+                        ("{{ d.title }}"){% if d != doc and d.related %}, which
+                            {% if d == d.related.source %}
+                                {{ d.relation|lower }}
+                                {{ d.related.target|prettystdname|urlize_ietf_docs }}
+                                ("{{ d.related.target.title }}")
+                            {% else %}
+                                was {{ d.relation|lower }}
+                                {{ d.related.source|prettystdname|urlize_ietf_docs }}
+                                ("{{ d.related.source.title }}")
+                            {% endif %}
+                        {% endif %}
                     </th>
                 </tr>
             </tbody>
             <tbody>
-                {% with doc.iprdocrel_set.all as doc_iprs %}
+                {% with d.iprdocrel_set.all as doc_iprs %}
                     {% if doc_iprs %}
                         {% for ipr in doc_iprs %}
                             {% if ipr.disclosure.state_id in states %}
@@ -81,7 +92,7 @@
                             <td></td>
                             <td></td>
                             <td>
-                                No IPR disclosures have been submitted directly on {{ doc.name|prettystdname|urlize_ietf_docs }}{% if iprs %},
+                                No IPR disclosures have been submitted directly on {{ d.name|prettystdname|urlize_ietf_docs }}{% if iprs %},
                                     but there are disclosures on {% if docs|length == 2 %}a related document{% else %}related documents{% endif %}, listed on this page{% endif %}.
                             </td>
                         </tr>

From 13aa072a1e6f50f40a3b33a823408f8eae3b6914 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 12 Sep 2024 13:46:03 -0300
Subject: [PATCH 064/656] chore(deps): pin importlib-metadata (#7927)

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 88b78e665b..c51ed9ac00 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -33,6 +33,7 @@ gunicorn>=20.1.0
 hashids>=1.3.1
 html2text>=2020.1.16    # Used only to clean comment field of secr/sreq
 html5lib>=1.1    # Only used in tests
+importlib-metadata<8.5.0  # indirect req of Markdown/inflect; https://github.com/ietf-tools/datatracker/issues/7924
 inflect>= 6.0.2
 jsonfield>=3.1.0    # for SubmissionCheck.  This is https://github.com/bradjasper/django-jsonfield/.
 jsonschema[format]>=4.2.1

From 8d608a1282bf84a8b5df0a9d12f16b25cf265f4b Mon Sep 17 00:00:00 2001
From: Sangho Na <sangho@nsh.nz>
Date: Fri, 13 Sep 2024 05:47:15 +1200
Subject: [PATCH 065/656] test: check HTML content with whitespace ignored
 (#7921)

---
 ietf/ipr/tests.py | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py
index a65b0d6c6c..b08e359462 100644
--- a/ietf/ipr/tests.py
+++ b/ietf/ipr/tests.py
@@ -203,26 +203,20 @@ def test_search(self):
         self.assertContains(r, "Total number of documents searched: <b>3</b>.")
         self.assertContains(
             r,
-            f"""Results for <a href="/doc/{rfc_new.name}/">{prettify_std_name(rfc_new.name)}</a>
-                        ("{rfc_new.title}")""",
+            f'Results for <a href="/doc/{rfc_new.name}/">{prettify_std_name(rfc_new.name)}</a> ("{rfc_new.title}")',
+            html=True,
         )
         self.assertContains(
             r,
-            f"""Results for <a href="/doc/{rfc.name}/">{prettify_std_name(rfc.name)}</a>
-                        ("{rfc.title}"), which
-                            
-                                was obsoleted by
-                                <a href="/doc/{rfc_new.name}/">{prettify_std_name(rfc_new.name)}</a>
-                                ("{rfc_new.title}")""",
+            f'Results for <a href="/doc/{rfc.name}/">{prettify_std_name(rfc.name)}</a> ("{rfc.title}"), '
+            f'which was obsoleted by <a href="/doc/{rfc_new.name}/">{prettify_std_name(rfc_new.name)}</a> ("{rfc_new.title}")',
+            html=True,
         )
         self.assertContains(
             r,
-            f"""Results for <a href="/doc/{draft.name}/">{prettify_std_name(draft.name)}</a>
-                        ("{draft.title}"), which
-                            
-                                became rfc
-                                <a href="/doc/{rfc.name}/">{prettify_std_name(rfc.name)}</a>
-                                ("{rfc.title}")""",
+            f'Results for <a href="/doc/{draft.name}/">{prettify_std_name(draft.name)}</a> ("{draft.title}"), '
+            f'which became rfc <a href="/doc/{rfc.name}/">{prettify_std_name(rfc.name)}</a> ("{rfc.title}")',
+            html=True,
         )
 
         # find by patent owner

From f0f2b6d8b6a7a44f85047c42fefe89c8ca586f09 Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Fri, 13 Sep 2024 05:47:53 +1200
Subject: [PATCH 066/656] test: Use timezone aware datetime (#7918)

* test: Use timezone aware datetime

* fixup! test: Use timezone aware datetime
---
 ietf/meeting/tests_models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py
index 0a089ee9e8..f2949a7bb5 100644
--- a/ietf/meeting/tests_models.py
+++ b/ietf/meeting/tests_models.py
@@ -158,7 +158,7 @@ def test_alpha_str(self):
 
     def test_session_recording_url(self):
         group_acronym = "foobar"
-        meeting_date = datetime.date.today()
+        meeting_date = date_today()
         meeting_number = 123
 
         # IETF meeting

From b8c6cb34dd335de88740563eab6ebfbd362b49f4 Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Fri, 13 Sep 2024 09:28:44 +1200
Subject: [PATCH 067/656] chore: Remove obsolete `version` attribute (#7931)

---
 docker-compose.yml               | 2 --
 docker/docker-compose.celery.yml | 4 ----
 docker/docker-compose.extend.yml | 2 --
 3 files changed, 8 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 413c04ff63..65b28f54fe 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
     app:
         build:
diff --git a/docker/docker-compose.celery.yml b/docker/docker-compose.celery.yml
index dedae2d004..b6cc3d09e8 100644
--- a/docker/docker-compose.celery.yml
+++ b/docker/docker-compose.celery.yml
@@ -1,7 +1,3 @@
-version: '2.4'
-# Use version 2.4 for mem_limit setting. Version 3+ uses deploy.resources.limits.memory
-# instead, but that only works for swarm with docker-compose 1.25.1.
-
 services:
   mq:
     image: rabbitmq:3-alpine
diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml
index d055c976f4..0538c0d3e9 100644
--- a/docker/docker-compose.extend.yml
+++ b/docker/docker-compose.extend.yml
@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
     app:
         ports:

From 65547a7a9dfa51d60ab31b561590ff4b1aee7033 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 12 Sep 2024 17:04:12 -0500
Subject: [PATCH 068/656] fix: rectify mixed types in gathering mailtrigger
 recipients (#7932)

---
 ietf/mailtrigger/models.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ietf/mailtrigger/models.py b/ietf/mailtrigger/models.py
index aea530083a..66b7139fa5 100644
--- a/ietf/mailtrigger/models.py
+++ b/ietf/mailtrigger/models.py
@@ -138,16 +138,16 @@ def gather_group_steering_group(self,**kwargs):
     def gather_stream_managers(self, **kwargs):
         addrs = []
         manager_map = dict(
-            ise  = '<rfc-ise@rfc-editor.org>',
-            irtf = '<irtf-chair@irtf.org>',
-            ietf = '<iesg@ietf.org>',
-            iab  = '<iab-chair@iab.org>',
+            ise  = ['<rfc-ise@rfc-editor.org>'],
+            irtf = ['<irtf-chair@irtf.org>'],
+            ietf = ['<iesg@ietf.org>'],
+            iab  = ['<iab-chair@iab.org>'],
             editorial = Role.objects.filter(group__acronym="rsab",name_id="chair").values_list("email__address", flat=True),
         )
         if 'streams' in kwargs:
             for stream in kwargs['streams']:
                 if stream in manager_map:
-                    addrs.append(manager_map[stream])
+                    addrs.extend(manager_map[stream])
         return addrs
 
     def gather_doc_stream_manager(self, **kwargs):
@@ -234,7 +234,7 @@ def gather_submission_submitter(self, **kwargs):
                 try:
                     submitter = Alias.objects.get(name=submission.submitter).person
                     if submitter and submitter.email():
-                        addrs.extend(["%s <%s>" % (submitter.name, submitter.email().address)])
+                        addrs.append(f"{submitter.name} <{submitter.email().address}>")
                 except (Alias.DoesNotExist, Alias.MultipleObjectsReturned):
                     pass
         return addrs

From f5c132a20a695915e288195b32191b318bd24b9b Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 13 Sep 2024 18:42:51 -0300
Subject: [PATCH 069/656] refactor: helper for session recording URL; fix test
 (#7933)

* refactor: session recording URL label helper

* test: update tests, avoid tz dependence

* test: use date_today()
---
 ietf/meeting/models.py       | 14 ++++----
 ietf/meeting/tests_models.py | 67 +++++++++++++++++-------------------
 2 files changed, 39 insertions(+), 42 deletions(-)

diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index 01b695bcea..e169077800 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -1331,16 +1331,18 @@ def onsite_tool_url(self):
             return url.format(session=self)
         return None
 
+    def _session_recording_url_label(self):
+        if self.meeting.type.slug == "ietf" and self.has_onsite_tool:
+            session_label = f"IETF{self.meeting.number}-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}"
+        else:
+            session_label = f"IETF-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}"
+        return session_label
+
     def session_recording_url(self):
         url_formatter = getattr(settings, "MEETECHO_SESSION_RECORDING_URL", "")
         url = None
         if url_formatter and self.video_stream_url:
-            if self.meeting.type.slug == "ietf" and self.has_onsite_tool:
-                session_label = f"IETF{self.meeting.number}-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}"
-            else:
-                session_label = f"IETF-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}"
-            url = url_formatter.format(session_label=session_label)
-
+            url = url_formatter.format(session_label=self._session_recording_url_label())
         return url
 
 
diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py
index f2949a7bb5..8457423c51 100644
--- a/ietf/meeting/tests_models.py
+++ b/ietf/meeting/tests_models.py
@@ -8,6 +8,7 @@
 from django.conf import settings
 from django.test import override_settings
 
+import ietf.meeting.models
 from ietf.group.factories import GroupFactory, GroupHistoryFactory
 from ietf.meeting.factories import MeetingFactory, SessionFactory, AttendedFactory, SessionPresentationFactory
 from ietf.meeting.models import Session
@@ -156,46 +157,40 @@ def test_alpha_str(self):
         self.assertEqual(Session._alpha_str(27 * 26 - 1), "zz")
         self.assertEqual(Session._alpha_str(27 * 26), "aaa")
 
-    def test_session_recording_url(self):
-        group_acronym = "foobar"
-        meeting_date = date_today()
-        meeting_number = 123
-
-        # IETF meeting
+    @patch.object(ietf.meeting.models.Session, "_session_recording_url_label", return_value="LABEL")
+    def test_session_recording_url(self, mock):
+        for session_type in ["ietf", "interim"]:
+            session = SessionFactory(meeting__type_id=session_type)
+            with override_settings():
+                if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"):
+                    del settings.MEETECHO_SESSION_RECORDING_URL
+                self.assertIsNone(session.session_recording_url())
+    
+                settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com"
+                self.assertEqual(session.session_recording_url(), "http://player.example.com")
+    
+                settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}"
+                self.assertEqual(session.session_recording_url(), "http://player.example.com?LABEL")
+
+    def test_session_recording_url_label_ietf(self):
         session = SessionFactory(
             meeting__type_id='ietf',
-            meeting__date=meeting_date,
-            group__acronym=group_acronym,
-            meeting__number=meeting_number,
+            meeting__date=date_today(),
+            meeting__number="123",
+            group__acronym="acro",
         )
-        with override_settings():
-            if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"):
-                del settings.MEETECHO_SESSION_RECORDING_URL
-            self.assertIsNone(session.session_recording_url())
-
-            settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com"
-            self.assertEqual(session.session_recording_url(), "http://player.example.com")
-
-            settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}"
-            self.assertIn(f"IETF{meeting_number}-{group_acronym.upper()}", session.session_recording_url())
-            self.assertIn(f"{meeting_date.strftime('%Y%m%d')}", session.session_recording_url())
-            self.assertTrue(session.session_recording_url().startswith("http://player.example.com"))
+        session_time = session.official_timeslotassignment().timeslot.time
+        self.assertEqual(
+            f"IETF123-ACRO-{session_time:%Y%m%d-%H%M}",  # n.b., time in label is UTC
+            session._session_recording_url_label())
 
-        # interim meeting
+    def test_session_recording_url_label_interim(self):
         session = SessionFactory(
             meeting__type_id='interim',
-            meeting__date=meeting_date,
-            group__acronym=group_acronym,
+            meeting__date=date_today(),
+            group__acronym="acro",
         )
-        with override_settings():
-            if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"):
-                del settings.MEETECHO_SESSION_RECORDING_URL
-            self.assertIsNone(session.session_recording_url())
-
-            settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com"
-            self.assertEqual(session.session_recording_url(), "http://player.example.com")
-
-            settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}"
-            self.assertIn(f"IETF-{group_acronym.upper()}", session.session_recording_url())
-            self.assertIn(f"{meeting_date.strftime('%Y%m%d')}", session.session_recording_url())
-            self.assertTrue(session.session_recording_url().startswith("http://player.example.com"))
+        session_time = session.official_timeslotassignment().timeslot.time
+        self.assertEqual(
+            f"IETF-ACRO-{session_time:%Y%m%d-%H%M}",  # n.b., time in label is UTC
+            session._session_recording_url_label())

From 3b5058a51693c3bb4bf526a089f08c906f584499 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 16 Sep 2024 08:56:51 -0500
Subject: [PATCH 070/656] fix: start to reconcile internal inconsistencies wrt
 multiple from values (#7935)

---
 ietf/utils/mail.py  | 14 +++++++++++++-
 ietf/utils/tests.py | 12 ++++++------
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/ietf/utils/mail.py b/ietf/utils/mail.py
index e747c74778..4585fdb846 100644
--- a/ietf/utils/mail.py
+++ b/ietf/utils/mail.py
@@ -92,7 +92,17 @@ def send_smtp(msg, bcc=None):
     '''
     mark = time.time()
     add_headers(msg)
-    (fname, frm) = parseaddr(msg.get('From'))
+    # N.B. We have a disconnect with most of this code assuming a From header value will only
+    # have one address.
+    # The frm computed here is only used as the envelope from. 
+    # Previous code simply ran `parseaddr(msg.get('From'))`, getting lucky if the string returned
+    # from the get had more than one address in it. Python 3.9.20 changes the behavior of parseaddr
+    # and that erroneous use of the function no longer gets lucky.
+    # For the short term, to match behavior to date as closely as possible, if we get a message
+    # that has multiple addresses in the From header, we will use the first for the envelope from
+    from_tuples = getaddresses(msg.get_all('From', [settings.DEFAULT_FROM_EMAIL]))
+    assertion('len(from_tuples)==1', note=f"send_smtp received multiple From addresses: {from_tuples}")
+    _ , frm = from_tuples[0]
     addrlist = msg.get_all('To') + msg.get_all('Cc', [])
     if bcc:
         addrlist += [bcc]
@@ -446,6 +456,8 @@ def parse_preformatted(preformatted, extra=None, override=None):
         values = msg.get_all(key, [])
         if values:
             values = getaddresses(values)
+            if key=='From':
+                assertion('len(values)<2', note=f'parse_preformatted is constructing a From with multiple values: {values}')
             del msg[key]
             msg[key] = ',\n    '.join(formataddr(v) for v in values)
     for key in ['Subject', ]:
diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py
index d435583e89..decdd778d9 100644
--- a/ietf/utils/tests.py
+++ b/ietf/utils/tests.py
@@ -53,7 +53,7 @@ class SendingMail(TestCase):
 
     def test_send_mail_preformatted(self):
         msg = """To: to1@example.com, to2@example.com
-From: from1@ietf.org, from2@ietf.org
+From: from1@ietf.org
 Cc: cc1@example.com, cc2@example.com
 Bcc: bcc1@example.com, bcc2@example.com
 Subject: subject
@@ -63,7 +63,7 @@ def test_send_mail_preformatted(self):
         send_mail_preformatted(None, msg, {}, {})
         recv = outbox[-1]
         self.assertSameEmail(recv['To'], '<to1@example.com>, <to2@example.com>')
-        self.assertSameEmail(recv['From'], 'from1@ietf.org, from2@ietf.org')
+        self.assertSameEmail(recv['From'], 'from1@ietf.org')
         self.assertSameEmail(recv['Cc'], 'cc1@example.com, cc2@example.com')
         self.assertSameEmail(recv['Bcc'], None)
         self.assertEqual(recv['Subject'], 'subject')
@@ -71,14 +71,14 @@ def test_send_mail_preformatted(self):
 
         override = {
             'To': 'oto1@example.net, oto2@example.net',
-            'From': 'ofrom1@ietf.org, ofrom2@ietf.org',
+            'From': 'ofrom1@ietf.org',
             'Cc': 'occ1@example.net, occ2@example.net',
             'Subject': 'osubject',
         }
         send_mail_preformatted(request=None, preformatted=msg, extra={}, override=override)
         recv = outbox[-1]
         self.assertSameEmail(recv['To'], '<oto1@example.net>, <oto2@example.net>')
-        self.assertSameEmail(recv['From'], 'ofrom1@ietf.org, ofrom2@ietf.org')
+        self.assertSameEmail(recv['From'], 'ofrom1@ietf.org')
         self.assertSameEmail(recv['Cc'], 'occ1@example.net, occ2@example.net')
         self.assertSameEmail(recv['Bcc'], None)
         self.assertEqual(recv['Subject'], 'osubject')
@@ -86,14 +86,14 @@ def test_send_mail_preformatted(self):
 
         override = {
             'To': ['<oto1@example.net>', 'oto2@example.net'],
-            'From': ['<ofrom1@ietf.org>', 'ofrom2@ietf.org'],
+            'From': ['<ofrom1@ietf.org>'],
             'Cc': ['<occ1@example.net>', 'occ2@example.net'],
             'Subject': 'osubject',
         }
         send_mail_preformatted(request=None, preformatted=msg, extra={}, override=override)
         recv = outbox[-1]
         self.assertSameEmail(recv['To'], '<oto1@example.net>, <oto2@example.net>')
-        self.assertSameEmail(recv['From'], '<ofrom1@ietf.org>, ofrom2@ietf.org')
+        self.assertSameEmail(recv['From'], '<ofrom1@ietf.org>')
         self.assertSameEmail(recv['Cc'], '<occ1@example.net>, occ2@example.net')
         self.assertSameEmail(recv['Bcc'], None)
         self.assertEqual(recv['Subject'], 'osubject')

From cc1eade4f8b2894adba41d658521b4b0d585e2a3 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 16 Sep 2024 08:57:05 -0500
Subject: [PATCH 071/656] fix: correct headers for charter evaluation email
 (#7937)

---
 ietf/templates/doc/charter/issue_ballot_mail.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ietf/templates/doc/charter/issue_ballot_mail.txt b/ietf/templates/doc/charter/issue_ballot_mail.txt
index 12fc44bbbc..914935bb12 100644
--- a/ietf/templates/doc/charter/issue_ballot_mail.txt
+++ b/ietf/templates/doc/charter/issue_ballot_mail.txt
@@ -1,6 +1,6 @@
-{% load ietf_filters %}{% autoescape off %}To: {{ to }} {% if cc %}
-Cc: {{ cc }}
-{% endif %}From: IESG Secretary <iesg-secretary@ietf.org>
+{% load ietf_filters %}{% autoescape off %}To: {{ to }}{% if cc %}
+Cc: {{ cc }}{% endif %}
+From: IESG Secretary <iesg-secretary@ietf.org>
 Reply-To: IESG Secretary <iesg-secretary@ietf.org>
 Subject: Evaluation: {{ doc.name }}
 

From 9d583ab9eb056de5407ec88ba631a07b2d609879 Mon Sep 17 00:00:00 2001
From: Sangho Na <sangho@nsh.nz>
Date: Tue, 17 Sep 2024 03:06:16 +1200
Subject: [PATCH 072/656] fix: Use email or name when building community list
 view (#7203)

* fix: Use email or name when building community list view

instead of email

* test: add test case

* chore: remove debug

* fix: use name in community list menu when no active email is found

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 ietf/community/tests.py                 | 13 ++++++++++---
 ietf/community/views.py                 |  1 +
 ietf/templates/community/list_menu.html |  8 ++++----
 ietf/templates/community/view_list.html |  2 +-
 4 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/ietf/community/tests.py b/ietf/community/tests.py
index d76347b70a..84e4370771 100644
--- a/ietf/community/tests.py
+++ b/ietf/community/tests.py
@@ -108,10 +108,8 @@ def email_or_name_set(self, person):
         return [e for e in Email.objects.filter(person=person)] + \
                [a for a in Alias.objects.filter(person=person)]
 
-    def test_view_list(self):
-        person = self.complex_person(user__username='plain')
+    def do_view_list_test(self, person):
         draft = WgDraftFactory()
-
         # without list
         for id in self.email_or_name_set(person):
             url = urlreverse(ietf.community.views.view_list, kwargs={ "email_or_name": id })
@@ -134,6 +132,15 @@ def test_view_list(self):
             self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'")
             self.assertContains(r, draft.name)
 
+    def test_view_list(self):
+        person = self.complex_person(user__username='plain')
+        self.do_view_list_test(person)
+        
+    def test_view_list_without_active_email(self):
+        person = self.complex_person(user__username='plain')
+        person.email_set.update(active=False)
+        self.do_view_list_test(person)
+
     def test_manage_personal_list(self):
         person = self.complex_person(user__username='plain')
         ad = Person.objects.get(user__username='ad')
diff --git a/ietf/community/views.py b/ietf/community/views.py
index 4a28a391f0..78b8144d60 100644
--- a/ietf/community/views.py
+++ b/ietf/community/views.py
@@ -68,6 +68,7 @@ def view_list(request, email_or_name=None):
         'meta': meta,
         'can_manage_list': can_manage_community_list(request.user, clist),
         'subscribed': subscribed,
+        "email_or_name": email_or_name,
     })
 
 @login_required
diff --git a/ietf/templates/community/list_menu.html b/ietf/templates/community/list_menu.html
index 0552c76a45..009d01152d 100644
--- a/ietf/templates/community/list_menu.html
+++ b/ietf/templates/community/list_menu.html
@@ -3,18 +3,18 @@
     <div class="btn-group" role="group" aria-labelledby="list-feeds">
         <a class="btn btn-primary"
            title="Feed of all changes"
-           href="{% if clist.group %}{% url "ietf.community.views.feed" acronym=clist.group.acronym %}{% else %}{% url "ietf.community.views.feed" email_or_name=clist.person.email_address %}{% endif %}">
+           href="{% if clist.group %}{% url "ietf.community.views.feed" acronym=clist.group.acronym %}{% else %}{% url "ietf.community.views.feed" email_or_name=email_or_name %}{% endif %}">
             <i class="bi bi-rss"></i> All changes
         </a>
         <a class="btn btn-primary"
            title="Feed of only significant state changes"
-           href="{% if clist.group %}{% url "ietf.community.views.feed" acronym=clist.group.acronym %}{% else %}{% url "ietf.community.views.feed" email_or_name=clist.person.email_address %}{% endif %}?significant=1">
+           href="{% if clist.group %}{% url "ietf.community.views.feed" acronym=clist.group.acronym %}{% else %}{% url "ietf.community.views.feed" email_or_name=email_or_name %}{% endif %}?significant=1">
             <i class="bi bi-rss"></i> Significant
         </a>
     </div>
     {% if clist.pk != None %}
         <a class="btn btn-primary"
-           href="{% if clist.group %}{% url "ietf.community.views.subscription" acronym=clist.group.acronym %}{% else %}{% url "ietf.community.views.subscription" email_or_name=clist.person.email_address %}{% endif %}">
+           href="{% if clist.group %}{% url "ietf.community.views.subscription" acronym=clist.group.acronym %}{% else %}{% url "ietf.community.views.subscription" email_or_name=email_or_name %}{% endif %}">
             <i class="bi bi-envelope"></i>
             {% if subscribed %}
                 Change subscription
@@ -24,7 +24,7 @@
         </a>
     {% endif %}
     <a class="btn btn-primary"
-       href="{% if clist.group %}{% url "ietf.community.views.export_to_csv" acronym=clist.group.acronym %}{% else %}{% url "ietf.community.views.export_to_csv" email_or_name=clist.person.email_address %}{% endif %}">
+       href="{% if clist.group %}{% url "ietf.community.views.export_to_csv" acronym=clist.group.acronym %}{% else %}{% url "ietf.community.views.export_to_csv" email_or_name=email_or_name %}{% endif %}">
         <i class="bi bi-file-ruled"></i> Export as CSV
     </a>
 </div>
diff --git a/ietf/templates/community/view_list.html b/ietf/templates/community/view_list.html
index 7ca9f52748..a543eaf7cf 100644
--- a/ietf/templates/community/view_list.html
+++ b/ietf/templates/community/view_list.html
@@ -12,7 +12,7 @@ <h1>{{ clist.long_name }}</h1>
     {% bootstrap_messages %}
     {% if can_manage_list %}
         <a class="btn btn-primary my-3"
-           href="{% url "ietf.community.views.manage_list" email_or_name=clist.person.email_address %}">
+           href="{% url "ietf.community.views.manage_list" email_or_name=email_or_name %}">
             <i class="bi bi-gear"></i>
             Manage list
         </a>

From d7be91f784e9fb77193b486fece4e1a0a2f70d9c Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 16 Sep 2024 10:28:13 -0500
Subject: [PATCH 073/656] fix: pin pydyf until weasyprint adjusts for its
 deprecations (#7945)

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index c51ed9ac00..2e6e2714d7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -49,6 +49,7 @@ oic>=1.3    # Used only by tests
 Pillow>=9.1.0
 psycopg2>=2.9.6
 pyang>=2.5.3
+pydyf>0.8.0,<0.10.0 # until weasyprint adjusts for 0.10.0 and later
 pyflakes>=2.4.0
 pyopenssl>=22.0.0    # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency
 pyquery>=1.4.3

From 8362b45c8e94d5bf3585f27b2e821388d9b9fb14 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 17 Sep 2024 13:05:56 -0500
Subject: [PATCH 074/656] fix: optimize and debug has_role and
 can_manage_some_groups (#7949)

* fix: optimize can_manage_some_groups

* fix: improve cache key

* refactor: extra_role_qs to kwargs and bugfix to cache key

* fix: restrict groupman_role matches to active states

* chore: styling, decommenting, black
---
 ietf/group/utils.py    |  21 ++++--
 ietf/ietfauth/utils.py | 162 +++++++++++++++++++++++++++++------------
 2 files changed, 131 insertions(+), 52 deletions(-)

diff --git a/ietf/group/utils.py b/ietf/group/utils.py
index 68b618120b..dcf9d83e6f 100644
--- a/ietf/group/utils.py
+++ b/ietf/group/utils.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 import datetime
 
+from itertools import chain
 from pathlib import Path
 
 from django.db.models import Q
@@ -153,17 +154,23 @@ def can_manage_materials(user, group):
 def can_manage_session_materials(user, group, session):
     return has_role(user, 'Secretariat') or (group.has_role(user, group.features.matman_roles) and not session.is_material_submission_cutoff())
 
-# Maybe this should be cached...
 def can_manage_some_groups(user):
     if not user.is_authenticated:
         return False
+    authroles = set(
+        chain.from_iterable(
+            GroupFeatures.objects.values_list("groupman_authroles", flat=True)
+        )
+    )
+    extra_role_qs = dict()
     for gf in GroupFeatures.objects.all():
-        for authrole in gf.groupman_authroles:
-            if has_role(user, authrole):
-                return True
-            if Role.objects.filter(name__in=gf.groupman_roles, group__type_id=gf.type_id, person__user=user).exists():
-                return True
-    return False          
+        extra_role_qs[f"{gf.type_id} groupman roles"] = Q(
+            name__in=gf.groupman_roles,
+            group__type_id=gf.type_id,
+            group__state__in=["active", "bof", "proposed"],
+        )
+    return has_role(user, authroles, extra_role_qs=extra_role_qs)
+       
 
 def can_provide_status_update(user, group):
     if not group.features.acts_like_wg:
diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py
index 6fa9cddbcb..d8bd67a4e0 100644
--- a/ietf/ietfauth/utils.py
+++ b/ietf/ietfauth/utils.py
@@ -38,9 +38,10 @@ def has_role(user, role_names, *args, **kwargs):
     """Determines whether user has any of the given standard roles
     given. Role names must be a list or, in case of a single value, a
     string."""
-    if not isinstance(role_names, (list, tuple)):
-        role_names = [ role_names ]
-    
+    extra_role_qs = kwargs.get("extra_role_qs", None)
+    if not isinstance(role_names, (list, tuple, set)):
+        role_names = [role_names]
+
     if not user or not user.is_authenticated:
         return False
 
@@ -48,7 +49,13 @@ def has_role(user, role_names, *args, **kwargs):
     if not hasattr(user, "roles_check_cache"):
         user.roles_check_cache = {}
 
-    key = frozenset(role_names)
+    keynames = set(role_names)
+    if extra_role_qs:
+        keynames.update(set(extra_role_qs.keys()))
+    year = kwargs.get("year", None)
+    if year is not None:
+        keynames.add(f"nomcomyear{year}")
+    key = frozenset(keynames)
     if key not in user.roles_check_cache:
         try:
             person = user.person
@@ -56,54 +63,119 @@ def has_role(user, role_names, *args, **kwargs):
             return False
 
         role_qs = {
-            "Area Director": Q(person=person, name__in=("pre-ad", "ad"), group__type="area", group__state="active"),
-            "Secretariat": Q(person=person, name="secr", group__acronym="secretariat"),
-            "IAB" : Q(person=person, name="member", group__acronym="iab"),
-            "IANA": Q(person=person, name="auth", group__acronym="iana"),
-            "RFC Editor": Q(person=person, name="auth", group__acronym="rpc"),
-            "ISE" : Q(person=person, name="chair", group__acronym="ise"),
-            "IAD": Q(person=person, name="admdir", group__acronym="ietf"),
-            "IETF Chair": Q(person=person, name="chair", group__acronym="ietf"),
-            "IETF Trust Chair": Q(person=person, name="chair", group__acronym="ietf-trust"),
-            "IRTF Chair": Q(person=person, name="chair", group__acronym="irtf"),
-            "RSAB Chair": Q(person=person, name="chair", group__acronym="rsab"),
-            "IAB Chair": Q(person=person, name="chair", group__acronym="iab"),
-            "IAB Executive Director": Q(person=person, name="execdir", group__acronym="iab"),
-            "IAB Group Chair": Q(person=person, name="chair", group__type="iab", group__state="active"),
-            "IAOC Chair": Q(person=person, name="chair", group__acronym="iaoc"),
-            "WG Chair": Q(person=person,name="chair", group__type="wg", group__state__in=["active","bof", "proposed"]),
-            "WG Secretary": Q(person=person,name="secr", group__type="wg", group__state__in=["active","bof", "proposed"]),
-            "RG Chair": Q(person=person,name="chair", group__type="rg", group__state__in=["active","proposed"]),
-            "RG Secretary": Q(person=person,name="secr", group__type="rg", group__state__in=["active","proposed"]),
-            "AG Secretary": Q(person=person,name="secr", group__type="ag", group__state__in=["active"]),
-            "RAG Secretary": Q(person=person,name="secr", group__type="rag", group__state__in=["active"]),
-            "Team Chair": Q(person=person,name="chair", group__type="team", group__state="active"),
-            "Program Lead": Q(person=person,name="lead", group__type="program", group__state="active"),
-            "Program Secretary": Q(person=person,name="secr", group__type="program", group__state="active"),
-            "Program Chair": Q(person=person,name="chair", group__type="program", group__state="active"),
-            "EDWG Chair": Q(person=person, name="chair", group__type="edwg", group__state="active"),
-            "Nomcom Chair": Q(person=person, name="chair", group__type="nomcom", group__acronym__icontains=kwargs.get('year', '0000')),
-            "Nomcom Advisor": Q(person=person, name="advisor", group__type="nomcom", group__acronym__icontains=kwargs.get('year', '0000')),
-            "Nomcom": Q(person=person, group__type="nomcom", group__acronym__icontains=kwargs.get('year', '0000')),
-            "Liaison Manager": Q(person=person,name="liaiman",group__type="sdo",group__state="active", ),
-            "Authorized Individual": Q(person=person,name="auth",group__type="sdo",group__state="active", ),
-            "Recording Manager": Q(person=person,name="recman",group__type="ietf",group__state="active", ),
-            "Reviewer": Q(person=person, name="reviewer", group__state="active"),
-            "Review Team Secretary": Q(person=person, name="secr", group__reviewteamsettings__isnull=False,group__state="active", ),
-            "IRSG Member": (Q(person=person, name="member", group__acronym="irsg") | Q(person=person, name="chair", group__acronym="irtf") | Q(person=person, name="atlarge", group__acronym="irsg")),
-            "RSAB Member": Q(person=person, name="member", group__acronym="rsab"), 
-            "Robot": Q(person=person, name="robot", group__acronym="secretariat"),
-            }
-
-        filter_expr = Q(pk__in=[])  # ensure empty set is returned if no other terms are added
+            "Area Director": Q(
+                name__in=("pre-ad", "ad"), group__type="area", group__state="active"
+            ),
+            "Secretariat": Q(name="secr", group__acronym="secretariat"),
+            "IAB": Q(name="member", group__acronym="iab"),
+            "IANA": Q(name="auth", group__acronym="iana"),
+            "RFC Editor": Q(name="auth", group__acronym="rpc"),
+            "ISE": Q(name="chair", group__acronym="ise"),
+            "IAD": Q(name="admdir", group__acronym="ietf"),
+            "IETF Chair": Q(name="chair", group__acronym="ietf"),
+            "IETF Trust Chair": Q(name="chair", group__acronym="ietf-trust"),
+            "IRTF Chair": Q(name="chair", group__acronym="irtf"),
+            "RSAB Chair": Q(name="chair", group__acronym="rsab"),
+            "IAB Chair": Q(name="chair", group__acronym="iab"),
+            "IAB Executive Director": Q(name="execdir", group__acronym="iab"),
+            "IAB Group Chair": Q(
+                name="chair", group__type="iab", group__state="active"
+            ),
+            "IAOC Chair": Q(name="chair", group__acronym="iaoc"),
+            "WG Chair": Q(
+                name="chair",
+                group__type="wg",
+                group__state__in=["active", "bof", "proposed"],
+            ),
+            "WG Secretary": Q(
+                name="secr",
+                group__type="wg",
+                group__state__in=["active", "bof", "proposed"],
+            ),
+            "RG Chair": Q(
+                name="chair", group__type="rg", group__state__in=["active", "proposed"]
+            ),
+            "RG Secretary": Q(
+                name="secr", group__type="rg", group__state__in=["active", "proposed"]
+            ),
+            "AG Secretary": Q(
+                name="secr", group__type="ag", group__state__in=["active"]
+            ),
+            "RAG Secretary": Q(
+                name="secr", group__type="rag", group__state__in=["active"]
+            ),
+            "Team Chair": Q(name="chair", group__type="team", group__state="active"),
+            "Program Lead": Q(
+                name="lead", group__type="program", group__state="active"
+            ),
+            "Program Secretary": Q(
+                name="secr", group__type="program", group__state="active"
+            ),
+            "Program Chair": Q(
+                name="chair", group__type="program", group__state="active"
+            ),
+            "EDWG Chair": Q(name="chair", group__type="edwg", group__state="active"),
+            "Nomcom Chair": Q(
+                name="chair",
+                group__type="nomcom",
+                group__acronym__icontains=kwargs.get("year", "0000"),
+            ),
+            "Nomcom Advisor": Q(
+                name="advisor",
+                group__type="nomcom",
+                group__acronym__icontains=kwargs.get("year", "0000"),
+            ),
+            "Nomcom": Q(
+                group__type="nomcom",
+                group__acronym__icontains=kwargs.get("year", "0000"),
+            ),
+            "Liaison Manager": Q(
+                name="liaiman",
+                group__type="sdo",
+                group__state="active",
+            ),
+            "Authorized Individual": Q(
+                name="auth",
+                group__type="sdo",
+                group__state="active",
+            ),
+            "Recording Manager": Q(
+                name="recman",
+                group__type="ietf",
+                group__state="active",
+            ),
+            "Reviewer": Q(name="reviewer", group__state="active"),
+            "Review Team Secretary": Q(
+                name="secr",
+                group__reviewteamsettings__isnull=False,
+                group__state="active",
+            ),
+            "IRSG Member": (
+                Q(name="member", group__acronym="irsg")
+                | Q(name="chair", group__acronym="irtf")
+                | Q(name="atlarge", group__acronym="irsg")
+            ),
+            "RSAB Member": Q(name="member", group__acronym="rsab"),
+            "Robot": Q(name="robot", group__acronym="secretariat"),
+        }
+
+        filter_expr = Q(
+            pk__in=[]
+        )  # ensure empty set is returned if no other terms are added
         for r in role_names:
             filter_expr |= role_qs[r]
+        if extra_role_qs:
+            for r in extra_role_qs:
+                filter_expr |= extra_role_qs[r]
 
-        user.roles_check_cache[key] = bool(Role.objects.filter(filter_expr).exists())
+        user.roles_check_cache[key] = bool(
+            Role.objects.filter(person=person).filter(filter_expr).exists()
+        )
 
     return user.roles_check_cache[key]
 
 
+
 # convenient decorator
 
 def passes_test_decorator(test_func, message):

From 97b719505ee162d0c55d57edbdd029f766c27a71 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 17 Sep 2024 13:06:10 -0500
Subject: [PATCH 075/656] test: focus on a helper rather than a whole view
 (#7951)

---
 ietf/meeting/tests_views.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index e4f62838de..f951eb682d 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -39,7 +39,7 @@
 from ietf.group.models import Group, Role, GroupFeatures
 from ietf.group.utils import can_manage_group
 from ietf.person.models import Person, PersonalApiKey
-from ietf.meeting.helpers import can_approve_interim_request, can_view_interim_request, preprocess_assignments_for_agenda
+from ietf.meeting.helpers import can_approve_interim_request, can_request_interim_meeting, can_view_interim_request, preprocess_assignments_for_agenda
 from ietf.meeting.helpers import send_interim_approval_request, AgendaKeywordTagger
 from ietf.meeting.helpers import send_interim_meeting_cancellation_notice, send_interim_session_cancellation_notice
 from ietf.meeting.helpers import send_interim_minutes_reminder, populate_important_dates, update_important_dates
@@ -7334,10 +7334,7 @@ def test_cannot_request_interim(self):
         for gf in GroupFeatures.objects.filter(has_meetings=True):
             for role_name in all_role_names - set(gf.groupman_roles):
                 role = RoleFactory(group__type_id=gf.type_id,name_id=role_name)
-                self.client.login(username=role.person.user.username, password=role.person.user.username+'+password')
-                r = self.client.get(url)
-                self.assertEqual(r.status_code, 403)
-                self.client.logout()
+                self.assertFalse(can_request_interim_meeting(role.person.user))
 
     def test_appears_on_upcoming(self):
         url = urlreverse('ietf.meeting.views.upcoming')

From c6389ba65fdefdce72661f3ee3d21f7d708da841 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 18 Sep 2024 16:02:00 -0300
Subject: [PATCH 076/656] fix: only send state change notifications once
 (#7953)

* feat: split state_change_event create / save

* refactor: avoid double-save in rfceditor.py

* feat: only send notifications on event creation

* test: update test_notification_signal_receiver()

* chore: update comment
---
 ietf/community/models.py |  5 ++++-
 ietf/community/tests.py  | 27 +++++++++++++++++----------
 ietf/doc/utils.py        | 25 ++++++++++++++++++++++---
 ietf/sync/rfceditor.py   | 13 +++++++------
 4 files changed, 50 insertions(+), 20 deletions(-)

diff --git a/ietf/community/models.py b/ietf/community/models.py
index b1295461d6..4d820eb0f1 100644
--- a/ietf/community/models.py
+++ b/ietf/community/models.py
@@ -104,12 +104,15 @@ def notify_events(sender, instance, **kwargs):
     if not isinstance(instance, DocEvent):
         return
 
+    if not kwargs.get("created", False):
+        return  # only notify on creation
+
     if instance.doc.type_id != 'draft':
         return
 
     if getattr(instance, "skip_community_list_notification", False):
         return
-
+    
     # kludge alert: queuing a celery task in response to a signal can cause unexpected attempts to
     # start a Celery task during tests. To prevent this, don't queue a celery task if we're running
     # tests.
diff --git a/ietf/community/tests.py b/ietf/community/tests.py
index 84e4370771..181e9e0fa6 100644
--- a/ietf/community/tests.py
+++ b/ietf/community/tests.py
@@ -17,6 +17,7 @@
 import ietf.community.views
 from ietf.group.models import Group
 from ietf.group.utils import setup_default_community_list_for_group
+from ietf.doc.factories import DocumentFactory
 from ietf.doc.models import State
 from ietf.doc.utils import add_state_change_event
 from ietf.person.models import Person, Email, Alias
@@ -439,39 +440,45 @@ def test_notification_signal_receiver(self, mock_notify_task):
         This implicitly tests that notify_events is hooked up to the post_save signal.
         """
         # Arbitrary model that's not a DocEvent
-        p = PersonFactory()
+        person = PersonFactory()
         mock_notify_task.reset_mock()  # clear any calls that resulted from the factories
         # be careful overriding SERVER_MODE - we do it here because the method
         # under test does not make this call when in "test" mode
         with override_settings(SERVER_MODE="not-test"):
-            p.save()
+            person.save()
         self.assertFalse(mock_notify_task.delay.called)
         
-        d = DocEventFactory()
-        mock_notify_task.reset_mock()  # clear any calls that resulted from the factories
+        # build a DocEvent that is not yet persisted
+        doc = DocumentFactory()
+        d = DocEventFactory.build(by=person, doc=doc)
+        # mock_notify_task.reset_mock()  # clear any calls that resulted from the factories
         # be careful overriding SERVER_MODE - we do it here because the method
         # under test does not make this call when in "test" mode
         with override_settings(SERVER_MODE="not-test"):
             d.save()
-        self.assertEqual(mock_notify_task.delay.call_count, 1)
+        self.assertEqual(mock_notify_task.delay.call_count, 1, "notify_task should be run on creation of DocEvent")
         self.assertEqual(mock_notify_task.delay.call_args, mock.call(event_id = d.pk))
         
         mock_notify_task.reset_mock()
+        with override_settings(SERVER_MODE="not-test"):
+            d.save()
+        self.assertFalse(mock_notify_task.delay.called, "notify_task should not be run save of on existing DocEvent")
+        
+        mock_notify_task.reset_mock()
+        d = DocEventFactory.build(by=person, doc=doc)
         d.skip_community_list_notification = True
         # be careful overriding SERVER_MODE - we do it here because the method
         # under test does not make this call when in "test" mode
         with override_settings(SERVER_MODE="not-test"):
             d.save()
-        self.assertFalse(mock_notify_task.delay.called)
+        self.assertFalse(mock_notify_task.delay.called, "notify_task should not run when skip_community_list_notification is set")
 
-        del(d.skip_community_list_notification)
-        d.doc.type_id="rfc"  # not "draft"
-        d.doc.save()
+        d = DocEventFactory.build(by=person, doc=DocumentFactory(type_id="rfc"))
         # be careful overriding SERVER_MODE - we do it here because the method
         # under test does not make this call when in "test" mode
         with override_settings(SERVER_MODE="not-test"):
             d.save()
-        self.assertFalse(mock_notify_task.delay.called)
+        self.assertFalse(mock_notify_task.delay.called, "notify_task should not run on a document with type 'rfc'")
 
     @mock.patch("ietf.utils.mail.send_mail_text")
     def test_notify_event_to_subscribers(self, mock_send_mail_text):
diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index 74000e598b..97243a20d6 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -398,8 +398,12 @@ def get_unicode_document_content(key, filename, codec='utf-8', errors='ignore'):
 def tags_suffix(tags):
     return ("::" + "::".join(t.name for t in tags)) if tags else ""
 
-def add_state_change_event(doc, by, prev_state, new_state, prev_tags=None, new_tags=None, timestamp=None):
-    """Add doc event to explain that state change just happened."""
+
+def new_state_change_event(doc, by, prev_state, new_state, prev_tags=None, new_tags=None, timestamp=None):
+    """Create unsaved doc event to explain that state change just happened
+    
+    Returns None if no state change occurred.
+    """
     if prev_state and new_state:
         assert prev_state.type_id == new_state.type_id
 
@@ -419,7 +423,22 @@ def add_state_change_event(doc, by, prev_state, new_state, prev_tags=None, new_t
         e.desc += " from %s" % (prev_state.name + tags_suffix(prev_tags))
     if timestamp:
         e.time = timestamp
-    e.save()
+    return e  # not saved!
+
+
+def add_state_change_event(doc, by, prev_state, new_state, prev_tags=None, new_tags=None, timestamp=None):
+    """Add doc event to explain that state change just happened.
+    
+    Returns None if no state change occurred.
+    
+    Note: Creating a state change DocEvent will trigger notifications to be sent to people subscribed
+    to the doc via a CommunityList on its first save(). If you need to adjust the event (say, changing
+    its desc) before that notification is sent, use new_state_change_event() instead and save the
+    event after making your changes. 
+    """
+    e = new_state_change_event(doc, by, prev_state, new_state, prev_tags, new_tags, timestamp)
+    if e is not None:
+        e.save()
     return e
 
 
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index d3371ea36c..6b3482f10d 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -21,7 +21,7 @@
 from ietf.doc.models import ( Document, State, StateType, DocEvent, DocRelationshipName,
     DocTagName, RelatedDocument, RelatedDocHistory )
 from ietf.doc.expire import move_draft_files_to_archive
-from ietf.doc.utils import add_state_change_event, prettify_std_name, update_action_holders
+from ietf.doc.utils import add_state_change_event, new_state_change_event, prettify_std_name, update_action_holders
 from ietf.group.models import Group
 from ietf.ipr.models import IprDocRel
 from ietf.name.models import StdLevelName, StreamName
@@ -202,11 +202,14 @@ def update_drafts_from_queue(drafts):
         if prev_state != next_state:
             d.set_state(next_state)
 
-            e = add_state_change_event(d, system, prev_state, next_state)
+            e = new_state_change_event(d, system, prev_state, next_state)  # unsaved
+            if e:
+                if auth48:
+                    e.desc = re.sub(r"(<b>.*</b>)", "<a href=\"%s\">\\1</a>" % auth48, e.desc)
+                e.save()
+                events.append(e)
 
             if auth48:
-                e.desc = re.sub(r"(<b>.*</b>)", "<a href=\"%s\">\\1</a>" % auth48, e.desc)
-                e.save()
                 # Create or update the auth48 URL whether or not this is a state expected to have one.
                 d.documenturl_set.update_or_create(
                     tag_id='auth48',  # look up existing based on this field
@@ -215,8 +218,6 @@ def update_drafts_from_queue(drafts):
             else:
                 # Remove any existing auth48 URL when an update does not have one.
                 d.documenturl_set.filter(tag_id='auth48').delete()
-            if e:
-                events.append(e)
 
             changed.add(name)
 

From 35074660dc18cb2ede7c209312dfb2ab32deb740 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Wed, 18 Sep 2024 12:08:01 -0700
Subject: [PATCH 077/656] chore: Prevent test suite artifact creation in work
 directory (#6438)

* chore: Prevent test suite artifact creation in work directory

Also remove a few other stale test assets while I'm here.

* Try and fix CI

* Change IDSUBMIT_REPOSITORY_PATH

* Make the dir

* test: clean up media/nomcom directories

* test: de-dup settings_temp_path_overrides

* chore: remove debug

* refactor: avoid premature import of test_utils

* refactor: drop useless lambda wrapper

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 dev/tests/settings_local.py          |     4 +-
 docker/configs/settings_local.py     |     4 +-
 docker/scripts/app-create-dirs.sh    |     8 +-
 ietf/meeting/tests_js.py             |    11 +-
 ietf/person/factories.py             |     7 +-
 ietf/settings_test.py                |    24 +-
 ietf/utils/test_utils.py             |    13 +-
 media/.gitignore                     |     1 -
 media/photo/nopictureavailable.jpg   |   Bin 1618 -> 0 bytes
 media/photo/profile-default.jpg      |   Bin 1397 -> 0 bytes
 test/data/profile-default.jpg        |   Bin 1397 -> 0 bytes
 test/data/youtube-discovery.json     | 10879 -------------------------
 test/data/youtube-playlistid.json    |     1 -
 test/data/youtube-playlistitems.json |     1 -
 test/lib/.gitignore                  |     1 -
 test/lib/README                      |     9 -
 test/media/floor/.gitignore          |     1 -
 test/media/photo/.gitignore          |     1 -
 18 files changed, 33 insertions(+), 10932 deletions(-)
 delete mode 100644 media/.gitignore
 delete mode 100644 media/photo/nopictureavailable.jpg
 delete mode 100644 media/photo/profile-default.jpg
 delete mode 100644 test/data/profile-default.jpg
 delete mode 100644 test/data/youtube-discovery.json
 delete mode 100644 test/data/youtube-playlistid.json
 delete mode 100644 test/data/youtube-playlistitems.json
 delete mode 100644 test/lib/.gitignore
 delete mode 100644 test/lib/README
 delete mode 100644 test/media/floor/.gitignore
 delete mode 100644 test/media/photo/.gitignore

diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py
index 8b5d90b1ec..20941359d4 100644
--- a/dev/tests/settings_local.py
+++ b/dev/tests/settings_local.py
@@ -17,8 +17,8 @@
 }
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
-IDSUBMIT_REPOSITORY_PATH = "test/id/"
-IDSUBMIT_STAGING_PATH = "test/staging/"
+IDSUBMIT_REPOSITORY_PATH = "/assets/ietfdata/doc/draft/repository"
+IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/"
 
 AGENDA_PATH = '/assets/www6s/proceedings/'
 MEETINGHOST_LOGO_PATH = AGENDA_PATH
diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py
index bcd04898ea..5d9859c19b 100644
--- a/docker/configs/settings_local.py
+++ b/docker/configs/settings_local.py
@@ -8,7 +8,7 @@
 from ietf.settings_postgresqldb import DATABASES   # pyflakes:ignore
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
-IDSUBMIT_STAGING_PATH = "test/staging/"
+IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/"
 
 AGENDA_PATH = '/assets/www6s/proceedings/'
 MEETINGHOST_LOGO_PATH = AGENDA_PATH
@@ -53,7 +53,7 @@
 FTP_DIR = '/assets/ftp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
-SLIDE_STAGING_PATH = 'test/staging/'
+SLIDE_STAGING_PATH = '/assets/www6s/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
 
diff --git a/docker/scripts/app-create-dirs.sh b/docker/scripts/app-create-dirs.sh
index b75c57767d..50431f4793 100755
--- a/docker/scripts/app-create-dirs.sh
+++ b/docker/scripts/app-create-dirs.sh
@@ -1,13 +1,6 @@
 #!/bin/bash
 
 for sub in \
-    test/id \
-    test/staging \
-    test/archive \
-    test/rfc \
-    test/media \
-    test/wiki/ietf \
-    data/nomcom_keys/public_keys \
     /assets/archive/id \
     /assets/collection \
     /assets/collection/draft-archive \
@@ -27,6 +20,7 @@ for sub in \
     /assets/ietfdata/derived \
     /assets/ietfdata/derived/bibxml \
     /assets/ietfdata/derived/bibxml/bibxml-ids \
+    /assets/ietfdata/doc/draft/repository \
     /assets/www6s \
     /assets/www6s/staging \
     /assets/www6s/wg-descriptions \
diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py
index 6199ed7eb5..b15aa70e73 100644
--- a/ietf/meeting/tests_js.py
+++ b/ietf/meeting/tests_js.py
@@ -5,7 +5,7 @@
 import time
 import datetime
 import shutil
-import os
+import tempfile
 import re
 
 from django.utils import timezone
@@ -939,13 +939,8 @@ def tearDown(self):
     def tempdir(self, label):
         # Borrowed from  test_utils.TestCase
         slug = slugify(self.__class__.__name__.replace('.','-'))
-        dirname = "tmp-{label}-{slug}-dir".format(**locals())
-        if 'VIRTUAL_ENV' in os.environ:
-            dirname = os.path.join(os.environ['VIRTUAL_ENV'], dirname)
-        path = os.path.abspath(dirname)
-        if not os.path.exists(path):
-            os.mkdir(path)
-        return path
+        suffix = "-{label}-{slug}-dir".format(**locals())
+        return tempfile.mkdtemp(suffix=suffix)
 
     def displayed_interims(self, groups=None):
         sessions = add_event_info_to_session_qs(
diff --git a/ietf/person/factories.py b/ietf/person/factories.py
index 2247fa9b2b..2012483c0d 100644
--- a/ietf/person/factories.py
+++ b/ietf/person/factories.py
@@ -8,7 +8,7 @@
 import faker.config
 import os
 import random
-import shutil
+from PIL import Image
 
 from unidecode import unidecode
 from unicodedata import normalize
@@ -103,10 +103,9 @@ def default_photo(obj, create, extracted, **kwargs): # pylint: disable=no-self-a
             media_name = "%s/%s.jpg" % (settings.PHOTOS_DIRNAME, photo_name)
             obj.photo = media_name
             obj.photo_thumb = media_name
-            photosrc = os.path.join(settings.TEST_DATA_DIR, "profile-default.jpg")
             photodst = os.path.join(settings.PHOTOS_DIR,  photo_name + '.jpg')
-            if not os.path.exists(photodst):
-                shutil.copy(photosrc, photodst)
+            img = Image.new('RGB', (200, 200))
+            img.save(photodst)
             def delete_file(file):
                 os.unlink(file)
             atexit.register(delete_file, photodst)
diff --git a/ietf/settings_test.py b/ietf/settings_test.py
index 024512a8db..94ca22c71b 100755
--- a/ietf/settings_test.py
+++ b/ietf/settings_test.py
@@ -9,9 +9,12 @@
 #   ./manage.py test --settings=settings_test doc.ChangeStateTestCase
 #
 
-import os 
+import atexit
+import os
+import shutil
+import tempfile
 from ietf.settings import *                                          # pyflakes:ignore
-from ietf.settings import TEST_CODE_COVERAGE_CHECKER, BASE_DIR, PHOTOS_DIRNAME
+from ietf.settings import TEST_CODE_COVERAGE_CHECKER
 import debug                            # pyflakes:ignore
 debug.debug = True
 
@@ -48,11 +51,20 @@ def __getitem__(self, item):
 if TEST_CODE_COVERAGE_CHECKER and not TEST_CODE_COVERAGE_CHECKER._started: # pyflakes:ignore
     TEST_CODE_COVERAGE_CHECKER.start()                          # pyflakes:ignore
 
-NOMCOM_PUBLIC_KEYS_DIR=os.path.abspath("tmp-nomcom-public-keys-dir")
 
-MEDIA_ROOT = os.path.join(os.path.dirname(BASE_DIR), 'test/media/') # pyflakes:ignore
-MEDIA_URL = '/test/media/'
-PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME                            # pyflakes:ignore
+def tempdir_with_cleanup(**kwargs):
+    """Utility to create a temporary dir and arrange cleanup"""
+    _dir = tempfile.mkdtemp(**kwargs)
+    atexit.register(shutil.rmtree, _dir)
+    return _dir
+
+
+NOMCOM_PUBLIC_KEYS_DIR = tempdir_with_cleanup(suffix="-nomcom-public-keys-dir")
+
+MEDIA_ROOT = tempdir_with_cleanup(suffix="-media")
+PHOTOS_DIRNAME = "photo"
+PHOTOS_DIR = os.path.join(MEDIA_ROOT, PHOTOS_DIRNAME)
+os.mkdir(PHOTOS_DIR)
 
 # Undo any developer-dependent middleware when running the tests
 MIDDLEWARE = [ c for c in MIDDLEWARE if not c in DEV_MIDDLEWARE ] # pyflakes:ignore
diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py
index ba35665a8d..86c5a0c1c3 100644
--- a/ietf/utils/test_utils.py
+++ b/ietf/utils/test_utils.py
@@ -34,7 +34,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-import os
+import tempfile
 import re
 import email
 import html5lib
@@ -239,13 +239,8 @@ def normalize(x):
 
     def tempdir(self, label):
         slug = slugify(self.__class__.__name__.replace('.','-'))
-        dirname = "tmp-{label}-{slug}-dir".format(**locals())
-        if 'VIRTUAL_ENV' in os.environ:
-            dirname = os.path.join(os.environ['VIRTUAL_ENV'], dirname)
-        path = os.path.abspath(dirname)
-        if not os.path.exists(path):
-            os.mkdir(path)
-        return path
+        suffix = "-{label}-{slug}-dir".format(**locals())
+        return tempfile.mkdtemp(suffix=suffix)
 
     def assertNoFormPostErrors(self, response, error_css_selector=".is-invalid"):
         """Try to fish out form errors, if none found at least check the
@@ -306,7 +301,7 @@ def setUp(self):
 
         # Replace settings paths with temporary directories.
         self._ietf_temp_dirs = {}  # trashed during tearDown, DO NOT put paths you care about in this
-        for setting in self.settings_temp_path_overrides:
+        for setting in set(self.settings_temp_path_overrides):
             self._ietf_temp_dirs[setting] = self.tempdir(slugify(setting))
         self._ietf_saved_context = django.test.utils.override_settings(**self._ietf_temp_dirs)
         self._ietf_saved_context.enable()
diff --git a/media/.gitignore b/media/.gitignore
deleted file mode 100644
index dfa8ca37ce..0000000000
--- a/media/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-/floor
diff --git a/media/photo/nopictureavailable.jpg b/media/photo/nopictureavailable.jpg
deleted file mode 100644
index 0895f9f57c74c6199a8eed323d71fe1caae52c9e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1618
zcmeH|+dmTu0LM3)<nCZ1%TOm}$keE53nO!%nY-LpE*YYu#N0NSa*b9<<D8DdBrJ+7
zcS`u2Qpq(dZA>k;R%}$~pEytF;rIK#{T{wAzdTR(6Uqv@3IG5=8S8}c`bFZuk(c|`
z37l)*0KhioXnT84ti3(dGco>jbnGbrz`CG-5FDjP&_s16MH=lH!^a-VId>PWyjQu@
z=THr+BV%?J5l6=&rVM#R@#nD{Psgk+J2z`Q96Zx3mMb;;pR{K}jlZf+qgdwG6!uO{
zK#ftX?-Wnom0Er|(y~0dqj~Ps=QLrJ;8yDAN09;$?@_%Uy+jiTuDEBa4d*1~7SWEG
z6jO7nwdHDjaS)T>I#(HlhvNAMf%EpgsUdG)&2BLo+Z`cV(6jpCtsn0y(u#mz?hr<j
zffqMQ^wG-!yT#$Wc8lZv<`*9>M7-%sLZuf(pZ);jyrh8$_!!!CwW8+(-yM}C7WZF=
zRJ+CS>v@ZRhFAVETiu)>2-{+idZKcKq`o`j<qQ;65EdjcIlPy<fa`auzshiw%5%<;
z$ZdM>rs<U>>S(#*nYoIlT6F;?h&jEzp3S)3_~5cZI+uzN!Z@b1aVzdKe9vS<ed)l>
zp5*@a!vSk+$Zg$u9f1|ALx~a+{?;0Iartn?NNNg3y74&0NlZ>;9Qm}A#uHNMrrV`|
zUoWF*^;2jWHIBAoE@!KG##v2PCq{P>n_+hKFO@*^_rauE-nfgBw$+5b3z)x=<yHLy
z(|RQ_v&mns0heFkF=hH_nZJ%37}HK;q~#zQvNG7ptvf)*HxA}@DY*H`suPWm%guu)
zqv|Lsa&XN?O`XdjHK(<Aal38Sj<U)}d)U2YTf@U4l=X*WtOLYcR5rLpS6avfzdW_t
zkP{T4flh~6$)JDNf&TWEGJX<{6_=ca0}WFy+qj8K^;$HDf{jV%Scj7~R<Ac7=UaN!
zRyxccVRcrA{;)jY8iaqtgsHS$G8)-00R?wS2Sb{JmS9%6HjYkJhws>W$U0vN7n?|9
zZokX;;JezMI27=_v&FS<CDt$LJVBQv_a6M^pYnmc0f_MI>IYv{$8`GBa&D6-*v3EJ
zOvRCT9kda%?xW*gfMJR#qFffNYaiKobFgca{ER$G9wYYz%akBTs@}5~%o6mILI%Y#
zLfwG*RB+19_Q)mr8K$#=QLNT(aAGfF0T%{e+ovQSVTYT{qBrs-V21gvV-S@H{f~Df
zL2NT0V4kE|M&!0dAb2k&l<kN3#JgFuZ8h@u*c0>15Z>P&`U-0duX!wbMXTYFc#XQ|
z30u|N*4pH1l8i(Oc~uwJ;<fxDVva21|MF-YUwKsPyvvKyaNQ=v0eZzUk9K$GrD4~?
zD17b|DPR{!FefS!Rl++}H3H2}+>|JFv(c+75;IOA3qaS)%o;L^+d2Mh`y9wxNS$h#
z?tzpzr76Cf__iW`vH`!4zB#cp)1SoTnr1$*LRLMQ7o2;II{u>p5Lu7`tY)MUJVUqQ
z0RR~Q7Gvvu0r;uZLZR=V=D^fd{(2^A2py^jjCW{&8()!)FE06#D%d1{JvP|QwXj+;
zY|r9f@H<mA@7~k>d}iV#4az7686bOo)NGF$DB2}c8==|f?7-9&`9RxkOH`QrTlKbV
z`4>C<)mn^VLF%_PTA;)elM04T`-rx~h4-~E#qPt|#gH%)8no*Fq5r@UKi@Jb9qAz3
erH0!=i&Q7I<(v2jXU|_>0l+%AWA537X8#0-_TYm6

diff --git a/media/photo/profile-default.jpg b/media/photo/profile-default.jpg
deleted file mode 100644
index d6b03e100415a0fa9effad25c793f33957929e2d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1397
zcmex=<NpH&0WUXCHwH#VMg|WcWcYuZ!I^=X2?RhSGZ3(_v48*v8yhPdCkGc7CkH1d
z7Y{!V7dIa_Cnv8UFCV{vfS>>ukC3pCfH06P05XITq?4J21E^7eo0A(TN+S4wfI*Oh
zA%!7@nNf*>Nsy6Qkn#T!25GQUnSsuQ0}gg37NE<3k~RVijLbkcb24+YLKHAEFfjuq
z*##916FG#0B^(2Tr#w_DY!neSG7m~hE-G2LQQ6pe;^vD$-6F__LQDsV3L;qr6}`p4
z!wfW8kXewyp5ekpK}JCV!P>Ier_+?rbWdKHE7*~#p#0QyQ}e|`{OR`!-(>uFVYbSi
z_fx^?Y8{@R=lZJ8ddOYb8nkom^=1i&%Z=CXH95KFgg%)Tdd($qGTX%sg3C5qe%w(L
z^&rfD#+(`RU43<LGlcpEDYL%c9wLyU{NeuQ;O~C-RNgq+FM6s{<gC1CYG3jV@0jbn
z+#dHh-e`9*DJdu^0SUdglP*Tq?+J|5eap9U{oE-ZnRm~5#J0TQdho%LX%kOAn`X7<
z#umva8HH4r1#%y`5Bz6vdHumL{2`BK*1?LZ$$Ynzc-v;qviz0tGR5A@y-+>>?GDyA
zi-2ZxuAlNP?sD`i&5v(aF-m8r{<J-~R*G-BQ>5D5cMQ!(_%4O9t>yW>v~RuU`$cX|
zNhwBKzKfkX{3pv!OW(o8g~bI(ur4nu_;qdjzj@35xu$7*Joy+ZYL<P=WgYL!n}Ncf
zzum+I|70|s04a9-u=C;9;Ew%%=YMT<)w*zg){2!yi~gh*$Z9KfF5Y`?X7hgr|AVeC
zI64|SI)KFPYuZP{<o5>L*YKNsA;djUZg*#P<Ao1a8R4!1@i&V8GvppMv$FoXEKyZA
z$H`5gY|FPT-vvrp%A5A8ntk$}x8lzGFUK-{tur50^v|hme0n@@_SOq^rzSbL#A;t&
zk~?efh3TBECQFy@KKM-5<WV!jQ{UbPbFWwkH5Gh!`WPL1*537&pftZM_Z)Yzm2Zn@
zMJ}KEImGLy>cpextM7Qv@%!X?FPsG&04%Owb|~v@3`+>FnaqFt^rGa3S*v`bS?p)t
z{lm7riGOx>%y+h}OD2^t@Ge=qSl7}bZX>^I`LVWW-yJWWU1nVBbGXC)w%mV)?1p>W
zc($GRqun4aTeC*o`+(G}lQT>@{t26#-`{$tTCA`6?BO4I$6IqZ-YWRJO-d``yOG)3
z!ry+@cPb`%eVHgAD99E+Gm^W_@Vs~NHnu2zy{pG6X8zT@`=dv8`CHz-zupCXGdwV3
zK2uCx-jTBF2V(qPjcZq=F>md^Q+xY(2G_?}>8pJgJJx=>l|Su;cBY!#ouKW3?`0SC
z9WrZ?pTA<e&FLrCi?@_)?_o>X5!f5^i+ygEl6=8`h6x3~;%6paa#b<1l3ZL;WO#I4
zc3bO`e3mj1nLt6t=$+;wfvIh>f^OH?15J<5GP!zWuK%7Dof+SF{+*FAPg}g<w(9n-
zD2bLp_RGa{&&ocJU7<52$UJh+`HySo1z)u~^Q^u%=hM6u+7t>-RVtd+rSaJ=Std8L
zR)nwIB=cNZd**v<rl;4^jsIk)Px%_p*;Cm2TsF@4*Pj=@ft?$LMQV<Hj$Sw0PQ%6o
Ol*}Pf4QI*!zX<?xt}tl;

diff --git a/test/data/profile-default.jpg b/test/data/profile-default.jpg
deleted file mode 100644
index d6b03e100415a0fa9effad25c793f33957929e2d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1397
zcmex=<NpH&0WUXCHwH#VMg|WcWcYuZ!I^=X2?RhSGZ3(_v48*v8yhPdCkGc7CkH1d
z7Y{!V7dIa_Cnv8UFCV{vfS>>ukC3pCfH06P05XITq?4J21E^7eo0A(TN+S4wfI*Oh
zA%!7@nNf*>Nsy6Qkn#T!25GQUnSsuQ0}gg37NE<3k~RVijLbkcb24+YLKHAEFfjuq
z*##916FG#0B^(2Tr#w_DY!neSG7m~hE-G2LQQ6pe;^vD$-6F__LQDsV3L;qr6}`p4
z!wfW8kXewyp5ekpK}JCV!P>Ier_+?rbWdKHE7*~#p#0QyQ}e|`{OR`!-(>uFVYbSi
z_fx^?Y8{@R=lZJ8ddOYb8nkom^=1i&%Z=CXH95KFgg%)Tdd($qGTX%sg3C5qe%w(L
z^&rfD#+(`RU43<LGlcpEDYL%c9wLyU{NeuQ;O~C-RNgq+FM6s{<gC1CYG3jV@0jbn
z+#dHh-e`9*DJdu^0SUdglP*Tq?+J|5eap9U{oE-ZnRm~5#J0TQdho%LX%kOAn`X7<
z#umva8HH4r1#%y`5Bz6vdHumL{2`BK*1?LZ$$Ynzc-v;qviz0tGR5A@y-+>>?GDyA
zi-2ZxuAlNP?sD`i&5v(aF-m8r{<J-~R*G-BQ>5D5cMQ!(_%4O9t>yW>v~RuU`$cX|
zNhwBKzKfkX{3pv!OW(o8g~bI(ur4nu_;qdjzj@35xu$7*Joy+ZYL<P=WgYL!n}Ncf
zzum+I|70|s04a9-u=C;9;Ew%%=YMT<)w*zg){2!yi~gh*$Z9KfF5Y`?X7hgr|AVeC
zI64|SI)KFPYuZP{<o5>L*YKNsA;djUZg*#P<Ao1a8R4!1@i&V8GvppMv$FoXEKyZA
z$H`5gY|FPT-vvrp%A5A8ntk$}x8lzGFUK-{tur50^v|hme0n@@_SOq^rzSbL#A;t&
zk~?efh3TBECQFy@KKM-5<WV!jQ{UbPbFWwkH5Gh!`WPL1*537&pftZM_Z)Yzm2Zn@
zMJ}KEImGLy>cpextM7Qv@%!X?FPsG&04%Owb|~v@3`+>FnaqFt^rGa3S*v`bS?p)t
z{lm7riGOx>%y+h}OD2^t@Ge=qSl7}bZX>^I`LVWW-yJWWU1nVBbGXC)w%mV)?1p>W
zc($GRqun4aTeC*o`+(G}lQT>@{t26#-`{$tTCA`6?BO4I$6IqZ-YWRJO-d``yOG)3
z!ry+@cPb`%eVHgAD99E+Gm^W_@Vs~NHnu2zy{pG6X8zT@`=dv8`CHz-zupCXGdwV3
zK2uCx-jTBF2V(qPjcZq=F>md^Q+xY(2G_?}>8pJgJJx=>l|Su;cBY!#ouKW3?`0SC
z9WrZ?pTA<e&FLrCi?@_)?_o>X5!f5^i+ygEl6=8`h6x3~;%6paa#b<1l3ZL;WO#I4
zc3bO`e3mj1nLt6t=$+;wfvIh>f^OH?15J<5GP!zWuK%7Dof+SF{+*FAPg}g<w(9n-
zD2bLp_RGa{&&ocJU7<52$UJh+`HySo1z)u~^Q^u%=hM6u+7t>-RVtd+rSaJ=Std8L
zR)nwIB=cNZd**v<rl;4^jsIk)Px%_p*;Cm2TsF@4*Pj=@ft?$LMQV<Hj$Sw0PQ%6o
Ol*}Pf4QI*!zX<?xt}tl;

diff --git a/test/data/youtube-discovery.json b/test/data/youtube-discovery.json
deleted file mode 100644
index 983e1650c2..0000000000
--- a/test/data/youtube-discovery.json
+++ /dev/null
@@ -1,10879 +0,0 @@
-{
- "kind": "discovery#restDescription",
- "etag": "\"YWOzh2SDasdU84ArJnpYek-OMdg/f81k8b4sv9uLeywfoj2KpL2xcPg\"",
- "discoveryVersion": "v1",
- "id": "youtube:v3",
- "name": "youtube",
- "canonicalName": "YouTube",
- "version": "v3",
- "revision": "20170130",
- "title": "YouTube Data API",
- "description": "Supports core YouTube features, such as uploading videos, creating and managing playlists, searching for content, and much more.",
- "ownerDomain": "google.com",
- "ownerName": "Google",
- "icons": {
-  "x16": "https://www.google.com/images/icons/product/youtube-16.png",
-  "x32": "https://www.google.com/images/icons/product/youtube-32.png"
- },
- "documentationLink": "https://developers.google.com/youtube/v3",
- "protocol": "rest",
- "baseUrl": "https://www.googleapis.com/youtube/v3/",
- "basePath": "/youtube/v3/",
- "rootUrl": "https://www.googleapis.com/",
- "servicePath": "youtube/v3/",
- "batchPath": "batch",
- "parameters": {
-  "alt": {
-   "type": "string",
-   "description": "Data format for the response.",
-   "default": "json",
-   "enum": [
-    "json"
-   ],
-   "enumDescriptions": [
-    "Responses with Content-Type of application/json"
-   ],
-   "location": "query"
-  },
-  "fields": {
-   "type": "string",
-   "description": "Selector specifying which fields to include in a partial response.",
-   "location": "query"
-  },
-  "key": {
-   "type": "string",
-   "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.",
-   "location": "query"
-  },
-  "oauth_token": {
-   "type": "string",
-   "description": "OAuth 2.0 token for the current user.",
-   "location": "query"
-  },
-  "prettyPrint": {
-   "type": "boolean",
-   "description": "Returns response with indentations and line breaks.",
-   "default": "true",
-   "location": "query"
-  },
-  "quotaUser": {
-   "type": "string",
-   "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. Overrides userIp if both are provided.",
-   "location": "query"
-  },
-  "userIp": {
-   "type": "string",
-   "description": "IP address of the site where the request originates. Use this if you want to enforce per-user limits.",
-   "location": "query"
-  }
- },
- "auth": {
-  "oauth2": {
-   "scopes": {
-    "https://www.googleapis.com/auth/youtube": {
-     "description": "Manage your YouTube account"
-    },
-    "https://www.googleapis.com/auth/youtube.force-ssl": {
-     "description": "Manage your YouTube account"
-    },
-    "https://www.googleapis.com/auth/youtube.readonly": {
-     "description": "View your YouTube account"
-    },
-    "https://www.googleapis.com/auth/youtube.upload": {
-     "description": "Manage your YouTube videos"
-    },
-    "https://www.googleapis.com/auth/youtubepartner": {
-     "description": "View and manage your assets and associated content on YouTube"
-    },
-    "https://www.googleapis.com/auth/youtubepartner-channel-audit": {
-     "description": "View private information of your YouTube channel relevant during the audit process with a YouTube partner"
-    }
-   }
-  }
- },
- "schemas": {
-  "AccessPolicy": {
-   "id": "AccessPolicy",
-   "type": "object",
-   "description": "Rights management policy for YouTube resources.",
-   "properties": {
-    "allowed": {
-     "type": "boolean",
-     "description": "The value of allowed indicates whether the access to the policy is allowed or denied by default."
-    },
-    "exception": {
-     "type": "array",
-     "description": "A list of region codes that identify countries where the default policy do not apply.",
-     "items": {
-      "type": "string"
-     }
-    }
-   }
-  },
-  "Activity": {
-   "id": "Activity",
-   "type": "object",
-   "description": "An activity resource contains information about an action that a particular channel, or user, has taken on YouTube.The actions reported in activity feeds include rating a video, sharing a video, marking a video as a favorite, commenting on a video, uploading a video, and so forth. Each activity resource identifies the type of action, the channel associated with the action, and the resource(s) associated with the action, such as the video that was rated or uploaded.",
-   "properties": {
-    "contentDetails": {
-     "$ref": "ActivityContentDetails",
-     "description": "The contentDetails object contains information about the content associated with the activity. For example, if the snippet.type value is videoRated, then the contentDetails object's content identifies the rated video."
-    },
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the activity."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#activity\".",
-     "default": "youtube#activity"
-    },
-    "snippet": {
-     "$ref": "ActivitySnippet",
-     "description": "The snippet object contains basic details about the activity, including the activity's type and group ID."
-    }
-   }
-  },
-  "ActivityContentDetails": {
-   "id": "ActivityContentDetails",
-   "type": "object",
-   "description": "Details about the content of an activity: the video that was shared, the channel that was subscribed to, etc.",
-   "properties": {
-    "bulletin": {
-     "$ref": "ActivityContentDetailsBulletin",
-     "description": "The bulletin object contains details about a channel bulletin post. This object is only present if the snippet.type is bulletin."
-    },
-    "channelItem": {
-     "$ref": "ActivityContentDetailsChannelItem",
-     "description": "The channelItem object contains details about a resource which was added to a channel. This property is only present if the snippet.type is channelItem."
-    },
-    "comment": {
-     "$ref": "ActivityContentDetailsComment",
-     "description": "The comment object contains information about a resource that received a comment. This property is only present if the snippet.type is comment."
-    },
-    "favorite": {
-     "$ref": "ActivityContentDetailsFavorite",
-     "description": "The favorite object contains information about a video that was marked as a favorite video. This property is only present if the snippet.type is favorite."
-    },
-    "like": {
-     "$ref": "ActivityContentDetailsLike",
-     "description": "The like object contains information about a resource that received a positive (like) rating. This property is only present if the snippet.type is like."
-    },
-    "playlistItem": {
-     "$ref": "ActivityContentDetailsPlaylistItem",
-     "description": "The playlistItem object contains information about a new playlist item. This property is only present if the snippet.type is playlistItem."
-    },
-    "promotedItem": {
-     "$ref": "ActivityContentDetailsPromotedItem",
-     "description": "The promotedItem object contains details about a resource which is being promoted. This property is only present if the snippet.type is promotedItem."
-    },
-    "recommendation": {
-     "$ref": "ActivityContentDetailsRecommendation",
-     "description": "The recommendation object contains information about a recommended resource. This property is only present if the snippet.type is recommendation."
-    },
-    "social": {
-     "$ref": "ActivityContentDetailsSocial",
-     "description": "The social object contains details about a social network post. This property is only present if the snippet.type is social."
-    },
-    "subscription": {
-     "$ref": "ActivityContentDetailsSubscription",
-     "description": "The subscription object contains information about a channel that a user subscribed to. This property is only present if the snippet.type is subscription."
-    },
-    "upload": {
-     "$ref": "ActivityContentDetailsUpload",
-     "description": "The upload object contains information about the uploaded video. This property is only present if the snippet.type is upload."
-    }
-   }
-  },
-  "ActivityContentDetailsBulletin": {
-   "id": "ActivityContentDetailsBulletin",
-   "type": "object",
-   "description": "Details about a channel bulletin post.",
-   "properties": {
-    "resourceId": {
-     "$ref": "ResourceId",
-     "description": "The resourceId object contains information that identifies the resource associated with a bulletin post."
-    }
-   }
-  },
-  "ActivityContentDetailsChannelItem": {
-   "id": "ActivityContentDetailsChannelItem",
-   "type": "object",
-   "description": "Details about a resource which was added to a channel.",
-   "properties": {
-    "resourceId": {
-     "$ref": "ResourceId",
-     "description": "The resourceId object contains information that identifies the resource that was added to the channel."
-    }
-   }
-  },
-  "ActivityContentDetailsComment": {
-   "id": "ActivityContentDetailsComment",
-   "type": "object",
-   "description": "Information about a resource that received a comment.",
-   "properties": {
-    "resourceId": {
-     "$ref": "ResourceId",
-     "description": "The resourceId object contains information that identifies the resource associated with the comment."
-    }
-   }
-  },
-  "ActivityContentDetailsFavorite": {
-   "id": "ActivityContentDetailsFavorite",
-   "type": "object",
-   "description": "Information about a video that was marked as a favorite video.",
-   "properties": {
-    "resourceId": {
-     "$ref": "ResourceId",
-     "description": "The resourceId object contains information that identifies the resource that was marked as a favorite."
-    }
-   }
-  },
-  "ActivityContentDetailsLike": {
-   "id": "ActivityContentDetailsLike",
-   "type": "object",
-   "description": "Information about a resource that received a positive (like) rating.",
-   "properties": {
-    "resourceId": {
-     "$ref": "ResourceId",
-     "description": "The resourceId object contains information that identifies the rated resource."
-    }
-   }
-  },
-  "ActivityContentDetailsPlaylistItem": {
-   "id": "ActivityContentDetailsPlaylistItem",
-   "type": "object",
-   "description": "Information about a new playlist item.",
-   "properties": {
-    "playlistId": {
-     "type": "string",
-     "description": "The value that YouTube uses to uniquely identify the playlist."
-    },
-    "playlistItemId": {
-     "type": "string",
-     "description": "ID of the item within the playlist."
-    },
-    "resourceId": {
-     "$ref": "ResourceId",
-     "description": "The resourceId object contains information about the resource that was added to the playlist."
-    }
-   }
-  },
-  "ActivityContentDetailsPromotedItem": {
-   "id": "ActivityContentDetailsPromotedItem",
-   "type": "object",
-   "description": "Details about a resource which is being promoted.",
-   "properties": {
-    "adTag": {
-     "type": "string",
-     "description": "The URL the client should fetch to request a promoted item."
-    },
-    "clickTrackingUrl": {
-     "type": "string",
-     "description": "The URL the client should ping to indicate that the user clicked through on this promoted item."
-    },
-    "creativeViewUrl": {
-     "type": "string",
-     "description": "The URL the client should ping to indicate that the user was shown this promoted item."
-    },
-    "ctaType": {
-     "type": "string",
-     "description": "The type of call-to-action, a message to the user indicating action that can be taken.",
-     "enum": [
-      "unspecified",
-      "visitAdvertiserSite"
-     ],
-     "enumDescriptions": [
-      "",
-      ""
-     ]
-    },
-    "customCtaButtonText": {
-     "type": "string",
-     "description": "The custom call-to-action button text. If specified, it will override the default button text for the cta_type."
-    },
-    "descriptionText": {
-     "type": "string",
-     "description": "The text description to accompany the promoted item."
-    },
-    "destinationUrl": {
-     "type": "string",
-     "description": "The URL the client should direct the user to, if the user chooses to visit the advertiser's website."
-    },
-    "forecastingUrl": {
-     "type": "array",
-     "description": "The list of forecasting URLs. The client should ping all of these URLs when a promoted item is not available, to indicate that a promoted item could have been shown.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "impressionUrl": {
-     "type": "array",
-     "description": "The list of impression URLs. The client should ping all of these URLs to indicate that the user was shown this promoted item.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "videoId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the promoted video."
-    }
-   }
-  },
-  "ActivityContentDetailsRecommendation": {
-   "id": "ActivityContentDetailsRecommendation",
-   "type": "object",
-   "description": "Information that identifies the recommended resource.",
-   "properties": {
-    "reason": {
-     "type": "string",
-     "description": "The reason that the resource is recommended to the user.",
-     "enum": [
-      "unspecified",
-      "videoFavorited",
-      "videoLiked",
-      "videoWatched"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "resourceId": {
-     "$ref": "ResourceId",
-     "description": "The resourceId object contains information that identifies the recommended resource."
-    },
-    "seedResourceId": {
-     "$ref": "ResourceId",
-     "description": "The seedResourceId object contains information about the resource that caused the recommendation."
-    }
-   }
-  },
-  "ActivityContentDetailsSocial": {
-   "id": "ActivityContentDetailsSocial",
-   "type": "object",
-   "description": "Details about a social network post.",
-   "properties": {
-    "author": {
-     "type": "string",
-     "description": "The author of the social network post."
-    },
-    "imageUrl": {
-     "type": "string",
-     "description": "An image of the post's author."
-    },
-    "referenceUrl": {
-     "type": "string",
-     "description": "The URL of the social network post."
-    },
-    "resourceId": {
-     "$ref": "ResourceId",
-     "description": "The resourceId object encapsulates information that identifies the resource associated with a social network post."
-    },
-    "type": {
-     "type": "string",
-     "description": "The name of the social network.",
-     "enum": [
-      "facebook",
-      "googlePlus",
-      "twitter",
-      "unspecified"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "ActivityContentDetailsSubscription": {
-   "id": "ActivityContentDetailsSubscription",
-   "type": "object",
-   "description": "Information about a channel that a user subscribed to.",
-   "properties": {
-    "resourceId": {
-     "$ref": "ResourceId",
-     "description": "The resourceId object contains information that identifies the resource that the user subscribed to."
-    }
-   }
-  },
-  "ActivityContentDetailsUpload": {
-   "id": "ActivityContentDetailsUpload",
-   "type": "object",
-   "description": "Information about the uploaded video.",
-   "properties": {
-    "videoId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the uploaded video."
-    }
-   }
-  },
-  "ActivityListResponse": {
-   "id": "ActivityListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of activities, or events, that match the request criteria.",
-     "items": {
-      "$ref": "Activity"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#activityListResponse\".",
-     "default": "youtube#activityListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "prevPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the previous page in the result set."
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "ActivitySnippet": {
-   "id": "ActivitySnippet",
-   "type": "object",
-   "description": "Basic details about an activity, including title, description, thumbnails, activity type and group.",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the channel associated with the activity."
-    },
-    "channelTitle": {
-     "type": "string",
-     "description": "Channel title for the channel responsible for this activity"
-    },
-    "description": {
-     "type": "string",
-     "description": "The description of the resource primarily associated with the activity.",
-     "annotations": {
-      "required": [
-       "youtube.activities.insert"
-      ]
-     }
-    },
-    "groupId": {
-     "type": "string",
-     "description": "The group ID associated with the activity. A group ID identifies user events that are associated with the same user and resource. For example, if a user rates a video and marks the same video as a favorite, the entries for those events would have the same group ID in the user's activity feed. In your user interface, you can avoid repetition by grouping events with the same groupId value."
-    },
-    "publishedAt": {
-     "type": "string",
-     "description": "The date and time that the video was uploaded. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "thumbnails": {
-     "$ref": "ThumbnailDetails",
-     "description": "A map of thumbnail images associated with the resource that is primarily associated with the activity. For each object in the map, the key is the name of the thumbnail image, and the value is an object that contains other information about the thumbnail."
-    },
-    "title": {
-     "type": "string",
-     "description": "The title of the resource primarily associated with the activity."
-    },
-    "type": {
-     "type": "string",
-     "description": "The type of activity that the resource describes.",
-     "enum": [
-      "bulletin",
-      "channelItem",
-      "comment",
-      "favorite",
-      "like",
-      "playlistItem",
-      "promotedItem",
-      "recommendation",
-      "social",
-      "subscription",
-      "upload"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "Caption": {
-   "id": "Caption",
-   "type": "object",
-   "description": "A caption resource represents a YouTube caption track. A caption track is associated with exactly one YouTube video.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the caption track.",
-     "annotations": {
-      "required": [
-       "youtube.captions.update"
-      ]
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#caption\".",
-     "default": "youtube#caption"
-    },
-    "snippet": {
-     "$ref": "CaptionSnippet",
-     "description": "The snippet object contains basic details about the caption."
-    }
-   }
-  },
-  "CaptionListResponse": {
-   "id": "CaptionListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of captions that match the request criteria.",
-     "items": {
-      "$ref": "Caption"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#captionListResponse\".",
-     "default": "youtube#captionListResponse"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "CaptionSnippet": {
-   "id": "CaptionSnippet",
-   "type": "object",
-   "description": "Basic details about a caption track, such as its language and name.",
-   "properties": {
-    "audioTrackType": {
-     "type": "string",
-     "description": "The type of audio track associated with the caption track.",
-     "enum": [
-      "commentary",
-      "descriptive",
-      "primary",
-      "unknown"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "failureReason": {
-     "type": "string",
-     "description": "The reason that YouTube failed to process the caption track. This property is only present if the state property's value is failed.",
-     "enum": [
-      "processingFailed",
-      "unknownFormat",
-      "unsupportedFormat"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    },
-    "isAutoSynced": {
-     "type": "boolean",
-     "description": "Indicates whether YouTube synchronized the caption track to the audio track in the video. The value will be true if a sync was explicitly requested when the caption track was uploaded. For example, when calling the captions.insert or captions.update methods, you can set the sync parameter to true to instruct YouTube to sync the uploaded track to the video. If the value is false, YouTube uses the time codes in the uploaded caption track to determine when to display captions."
-    },
-    "isCC": {
-     "type": "boolean",
-     "description": "Indicates whether the track contains closed captions for the deaf and hard of hearing. The default value is false."
-    },
-    "isDraft": {
-     "type": "boolean",
-     "description": "Indicates whether the caption track is a draft. If the value is true, then the track is not publicly visible. The default value is false."
-    },
-    "isEasyReader": {
-     "type": "boolean",
-     "description": "Indicates whether caption track is formatted for \"easy reader,\" meaning it is at a third-grade level for language learners. The default value is false."
-    },
-    "isLarge": {
-     "type": "boolean",
-     "description": "Indicates whether the caption track uses large text for the vision-impaired. The default value is false."
-    },
-    "language": {
-     "type": "string",
-     "description": "The language of the caption track. The property value is a BCP-47 language tag.",
-     "annotations": {
-      "required": [
-       "youtube.captions.insert"
-      ]
-     }
-    },
-    "lastUpdated": {
-     "type": "string",
-     "description": "The date and time when the caption track was last updated. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "name": {
-     "type": "string",
-     "description": "The name of the caption track. The name is intended to be visible to the user as an option during playback.",
-     "annotations": {
-      "required": [
-       "youtube.captions.insert"
-      ]
-     }
-    },
-    "status": {
-     "type": "string",
-     "description": "The caption track's status.",
-     "enum": [
-      "failed",
-      "serving",
-      "syncing"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    },
-    "trackKind": {
-     "type": "string",
-     "description": "The caption track's type.",
-     "enum": [
-      "ASR",
-      "forced",
-      "standard"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    },
-    "videoId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the video associated with the caption track.",
-     "annotations": {
-      "required": [
-       "youtube.captions.insert"
-      ]
-     }
-    }
-   }
-  },
-  "CdnSettings": {
-   "id": "CdnSettings",
-   "type": "object",
-   "description": "Brief description of the live stream cdn settings.",
-   "properties": {
-    "format": {
-     "type": "string",
-     "description": "The format of the video stream that you are sending to Youtube.",
-     "annotations": {
-      "required": [
-       "youtube.liveStreams.insert",
-       "youtube.liveStreams.update"
-      ]
-     }
-    },
-    "frameRate": {
-     "type": "string",
-     "description": "The frame rate of the inbound video data.",
-     "enum": [
-      "30fps",
-      "60fps"
-     ],
-     "enumDescriptions": [
-      "",
-      ""
-     ]
-    },
-    "ingestionInfo": {
-     "$ref": "IngestionInfo",
-     "description": "The ingestionInfo object contains information that YouTube provides that you need to transmit your RTMP or HTTP stream to YouTube."
-    },
-    "ingestionType": {
-     "type": "string",
-     "description": "The method or protocol used to transmit the video stream.",
-     "enum": [
-      "dash",
-      "rtmp"
-     ],
-     "enumDescriptions": [
-      "",
-      ""
-     ],
-     "annotations": {
-      "required": [
-       "youtube.liveStreams.insert",
-       "youtube.liveStreams.update"
-      ]
-     }
-    },
-    "resolution": {
-     "type": "string",
-     "description": "The resolution of the inbound video data.",
-     "enum": [
-      "1080p",
-      "1440p",
-      "2160p",
-      "240p",
-      "360p",
-      "480p",
-      "720p"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "Channel": {
-   "id": "Channel",
-   "type": "object",
-   "description": "A channel resource contains information about a YouTube channel.",
-   "properties": {
-    "auditDetails": {
-     "$ref": "ChannelAuditDetails",
-     "description": "The auditionDetails object encapsulates channel data that is relevant for YouTube Partners during the audition process."
-    },
-    "brandingSettings": {
-     "$ref": "ChannelBrandingSettings",
-     "description": "The brandingSettings object encapsulates information about the branding of the channel."
-    },
-    "contentDetails": {
-     "$ref": "ChannelContentDetails",
-     "description": "The contentDetails object encapsulates information about the channel's content."
-    },
-    "contentOwnerDetails": {
-     "$ref": "ChannelContentOwnerDetails",
-     "description": "The contentOwnerDetails object encapsulates channel data that is relevant for YouTube Partners linked with the channel."
-    },
-    "conversionPings": {
-     "$ref": "ChannelConversionPings",
-     "description": "The conversionPings object encapsulates information about conversion pings that need to be respected by the channel."
-    },
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the channel."
-    },
-    "invideoPromotion": {
-     "$ref": "InvideoPromotion",
-     "description": "The invideoPromotion object encapsulates information about promotion campaign associated with the channel."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#channel\".",
-     "default": "youtube#channel"
-    },
-    "localizations": {
-     "type": "object",
-     "description": "Localizations for different languages",
-     "additionalProperties": {
-      "$ref": "ChannelLocalization",
-      "description": "The language tag, using string since map_key require simple types."
-     }
-    },
-    "snippet": {
-     "$ref": "ChannelSnippet",
-     "description": "The snippet object contains basic details about the channel, such as its title, description, and thumbnail images."
-    },
-    "statistics": {
-     "$ref": "ChannelStatistics",
-     "description": "The statistics object encapsulates statistics for the channel."
-    },
-    "status": {
-     "$ref": "ChannelStatus",
-     "description": "The status object encapsulates information about the privacy status of the channel."
-    },
-    "topicDetails": {
-     "$ref": "ChannelTopicDetails",
-     "description": "The topicDetails object encapsulates information about Freebase topics associated with the channel."
-    }
-   }
-  },
-  "ChannelAuditDetails": {
-   "id": "ChannelAuditDetails",
-   "type": "object",
-   "description": "The auditDetails object encapsulates channel data that is relevant for YouTube Partners during the audit process.",
-   "properties": {
-    "communityGuidelinesGoodStanding": {
-     "type": "boolean",
-     "description": "Whether or not the channel respects the community guidelines."
-    },
-    "contentIdClaimsGoodStanding": {
-     "type": "boolean",
-     "description": "Whether or not the channel has any unresolved claims."
-    },
-    "copyrightStrikesGoodStanding": {
-     "type": "boolean",
-     "description": "Whether or not the channel has any copyright strikes."
-    },
-    "overallGoodStanding": {
-     "type": "boolean",
-     "description": "Describes the general state of the channel. This field will always show if there are any issues whatsoever with the channel. Currently this field represents the result of the logical and operation over the community guidelines good standing, the copyright strikes good standing and the content ID claims good standing, but this may change in the future."
-    }
-   }
-  },
-  "ChannelBannerResource": {
-   "id": "ChannelBannerResource",
-   "type": "object",
-   "description": "A channel banner returned as the response to a channel_banner.insert call.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#channelBannerResource\".",
-     "default": "youtube#channelBannerResource"
-    },
-    "url": {
-     "type": "string",
-     "description": "The URL of this banner image."
-    }
-   }
-  },
-  "ChannelBrandingSettings": {
-   "id": "ChannelBrandingSettings",
-   "type": "object",
-   "description": "Branding properties of a YouTube channel.",
-   "properties": {
-    "channel": {
-     "$ref": "ChannelSettings",
-     "description": "Branding properties for the channel view."
-    },
-    "hints": {
-     "type": "array",
-     "description": "Additional experimental branding properties.",
-     "items": {
-      "$ref": "PropertyValue"
-     }
-    },
-    "image": {
-     "$ref": "ImageSettings",
-     "description": "Branding properties for branding images."
-    },
-    "watch": {
-     "$ref": "WatchSettings",
-     "description": "Branding properties for the watch page."
-    }
-   }
-  },
-  "ChannelContentDetails": {
-   "id": "ChannelContentDetails",
-   "type": "object",
-   "description": "Details about the content of a channel.",
-   "properties": {
-    "relatedPlaylists": {
-     "type": "object",
-     "properties": {
-      "favorites": {
-       "type": "string",
-       "description": "The ID of the playlist that contains the channel\"s favorite videos. Use the  playlistItems.insert and  playlistItems.delete to add or remove items from that list."
-      },
-      "likes": {
-       "type": "string",
-       "description": "The ID of the playlist that contains the channel\"s liked videos. Use the   playlistItems.insert and  playlistItems.delete to add or remove items from that list."
-      },
-      "uploads": {
-       "type": "string",
-       "description": "The ID of the playlist that contains the channel\"s uploaded videos. Use the  videos.insert method to upload new videos and the videos.delete method to delete previously uploaded videos."
-      },
-      "watchHistory": {
-       "type": "string",
-       "description": "The ID of the playlist that contains the channel\"s watch history. Use the  playlistItems.insert and  playlistItems.delete to add or remove items from that list."
-      },
-      "watchLater": {
-       "type": "string",
-       "description": "The ID of the playlist that contains the channel\"s watch later playlist. Use the playlistItems.insert and  playlistItems.delete to add or remove items from that list."
-      }
-     }
-    }
-   }
-  },
-  "ChannelContentOwnerDetails": {
-   "id": "ChannelContentOwnerDetails",
-   "type": "object",
-   "description": "The contentOwnerDetails object encapsulates channel data that is relevant for YouTube Partners linked with the channel.",
-   "properties": {
-    "contentOwner": {
-     "type": "string",
-     "description": "The ID of the content owner linked to the channel."
-    },
-    "timeLinked": {
-     "type": "string",
-     "description": "The date and time of when the channel was linked to the content owner. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    }
-   }
-  },
-  "ChannelConversionPing": {
-   "id": "ChannelConversionPing",
-   "type": "object",
-   "description": "Pings that the app shall fire (authenticated by biscotti cookie). Each ping has a context, in which the app must fire the ping, and a url identifying the ping.",
-   "properties": {
-    "context": {
-     "type": "string",
-     "description": "Defines the context of the ping.",
-     "enum": [
-      "cview",
-      "subscribe",
-      "unsubscribe"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    },
-    "conversionUrl": {
-     "type": "string",
-     "description": "The url (without the schema) that the player shall send the ping to. It's at caller's descretion to decide which schema to use (http vs https) Example of a returned url: //googleads.g.doubleclick.net/pagead/ viewthroughconversion/962985656/?data=path%3DtHe_path%3Btype%3D cview%3Butuid%3DGISQtTNGYqaYl4sKxoVvKA&labe=default The caller must append biscotti authentication (ms param in case of mobile, for example) to this ping."
-    }
-   }
-  },
-  "ChannelConversionPings": {
-   "id": "ChannelConversionPings",
-   "type": "object",
-   "description": "The conversionPings object encapsulates information about conversion pings that need to be respected by the channel.",
-   "properties": {
-    "pings": {
-     "type": "array",
-     "description": "Pings that the app shall fire (authenticated by biscotti cookie). Each ping has a context, in which the app must fire the ping, and a url identifying the ping.",
-     "items": {
-      "$ref": "ChannelConversionPing"
-     }
-    }
-   }
-  },
-  "ChannelListResponse": {
-   "id": "ChannelListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of channels that match the request criteria.",
-     "items": {
-      "$ref": "Channel"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#channelListResponse\".",
-     "default": "youtube#channelListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "prevPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the previous page in the result set."
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "ChannelLocalization": {
-   "id": "ChannelLocalization",
-   "type": "object",
-   "description": "Channel localization setting",
-   "properties": {
-    "description": {
-     "type": "string",
-     "description": "The localized strings for channel's description."
-    },
-    "title": {
-     "type": "string",
-     "description": "The localized strings for channel's title."
-    }
-   }
-  },
-  "ChannelProfileDetails": {
-   "id": "ChannelProfileDetails",
-   "type": "object",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "description": "The YouTube channel ID."
-    },
-    "channelUrl": {
-     "type": "string",
-     "description": "The channel's URL."
-    },
-    "displayName": {
-     "type": "string",
-     "description": "The channel's display name."
-    },
-    "profileImageUrl": {
-     "type": "string",
-     "description": "The channels's avatar URL."
-    }
-   }
-  },
-  "ChannelSection": {
-   "id": "ChannelSection",
-   "type": "object",
-   "properties": {
-    "contentDetails": {
-     "$ref": "ChannelSectionContentDetails",
-     "description": "The contentDetails object contains details about the channel section content, such as a list of playlists or channels featured in the section."
-    },
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the channel section."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#channelSection\".",
-     "default": "youtube#channelSection"
-    },
-    "localizations": {
-     "type": "object",
-     "description": "Localizations for different languages",
-     "additionalProperties": {
-      "$ref": "ChannelSectionLocalization",
-      "description": "The language tag, using string since map_key require simple types."
-     }
-    },
-    "snippet": {
-     "$ref": "ChannelSectionSnippet",
-     "description": "The snippet object contains basic details about the channel section, such as its type, style and title."
-    },
-    "targeting": {
-     "$ref": "ChannelSectionTargeting",
-     "description": "The targeting object contains basic targeting settings about the channel section."
-    }
-   }
-  },
-  "ChannelSectionContentDetails": {
-   "id": "ChannelSectionContentDetails",
-   "type": "object",
-   "description": "Details about a channelsection, including playlists and channels.",
-   "properties": {
-    "channels": {
-     "type": "array",
-     "description": "The channel ids for type multiple_channels.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "playlists": {
-     "type": "array",
-     "description": "The playlist ids for type single_playlist and multiple_playlists. For singlePlaylist, only one playlistId is allowed.",
-     "items": {
-      "type": "string"
-     }
-    }
-   }
-  },
-  "ChannelSectionListResponse": {
-   "id": "ChannelSectionListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of ChannelSections that match the request criteria.",
-     "items": {
-      "$ref": "ChannelSection"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#channelSectionListResponse\".",
-     "default": "youtube#channelSectionListResponse"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "ChannelSectionLocalization": {
-   "id": "ChannelSectionLocalization",
-   "type": "object",
-   "description": "ChannelSection localization setting",
-   "properties": {
-    "title": {
-     "type": "string",
-     "description": "The localized strings for channel section's title."
-    }
-   }
-  },
-  "ChannelSectionSnippet": {
-   "id": "ChannelSectionSnippet",
-   "type": "object",
-   "description": "Basic details about a channel section, including title, style and position.",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the channel that published the channel section."
-    },
-    "defaultLanguage": {
-     "type": "string",
-     "description": "The language of the channel section's default title and description."
-    },
-    "localized": {
-     "$ref": "ChannelSectionLocalization",
-     "description": "Localized title, read-only."
-    },
-    "position": {
-     "type": "integer",
-     "description": "The position of the channel section in the channel.",
-     "format": "uint32"
-    },
-    "style": {
-     "type": "string",
-     "description": "The style of the channel section.",
-     "enum": [
-      "channelsectionStyleUndefined",
-      "horizontalRow",
-      "verticalList"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    },
-    "title": {
-     "type": "string",
-     "description": "The channel section's title for multiple_playlists and multiple_channels."
-    },
-    "type": {
-     "type": "string",
-     "description": "The type of the channel section.",
-     "enum": [
-      "allPlaylists",
-      "channelsectionTypeUndefined",
-      "completedEvents",
-      "likedPlaylists",
-      "likes",
-      "liveEvents",
-      "multipleChannels",
-      "multiplePlaylists",
-      "popularUploads",
-      "postedPlaylists",
-      "postedVideos",
-      "recentActivity",
-      "recentPosts",
-      "recentUploads",
-      "singlePlaylist",
-      "subscriptions",
-      "upcomingEvents"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "ChannelSectionTargeting": {
-   "id": "ChannelSectionTargeting",
-   "type": "object",
-   "description": "ChannelSection targeting setting.",
-   "properties": {
-    "countries": {
-     "type": "array",
-     "description": "The country the channel section is targeting.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "languages": {
-     "type": "array",
-     "description": "The language the channel section is targeting.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "regions": {
-     "type": "array",
-     "description": "The region the channel section is targeting.",
-     "items": {
-      "type": "string"
-     }
-    }
-   }
-  },
-  "ChannelSettings": {
-   "id": "ChannelSettings",
-   "type": "object",
-   "description": "Branding properties for the channel view.",
-   "properties": {
-    "country": {
-     "type": "string",
-     "description": "The country of the channel."
-    },
-    "defaultLanguage": {
-     "type": "string"
-    },
-    "defaultTab": {
-     "type": "string",
-     "description": "Which content tab users should see when viewing the channel."
-    },
-    "description": {
-     "type": "string",
-     "description": "Specifies the channel description."
-    },
-    "featuredChannelsTitle": {
-     "type": "string",
-     "description": "Title for the featured channels tab."
-    },
-    "featuredChannelsUrls": {
-     "type": "array",
-     "description": "The list of featured channels.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "keywords": {
-     "type": "string",
-     "description": "Lists keywords associated with the channel, comma-separated."
-    },
-    "moderateComments": {
-     "type": "boolean",
-     "description": "Whether user-submitted comments left on the channel page need to be approved by the channel owner to be publicly visible."
-    },
-    "profileColor": {
-     "type": "string",
-     "description": "A prominent color that can be rendered on this channel page."
-    },
-    "showBrowseView": {
-     "type": "boolean",
-     "description": "Whether the tab to browse the videos should be displayed."
-    },
-    "showRelatedChannels": {
-     "type": "boolean",
-     "description": "Whether related channels should be proposed."
-    },
-    "title": {
-     "type": "string",
-     "description": "Specifies the channel title."
-    },
-    "trackingAnalyticsAccountId": {
-     "type": "string",
-     "description": "The ID for a Google Analytics account to track and measure traffic to the channels."
-    },
-    "unsubscribedTrailer": {
-     "type": "string",
-     "description": "The trailer of the channel, for users that are not subscribers."
-    }
-   }
-  },
-  "ChannelSnippet": {
-   "id": "ChannelSnippet",
-   "type": "object",
-   "description": "Basic details about a channel, including title, description and thumbnails. Next available id: 15.",
-   "properties": {
-    "country": {
-     "type": "string",
-     "description": "The country of the channel."
-    },
-    "customUrl": {
-     "type": "string",
-     "description": "The custom url of the channel."
-    },
-    "defaultLanguage": {
-     "type": "string",
-     "description": "The language of the channel's default title and description."
-    },
-    "description": {
-     "type": "string",
-     "description": "The description of the channel."
-    },
-    "localized": {
-     "$ref": "ChannelLocalization",
-     "description": "Localized title and description, read-only."
-    },
-    "publishedAt": {
-     "type": "string",
-     "description": "The date and time that the channel was created. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "thumbnails": {
-     "$ref": "ThumbnailDetails",
-     "description": "A map of thumbnail images associated with the channel. For each object in the map, the key is the name of the thumbnail image, and the value is an object that contains other information about the thumbnail."
-    },
-    "title": {
-     "type": "string",
-     "description": "The channel's title."
-    }
-   }
-  },
-  "ChannelStatistics": {
-   "id": "ChannelStatistics",
-   "type": "object",
-   "description": "Statistics about a channel: number of subscribers, number of videos in the channel, etc.",
-   "properties": {
-    "commentCount": {
-     "type": "string",
-     "description": "The number of comments for the channel.",
-     "format": "uint64"
-    },
-    "hiddenSubscriberCount": {
-     "type": "boolean",
-     "description": "Whether or not the number of subscribers is shown for this user."
-    },
-    "subscriberCount": {
-     "type": "string",
-     "description": "The number of subscribers that the channel has.",
-     "format": "uint64"
-    },
-    "videoCount": {
-     "type": "string",
-     "description": "The number of videos uploaded to the channel.",
-     "format": "uint64"
-    },
-    "viewCount": {
-     "type": "string",
-     "description": "The number of times the channel has been viewed.",
-     "format": "uint64"
-    }
-   }
-  },
-  "ChannelStatus": {
-   "id": "ChannelStatus",
-   "type": "object",
-   "description": "JSON template for the status part of a channel.",
-   "properties": {
-    "isLinked": {
-     "type": "boolean",
-     "description": "If true, then the user is linked to either a YouTube username or G+ account. Otherwise, the user doesn't have a public YouTube identity."
-    },
-    "longUploadsStatus": {
-     "type": "string",
-     "description": "The long uploads status of this channel. See",
-     "enum": [
-      "allowed",
-      "disallowed",
-      "eligible",
-      "longUploadsUnspecified"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "privacyStatus": {
-     "type": "string",
-     "description": "Privacy status of the channel.",
-     "enum": [
-      "private",
-      "public",
-      "unlisted"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "ChannelTopicDetails": {
-   "id": "ChannelTopicDetails",
-   "type": "object",
-   "description": "Freebase topic information related to the channel.",
-   "properties": {
-    "topicCategories": {
-     "type": "array",
-     "description": "A list of Wikipedia URLs that describe the channel's content.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "topicIds": {
-     "type": "array",
-     "description": "A list of Freebase topic IDs associated with the channel. You can retrieve information about each topic using the Freebase Topic API.",
-     "items": {
-      "type": "string"
-     }
-    }
-   }
-  },
-  "Comment": {
-   "id": "Comment",
-   "type": "object",
-   "description": "A comment represents a single YouTube comment.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the comment."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#comment\".",
-     "default": "youtube#comment"
-    },
-    "snippet": {
-     "$ref": "CommentSnippet",
-     "description": "The snippet object contains basic details about the comment."
-    }
-   }
-  },
-  "CommentListResponse": {
-   "id": "CommentListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of comments that match the request criteria.",
-     "items": {
-      "$ref": "Comment"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#commentListResponse\".",
-     "default": "youtube#commentListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "CommentSnippet": {
-   "id": "CommentSnippet",
-   "type": "object",
-   "description": "Basic details about a comment, such as its author and text.",
-   "properties": {
-    "authorChannelId": {
-     "type": "any",
-     "description": "The id of the author's YouTube channel, if any."
-    },
-    "authorChannelUrl": {
-     "type": "string",
-     "description": "Link to the author's YouTube channel, if any."
-    },
-    "authorDisplayName": {
-     "type": "string",
-     "description": "The name of the user who posted the comment."
-    },
-    "authorProfileImageUrl": {
-     "type": "string",
-     "description": "The URL for the avatar of the user who posted the comment."
-    },
-    "canRate": {
-     "type": "boolean",
-     "description": "Whether the current viewer can rate this comment."
-    },
-    "channelId": {
-     "type": "string",
-     "description": "The id of the corresponding YouTube channel. In case of a channel comment this is the channel the comment refers to. In case of a video comment it's the video's channel."
-    },
-    "likeCount": {
-     "type": "integer",
-     "description": "The total number of likes this comment has received.",
-     "format": "uint32"
-    },
-    "moderationStatus": {
-     "type": "string",
-     "description": "The comment's moderation status. Will not be set if the comments were requested through the id filter.",
-     "enum": [
-      "heldForReview",
-      "likelySpam",
-      "published",
-      "rejected"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "parentId": {
-     "type": "string",
-     "description": "The unique id of the parent comment, only set for replies."
-    },
-    "publishedAt": {
-     "type": "string",
-     "description": "The date and time when the comment was orignally published. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "textDisplay": {
-     "type": "string",
-     "description": "The comment's text. The format is either plain text or HTML dependent on what has been requested. Even the plain text representation may differ from the text originally posted in that it may replace video links with video titles etc."
-    },
-    "textOriginal": {
-     "type": "string",
-     "description": "The comment's original raw text as initially posted or last updated. The original text will only be returned if it is accessible to the viewer, which is only guaranteed if the viewer is the comment's author."
-    },
-    "updatedAt": {
-     "type": "string",
-     "description": "The date and time when was last updated . The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "videoId": {
-     "type": "string",
-     "description": "The ID of the video the comment refers to, if any."
-    },
-    "viewerRating": {
-     "type": "string",
-     "description": "The rating the viewer has given to this comment. For the time being this will never return RATE_TYPE_DISLIKE and instead return RATE_TYPE_NONE. This may change in the future.",
-     "enum": [
-      "dislike",
-      "like",
-      "none",
-      "unspecified"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "CommentThread": {
-   "id": "CommentThread",
-   "type": "object",
-   "description": "A comment thread represents information that applies to a top level comment and all its replies. It can also include the top level comment itself and some of the replies.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the comment thread."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#commentThread\".",
-     "default": "youtube#commentThread"
-    },
-    "replies": {
-     "$ref": "CommentThreadReplies",
-     "description": "The replies object contains a limited number of replies (if any) to the top level comment found in the snippet."
-    },
-    "snippet": {
-     "$ref": "CommentThreadSnippet",
-     "description": "The snippet object contains basic details about the comment thread and also the top level comment."
-    }
-   }
-  },
-  "CommentThreadListResponse": {
-   "id": "CommentThreadListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of comment threads that match the request criteria.",
-     "items": {
-      "$ref": "CommentThread"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#commentThreadListResponse\".",
-     "default": "youtube#commentThreadListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "CommentThreadReplies": {
-   "id": "CommentThreadReplies",
-   "type": "object",
-   "description": "Comments written in (direct or indirect) reply to the top level comment.",
-   "properties": {
-    "comments": {
-     "type": "array",
-     "description": "A limited number of replies. Unless the number of replies returned equals total_reply_count in the snippet the returned replies are only a subset of the total number of replies.",
-     "items": {
-      "$ref": "Comment"
-     }
-    }
-   }
-  },
-  "CommentThreadSnippet": {
-   "id": "CommentThreadSnippet",
-   "type": "object",
-   "description": "Basic details about a comment thread.",
-   "properties": {
-    "canReply": {
-     "type": "boolean",
-     "description": "Whether the current viewer of the thread can reply to it. This is viewer specific - other viewers may see a different value for this field."
-    },
-    "channelId": {
-     "type": "string",
-     "description": "The YouTube channel the comments in the thread refer to or the channel with the video the comments refer to. If video_id isn't set the comments refer to the channel itself."
-    },
-    "isPublic": {
-     "type": "boolean",
-     "description": "Whether the thread (and therefore all its comments) is visible to all YouTube users."
-    },
-    "topLevelComment": {
-     "$ref": "Comment",
-     "description": "The top level comment of this thread."
-    },
-    "totalReplyCount": {
-     "type": "integer",
-     "description": "The total number of replies (not including the top level comment).",
-     "format": "uint32"
-    },
-    "videoId": {
-     "type": "string",
-     "description": "The ID of the video the comments refer to, if any. No video_id implies a channel discussion comment."
-    }
-   }
-  },
-  "ContentRating": {
-   "id": "ContentRating",
-   "type": "object",
-   "description": "Ratings schemes. The country-specific ratings are mostly for movies and shows. NEXT_ID: 69",
-   "properties": {
-    "acbRating": {
-     "type": "string",
-     "description": "The video's Australian Classification Board (ACB) or Australian Communications and Media Authority (ACMA) rating. ACMA ratings are used to classify children's television programming.",
-     "enum": [
-      "acbC",
-      "acbE",
-      "acbG",
-      "acbM",
-      "acbMa15plus",
-      "acbP",
-      "acbPg",
-      "acbR18plus",
-      "acbUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "agcomRating": {
-     "type": "string",
-     "description": "The video's rating from Italy's Autorità per le Garanzie nelle Comunicazioni (AGCOM).",
-     "enum": [
-      "agcomT",
-      "agcomUnrated",
-      "agcomVm14",
-      "agcomVm18"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "anatelRating": {
-     "type": "string",
-     "description": "The video's Anatel (Asociación Nacional de Televisión) rating for Chilean television.",
-     "enum": [
-      "anatelA",
-      "anatelF",
-      "anatelI",
-      "anatelI10",
-      "anatelI12",
-      "anatelI7",
-      "anatelR",
-      "anatelUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "bbfcRating": {
-     "type": "string",
-     "description": "The video's British Board of Film Classification (BBFC) rating.",
-     "enum": [
-      "bbfc12",
-      "bbfc12a",
-      "bbfc15",
-      "bbfc18",
-      "bbfcPg",
-      "bbfcR18",
-      "bbfcU",
-      "bbfcUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "bfvcRating": {
-     "type": "string",
-     "description": "The video's rating from Thailand's Board of Film and Video Censors.",
-     "enum": [
-      "bfvc13",
-      "bfvc15",
-      "bfvc18",
-      "bfvc20",
-      "bfvcB",
-      "bfvcE",
-      "bfvcG",
-      "bfvcUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "bmukkRating": {
-     "type": "string",
-     "description": "The video's rating from the Austrian Board of Media Classification (Bundesministerium für Unterricht, Kunst und Kultur).",
-     "enum": [
-      "bmukk10",
-      "bmukk12",
-      "bmukk14",
-      "bmukk16",
-      "bmukk6",
-      "bmukk8",
-      "bmukkAa",
-      "bmukkUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "catvRating": {
-     "type": "string",
-     "description": "Rating system for Canadian TV - Canadian TV Classification System The video's rating from the Canadian Radio-Television and Telecommunications Commission (CRTC) for Canadian English-language broadcasts. For more information, see the Canadian Broadcast Standards Council website.",
-     "enum": [
-      "catv14plus",
-      "catv18plus",
-      "catvC",
-      "catvC8",
-      "catvG",
-      "catvPg",
-      "catvUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "catvfrRating": {
-     "type": "string",
-     "description": "The video's rating from the Canadian Radio-Television and Telecommunications Commission (CRTC) for Canadian French-language broadcasts. For more information, see the Canadian Broadcast Standards Council website.",
-     "enum": [
-      "catvfr13plus",
-      "catvfr16plus",
-      "catvfr18plus",
-      "catvfr8plus",
-      "catvfrG",
-      "catvfrUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "cbfcRating": {
-     "type": "string",
-     "description": "The video's Central Board of Film Certification (CBFC - India) rating.",
-     "enum": [
-      "cbfcA",
-      "cbfcS",
-      "cbfcU",
-      "cbfcUA",
-      "cbfcUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "cccRating": {
-     "type": "string",
-     "description": "The video's Consejo de Calificación Cinematográfica (Chile) rating.",
-     "enum": [
-      "ccc14",
-      "ccc18",
-      "ccc18s",
-      "ccc18v",
-      "ccc6",
-      "cccTe",
-      "cccUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "cceRating": {
-     "type": "string",
-     "description": "The video's rating from Portugal's Comissão de Classificação de Espect´culos.",
-     "enum": [
-      "cceM12",
-      "cceM14",
-      "cceM16",
-      "cceM18",
-      "cceM4",
-      "cceM6",
-      "cceUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "chfilmRating": {
-     "type": "string",
-     "description": "The video's rating in Switzerland.",
-     "enum": [
-      "chfilm0",
-      "chfilm12",
-      "chfilm16",
-      "chfilm18",
-      "chfilm6",
-      "chfilmUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "chvrsRating": {
-     "type": "string",
-     "description": "The video's Canadian Home Video Rating System (CHVRS) rating.",
-     "enum": [
-      "chvrs14a",
-      "chvrs18a",
-      "chvrsE",
-      "chvrsG",
-      "chvrsPg",
-      "chvrsR",
-      "chvrsUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "cicfRating": {
-     "type": "string",
-     "description": "The video's rating from the Commission de Contrôle des Films (Belgium).",
-     "enum": [
-      "cicfE",
-      "cicfKntEna",
-      "cicfKtEa",
-      "cicfUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "cnaRating": {
-     "type": "string",
-     "description": "The video's rating from Romania's CONSILIUL NATIONAL AL AUDIOVIZUALULUI (CNA).",
-     "enum": [
-      "cna12",
-      "cna15",
-      "cna18",
-      "cna18plus",
-      "cnaAp",
-      "cnaUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "cncRating": {
-     "type": "string",
-     "description": "Rating system in France - Commission de classification cinematographique",
-     "enum": [
-      "cnc10",
-      "cnc12",
-      "cnc16",
-      "cnc18",
-      "cncE",
-      "cncT",
-      "cncUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "csaRating": {
-     "type": "string",
-     "description": "The video's rating from France's Conseil supérieur de l?audiovisuel, which rates broadcast content.",
-     "enum": [
-      "csa10",
-      "csa12",
-      "csa16",
-      "csa18",
-      "csaInterdiction",
-      "csaT",
-      "csaUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "cscfRating": {
-     "type": "string",
-     "description": "The video's rating from Luxembourg's Commission de surveillance de la classification des films (CSCF).",
-     "enum": [
-      "cscf12",
-      "cscf16",
-      "cscf18",
-      "cscf6",
-      "cscf9",
-      "cscfA",
-      "cscfAl",
-      "cscfUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "czfilmRating": {
-     "type": "string",
-     "description": "The video's rating in the Czech Republic.",
-     "enum": [
-      "czfilm12",
-      "czfilm14",
-      "czfilm18",
-      "czfilmU",
-      "czfilmUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "djctqRating": {
-     "type": "string",
-     "description": "The video's Departamento de Justiça, Classificação, Qualificação e Títulos (DJCQT - Brazil) rating.",
-     "enum": [
-      "djctq10",
-      "djctq12",
-      "djctq14",
-      "djctq16",
-      "djctq18",
-      "djctqL",
-      "djctqUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "djctqRatingReasons": {
-     "type": "array",
-     "description": "Reasons that explain why the video received its DJCQT (Brazil) rating.",
-     "items": {
-      "type": "string",
-      "enum": [
-       "djctqCriminalActs",
-       "djctqDrugs",
-       "djctqExplicitSex",
-       "djctqExtremeViolence",
-       "djctqIllegalDrugs",
-       "djctqImpactingContent",
-       "djctqInappropriateLanguage",
-       "djctqLegalDrugs",
-       "djctqNudity",
-       "djctqSex",
-       "djctqSexualContent",
-       "djctqViolence"
-      ],
-      "enumDescriptions": [
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       ""
-      ]
-     }
-    },
-    "ecbmctRating": {
-     "type": "string",
-     "description": "Rating system in Turkey - Evaluation and Classification Board of the Ministry of Culture and Tourism",
-     "enum": [
-      "ecbmct13a",
-      "ecbmct13plus",
-      "ecbmct15a",
-      "ecbmct15plus",
-      "ecbmct18plus",
-      "ecbmct7a",
-      "ecbmct7plus",
-      "ecbmctG",
-      "ecbmctUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "eefilmRating": {
-     "type": "string",
-     "description": "The video's rating in Estonia.",
-     "enum": [
-      "eefilmK12",
-      "eefilmK14",
-      "eefilmK16",
-      "eefilmK6",
-      "eefilmL",
-      "eefilmMs12",
-      "eefilmMs6",
-      "eefilmPere",
-      "eefilmUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "egfilmRating": {
-     "type": "string",
-     "description": "The video's rating in Egypt.",
-     "enum": [
-      "egfilm18",
-      "egfilmBn",
-      "egfilmGn",
-      "egfilmUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "eirinRating": {
-     "type": "string",
-     "description": "The video's Eirin (映倫) rating. Eirin is the Japanese rating system.",
-     "enum": [
-      "eirinG",
-      "eirinPg12",
-      "eirinR15plus",
-      "eirinR18plus",
-      "eirinUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "fcbmRating": {
-     "type": "string",
-     "description": "The video's rating from Malaysia's Film Censorship Board.",
-     "enum": [
-      "fcbm18",
-      "fcbm18pa",
-      "fcbm18pl",
-      "fcbm18sg",
-      "fcbm18sx",
-      "fcbmP13",
-      "fcbmPg13",
-      "fcbmU",
-      "fcbmUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "fcoRating": {
-     "type": "string",
-     "description": "The video's rating from Hong Kong's Office for Film, Newspaper and Article Administration.",
-     "enum": [
-      "fcoI",
-      "fcoIi",
-      "fcoIia",
-      "fcoIib",
-      "fcoIii",
-      "fcoUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "fmocRating": {
-     "type": "string",
-     "description": "This property has been deprecated. Use the contentDetails.contentRating.cncRating instead.",
-     "enum": [
-      "fmoc10",
-      "fmoc12",
-      "fmoc16",
-      "fmoc18",
-      "fmocE",
-      "fmocU",
-      "fmocUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "fpbRating": {
-     "type": "string",
-     "description": "The video's rating from South Africa's Film and Publication Board.",
-     "enum": [
-      "fpb10",
-      "fpb1012Pg",
-      "fpb13",
-      "fpb16",
-      "fpb18",
-      "fpb79Pg",
-      "fpbA",
-      "fpbPg",
-      "fpbUnrated",
-      "fpbX18",
-      "fpbXx"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "fpbRatingReasons": {
-     "type": "array",
-     "description": "Reasons that explain why the video received its FPB (South Africa) rating.",
-     "items": {
-      "type": "string",
-      "enum": [
-       "fpbBlasphemy",
-       "fpbCriminalTechniques",
-       "fpbDrugs",
-       "fpbHorror",
-       "fpbImitativeActsTechniques",
-       "fpbLanguage",
-       "fpbNudity",
-       "fpbPrejudice",
-       "fpbSex",
-       "fpbSexualViolence",
-       "fpbViolence"
-      ],
-      "enumDescriptions": [
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       ""
-      ]
-     }
-    },
-    "fskRating": {
-     "type": "string",
-     "description": "The video's Freiwillige Selbstkontrolle der Filmwirtschaft (FSK - Germany) rating.",
-     "enum": [
-      "fsk0",
-      "fsk12",
-      "fsk16",
-      "fsk18",
-      "fsk6",
-      "fskUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "grfilmRating": {
-     "type": "string",
-     "description": "The video's rating in Greece.",
-     "enum": [
-      "grfilmE",
-      "grfilmK",
-      "grfilmK12",
-      "grfilmK13",
-      "grfilmK15",
-      "grfilmK17",
-      "grfilmK18",
-      "grfilmUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "icaaRating": {
-     "type": "string",
-     "description": "The video's Instituto de la Cinematografía y de las Artes Audiovisuales (ICAA - Spain) rating.",
-     "enum": [
-      "icaa12",
-      "icaa13",
-      "icaa16",
-      "icaa18",
-      "icaa7",
-      "icaaApta",
-      "icaaUnrated",
-      "icaaX"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "ifcoRating": {
-     "type": "string",
-     "description": "The video's Irish Film Classification Office (IFCO - Ireland) rating. See the IFCO website for more information.",
-     "enum": [
-      "ifco12",
-      "ifco12a",
-      "ifco15",
-      "ifco15a",
-      "ifco16",
-      "ifco18",
-      "ifcoG",
-      "ifcoPg",
-      "ifcoUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "ilfilmRating": {
-     "type": "string",
-     "description": "The video's rating in Israel.",
-     "enum": [
-      "ilfilm12",
-      "ilfilm16",
-      "ilfilm18",
-      "ilfilmAa",
-      "ilfilmUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "incaaRating": {
-     "type": "string",
-     "description": "The video's INCAA (Instituto Nacional de Cine y Artes Audiovisuales - Argentina) rating.",
-     "enum": [
-      "incaaAtp",
-      "incaaC",
-      "incaaSam13",
-      "incaaSam16",
-      "incaaSam18",
-      "incaaUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "kfcbRating": {
-     "type": "string",
-     "description": "The video's rating from the Kenya Film Classification Board.",
-     "enum": [
-      "kfcb16plus",
-      "kfcbG",
-      "kfcbPg",
-      "kfcbR",
-      "kfcbUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "kijkwijzerRating": {
-     "type": "string",
-     "description": "voor de Classificatie van Audiovisuele Media (Netherlands).",
-     "enum": [
-      "kijkwijzer12",
-      "kijkwijzer16",
-      "kijkwijzer18",
-      "kijkwijzer6",
-      "kijkwijzer9",
-      "kijkwijzerAl",
-      "kijkwijzerUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "kmrbRating": {
-     "type": "string",
-     "description": "The video's Korea Media Rating Board (영상물등급위원회) rating. The KMRB rates videos in South Korea.",
-     "enum": [
-      "kmrb12plus",
-      "kmrb15plus",
-      "kmrbAll",
-      "kmrbR",
-      "kmrbTeenr",
-      "kmrbUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "lsfRating": {
-     "type": "string",
-     "description": "The video's rating from Indonesia's Lembaga Sensor Film.",
-     "enum": [
-      "lsf13",
-      "lsf17",
-      "lsf21",
-      "lsfA",
-      "lsfBo",
-      "lsfD",
-      "lsfR",
-      "lsfSu",
-      "lsfUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "mccaaRating": {
-     "type": "string",
-     "description": "The video's rating from Malta's Film Age-Classification Board.",
-     "enum": [
-      "mccaa12",
-      "mccaa12a",
-      "mccaa14",
-      "mccaa15",
-      "mccaa16",
-      "mccaa18",
-      "mccaaPg",
-      "mccaaU",
-      "mccaaUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "mccypRating": {
-     "type": "string",
-     "description": "The video's rating from the Danish Film Institute's (Det Danske Filminstitut) Media Council for Children and Young People.",
-     "enum": [
-      "mccyp11",
-      "mccyp15",
-      "mccyp7",
-      "mccypA",
-      "mccypUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "mcstRating": {
-     "type": "string",
-     "description": "The video's rating system for Vietnam - MCST",
-     "enum": [
-      "mcst0",
-      "mcst16plus",
-      "mcstC13",
-      "mcstC16",
-      "mcstC18",
-      "mcstGPg",
-      "mcstP",
-      "mcstUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "mdaRating": {
-     "type": "string",
-     "description": "The video's rating from Singapore's Media Development Authority (MDA) and, specifically, it's Board of Film Censors (BFC).",
-     "enum": [
-      "mdaG",
-      "mdaM18",
-      "mdaNc16",
-      "mdaPg",
-      "mdaPg13",
-      "mdaR21",
-      "mdaUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "medietilsynetRating": {
-     "type": "string",
-     "description": "The video's rating from Medietilsynet, the Norwegian Media Authority.",
-     "enum": [
-      "medietilsynet11",
-      "medietilsynet12",
-      "medietilsynet15",
-      "medietilsynet18",
-      "medietilsynet6",
-      "medietilsynet7",
-      "medietilsynet9",
-      "medietilsynetA",
-      "medietilsynetUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "mekuRating": {
-     "type": "string",
-     "description": "The video's rating from Finland's Kansallinen Audiovisuaalinen Instituutti (National Audiovisual Institute).",
-     "enum": [
-      "meku12",
-      "meku16",
-      "meku18",
-      "meku7",
-      "mekuS",
-      "mekuUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "mibacRating": {
-     "type": "string",
-     "description": "The video's rating from the Ministero dei Beni e delle Attività Culturali e del Turismo (Italy).",
-     "enum": [
-      "mibacT",
-      "mibacUnrated",
-      "mibacVap",
-      "mibacVm12",
-      "mibacVm14",
-      "mibacVm18"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "mocRating": {
-     "type": "string",
-     "description": "The video's Ministerio de Cultura (Colombia) rating.",
-     "enum": [
-      "moc12",
-      "moc15",
-      "moc18",
-      "moc7",
-      "mocBanned",
-      "mocE",
-      "mocT",
-      "mocUnrated",
-      "mocX"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "moctwRating": {
-     "type": "string",
-     "description": "The video's rating from Taiwan's Ministry of Culture (文化部).",
-     "enum": [
-      "moctwG",
-      "moctwP",
-      "moctwPg",
-      "moctwR",
-      "moctwR12",
-      "moctwR15",
-      "moctwUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "mpaaRating": {
-     "type": "string",
-     "description": "The video's Motion Picture Association of America (MPAA) rating.",
-     "enum": [
-      "mpaaG",
-      "mpaaNc17",
-      "mpaaPg",
-      "mpaaPg13",
-      "mpaaR",
-      "mpaaUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "mtrcbRating": {
-     "type": "string",
-     "description": "The video's rating from the Movie and Television Review and Classification Board (Philippines).",
-     "enum": [
-      "mtrcbG",
-      "mtrcbPg",
-      "mtrcbR13",
-      "mtrcbR16",
-      "mtrcbR18",
-      "mtrcbUnrated",
-      "mtrcbX"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "nbcRating": {
-     "type": "string",
-     "description": "The video's rating from the Maldives National Bureau of Classification.",
-     "enum": [
-      "nbc12plus",
-      "nbc15plus",
-      "nbc18plus",
-      "nbc18plusr",
-      "nbcG",
-      "nbcPg",
-      "nbcPu",
-      "nbcUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "nbcplRating": {
-     "type": "string",
-     "description": "The video's rating in Poland.",
-     "enum": [
-      "nbcpl18plus",
-      "nbcplI",
-      "nbcplIi",
-      "nbcplIii",
-      "nbcplIv",
-      "nbcplUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "nfrcRating": {
-     "type": "string",
-     "description": "The video's rating from the Bulgarian National Film Center.",
-     "enum": [
-      "nfrcA",
-      "nfrcB",
-      "nfrcC",
-      "nfrcD",
-      "nfrcUnrated",
-      "nfrcX"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "nfvcbRating": {
-     "type": "string",
-     "description": "The video's rating from Nigeria's National Film and Video Censors Board.",
-     "enum": [
-      "nfvcb12",
-      "nfvcb12a",
-      "nfvcb15",
-      "nfvcb18",
-      "nfvcbG",
-      "nfvcbPg",
-      "nfvcbRe",
-      "nfvcbUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "nkclvRating": {
-     "type": "string",
-     "description": "The video's rating from the Nacionãlais Kino centrs (National Film Centre of Latvia).",
-     "enum": [
-      "nkclv12plus",
-      "nkclv18plus",
-      "nkclv7plus",
-      "nkclvU",
-      "nkclvUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "oflcRating": {
-     "type": "string",
-     "description": "The video's Office of Film and Literature Classification (OFLC - New Zealand) rating.",
-     "enum": [
-      "oflcG",
-      "oflcM",
-      "oflcPg",
-      "oflcR13",
-      "oflcR15",
-      "oflcR16",
-      "oflcR18",
-      "oflcRp13",
-      "oflcRp16",
-      "oflcUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "pefilmRating": {
-     "type": "string",
-     "description": "The video's rating in Peru.",
-     "enum": [
-      "pefilm14",
-      "pefilm18",
-      "pefilmPg",
-      "pefilmPt",
-      "pefilmUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "rcnofRating": {
-     "type": "string",
-     "description": "The video's rating from the Hungarian Nemzeti Filmiroda, the Rating Committee of the National Office of Film.",
-     "enum": [
-      "rcnofI",
-      "rcnofIi",
-      "rcnofIii",
-      "rcnofIv",
-      "rcnofUnrated",
-      "rcnofV",
-      "rcnofVi"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "resorteviolenciaRating": {
-     "type": "string",
-     "description": "The video's rating in Venezuela.",
-     "enum": [
-      "resorteviolenciaA",
-      "resorteviolenciaB",
-      "resorteviolenciaC",
-      "resorteviolenciaD",
-      "resorteviolenciaE",
-      "resorteviolenciaUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "rtcRating": {
-     "type": "string",
-     "description": "The video's General Directorate of Radio, Television and Cinematography (Mexico) rating.",
-     "enum": [
-      "rtcA",
-      "rtcAa",
-      "rtcB",
-      "rtcB15",
-      "rtcC",
-      "rtcD",
-      "rtcUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "rteRating": {
-     "type": "string",
-     "description": "The video's rating from Ireland's Raidió Teilifís Éireann.",
-     "enum": [
-      "rteCh",
-      "rteGa",
-      "rteMa",
-      "rtePs",
-      "rteUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "russiaRating": {
-     "type": "string",
-     "description": "The video's National Film Registry of the Russian Federation (MKRF - Russia) rating.",
-     "enum": [
-      "russia0",
-      "russia12",
-      "russia16",
-      "russia18",
-      "russia6",
-      "russiaUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "skfilmRating": {
-     "type": "string",
-     "description": "The video's rating in Slovakia.",
-     "enum": [
-      "skfilmG",
-      "skfilmP2",
-      "skfilmP5",
-      "skfilmP8",
-      "skfilmUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "smaisRating": {
-     "type": "string",
-     "description": "The video's rating in Iceland.",
-     "enum": [
-      "smais12",
-      "smais14",
-      "smais16",
-      "smais18",
-      "smais7",
-      "smaisL",
-      "smaisUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "smsaRating": {
-     "type": "string",
-     "description": "The video's rating from Statens medieråd (Sweden's National Media Council).",
-     "enum": [
-      "smsa11",
-      "smsa15",
-      "smsa7",
-      "smsaA",
-      "smsaUnrated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "tvpgRating": {
-     "type": "string",
-     "description": "The video's TV Parental Guidelines (TVPG) rating.",
-     "enum": [
-      "pg14",
-      "tvpgG",
-      "tvpgMa",
-      "tvpgPg",
-      "tvpgUnrated",
-      "tvpgY",
-      "tvpgY7",
-      "tvpgY7Fv"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "ytRating": {
-     "type": "string",
-     "description": "A rating that YouTube uses to identify age-restricted content.",
-     "enum": [
-      "ytAgeRestricted"
-     ],
-     "enumDescriptions": [
-      ""
-     ]
-    }
-   }
-  },
-  "FanFundingEvent": {
-   "id": "FanFundingEvent",
-   "type": "object",
-   "description": "A fanFundingEvent resource represents a fan funding event on a YouTube channel. Fan funding events occur when a user gives one-time monetary support to the channel owner.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube assigns to uniquely identify the fan funding event."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#fanFundingEvent\".",
-     "default": "youtube#fanFundingEvent"
-    },
-    "snippet": {
-     "$ref": "FanFundingEventSnippet",
-     "description": "The snippet object contains basic details about the fan funding event."
-    }
-   }
-  },
-  "FanFundingEventListResponse": {
-   "id": "FanFundingEventListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of fan funding events that match the request criteria.",
-     "items": {
-      "$ref": "FanFundingEvent"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#fanFundingEventListResponse\".",
-     "default": "youtube#fanFundingEventListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "FanFundingEventSnippet": {
-   "id": "FanFundingEventSnippet",
-   "type": "object",
-   "properties": {
-    "amountMicros": {
-     "type": "string",
-     "description": "The amount of funding in micros of fund_currency. e.g., 1 is represented",
-     "format": "uint64"
-    },
-    "channelId": {
-     "type": "string",
-     "description": "Channel id where the funding event occurred."
-    },
-    "commentText": {
-     "type": "string",
-     "description": "The text contents of the comment left by the user."
-    },
-    "createdAt": {
-     "type": "string",
-     "description": "The date and time when the funding occurred. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "currency": {
-     "type": "string",
-     "description": "The currency in which the fund was made. ISO 4217."
-    },
-    "displayString": {
-     "type": "string",
-     "description": "A rendered string that displays the fund amount and currency (e.g., \"$1.00\"). The string is rendered for the given language."
-    },
-    "supporterDetails": {
-     "$ref": "ChannelProfileDetails",
-     "description": "Details about the supporter. Only filled if the event was made public by the user."
-    }
-   }
-  },
-  "GeoPoint": {
-   "id": "GeoPoint",
-   "type": "object",
-   "description": "Geographical coordinates of a point, in WGS84.",
-   "properties": {
-    "altitude": {
-     "type": "number",
-     "description": "Altitude above the reference ellipsoid, in meters.",
-     "format": "double"
-    },
-    "latitude": {
-     "type": "number",
-     "description": "Latitude in degrees.",
-     "format": "double"
-    },
-    "longitude": {
-     "type": "number",
-     "description": "Longitude in degrees.",
-     "format": "double"
-    }
-   }
-  },
-  "GuideCategory": {
-   "id": "GuideCategory",
-   "type": "object",
-   "description": "A guideCategory resource identifies a category that YouTube algorithmically assigns based on a channel's content or other indicators, such as the channel's popularity. The list is similar to video categories, with the difference being that a video's uploader can assign a video category but only YouTube can assign a channel category.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the guide category."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#guideCategory\".",
-     "default": "youtube#guideCategory"
-    },
-    "snippet": {
-     "$ref": "GuideCategorySnippet",
-     "description": "The snippet object contains basic details about the category, such as its title."
-    }
-   }
-  },
-  "GuideCategoryListResponse": {
-   "id": "GuideCategoryListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of categories that can be associated with YouTube channels. In this map, the category ID is the map key, and its value is the corresponding guideCategory resource.",
-     "items": {
-      "$ref": "GuideCategory"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#guideCategoryListResponse\".",
-     "default": "youtube#guideCategoryListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "prevPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the previous page in the result set."
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "GuideCategorySnippet": {
-   "id": "GuideCategorySnippet",
-   "type": "object",
-   "description": "Basic details about a guide category.",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "default": "UCBR8-60-B28hp2BmDPdntcQ"
-    },
-    "title": {
-     "type": "string",
-     "description": "Description of the guide category."
-    }
-   }
-  },
-  "I18nLanguage": {
-   "id": "I18nLanguage",
-   "type": "object",
-   "description": "An i18nLanguage resource identifies a UI language currently supported by YouTube.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the i18n language."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#i18nLanguage\".",
-     "default": "youtube#i18nLanguage"
-    },
-    "snippet": {
-     "$ref": "I18nLanguageSnippet",
-     "description": "The snippet object contains basic details about the i18n language, such as language code and human-readable name."
-    }
-   }
-  },
-  "I18nLanguageListResponse": {
-   "id": "I18nLanguageListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of supported i18n languages. In this map, the i18n language ID is the map key, and its value is the corresponding i18nLanguage resource.",
-     "items": {
-      "$ref": "I18nLanguage"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#i18nLanguageListResponse\".",
-     "default": "youtube#i18nLanguageListResponse"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "I18nLanguageSnippet": {
-   "id": "I18nLanguageSnippet",
-   "type": "object",
-   "description": "Basic details about an i18n language, such as language code and human-readable name.",
-   "properties": {
-    "hl": {
-     "type": "string",
-     "description": "A short BCP-47 code that uniquely identifies a language."
-    },
-    "name": {
-     "type": "string",
-     "description": "The human-readable name of the language in the language itself."
-    }
-   }
-  },
-  "I18nRegion": {
-   "id": "I18nRegion",
-   "type": "object",
-   "description": "A i18nRegion resource identifies a region where YouTube is available.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the i18n region."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#i18nRegion\".",
-     "default": "youtube#i18nRegion"
-    },
-    "snippet": {
-     "$ref": "I18nRegionSnippet",
-     "description": "The snippet object contains basic details about the i18n region, such as region code and human-readable name."
-    }
-   }
-  },
-  "I18nRegionListResponse": {
-   "id": "I18nRegionListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of regions where YouTube is available. In this map, the i18n region ID is the map key, and its value is the corresponding i18nRegion resource.",
-     "items": {
-      "$ref": "I18nRegion"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#i18nRegionListResponse\".",
-     "default": "youtube#i18nRegionListResponse"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "I18nRegionSnippet": {
-   "id": "I18nRegionSnippet",
-   "type": "object",
-   "description": "Basic details about an i18n region, such as region code and human-readable name.",
-   "properties": {
-    "gl": {
-     "type": "string",
-     "description": "The region code as a 2-letter ISO country code."
-    },
-    "name": {
-     "type": "string",
-     "description": "The human-readable name of the region."
-    }
-   }
-  },
-  "ImageSettings": {
-   "id": "ImageSettings",
-   "type": "object",
-   "description": "Branding properties for images associated with the channel.",
-   "properties": {
-    "backgroundImageUrl": {
-     "$ref": "LocalizedProperty",
-     "description": "The URL for the background image shown on the video watch page. The image should be 1200px by 615px, with a maximum file size of 128k."
-    },
-    "bannerExternalUrl": {
-     "type": "string",
-     "description": "This is used only in update requests; if it's set, we use this URL to generate all of the above banner URLs."
-    },
-    "bannerImageUrl": {
-     "type": "string",
-     "description": "Banner image. Desktop size (1060x175)."
-    },
-    "bannerMobileExtraHdImageUrl": {
-     "type": "string",
-     "description": "Banner image. Mobile size high resolution (1440x395)."
-    },
-    "bannerMobileHdImageUrl": {
-     "type": "string",
-     "description": "Banner image. Mobile size high resolution (1280x360)."
-    },
-    "bannerMobileImageUrl": {
-     "type": "string",
-     "description": "Banner image. Mobile size (640x175)."
-    },
-    "bannerMobileLowImageUrl": {
-     "type": "string",
-     "description": "Banner image. Mobile size low resolution (320x88)."
-    },
-    "bannerMobileMediumHdImageUrl": {
-     "type": "string",
-     "description": "Banner image. Mobile size medium/high resolution (960x263)."
-    },
-    "bannerTabletExtraHdImageUrl": {
-     "type": "string",
-     "description": "Banner image. Tablet size extra high resolution (2560x424)."
-    },
-    "bannerTabletHdImageUrl": {
-     "type": "string",
-     "description": "Banner image. Tablet size high resolution (2276x377)."
-    },
-    "bannerTabletImageUrl": {
-     "type": "string",
-     "description": "Banner image. Tablet size (1707x283)."
-    },
-    "bannerTabletLowImageUrl": {
-     "type": "string",
-     "description": "Banner image. Tablet size low resolution (1138x188)."
-    },
-    "bannerTvHighImageUrl": {
-     "type": "string",
-     "description": "Banner image. TV size high resolution (1920x1080)."
-    },
-    "bannerTvImageUrl": {
-     "type": "string",
-     "description": "Banner image. TV size extra high resolution (2120x1192)."
-    },
-    "bannerTvLowImageUrl": {
-     "type": "string",
-     "description": "Banner image. TV size low resolution (854x480)."
-    },
-    "bannerTvMediumImageUrl": {
-     "type": "string",
-     "description": "Banner image. TV size medium resolution (1280x720)."
-    },
-    "largeBrandedBannerImageImapScript": {
-     "$ref": "LocalizedProperty",
-     "description": "The image map script for the large banner image."
-    },
-    "largeBrandedBannerImageUrl": {
-     "$ref": "LocalizedProperty",
-     "description": "The URL for the 854px by 70px image that appears below the video player in the expanded video view of the video watch page."
-    },
-    "smallBrandedBannerImageImapScript": {
-     "$ref": "LocalizedProperty",
-     "description": "The image map script for the small banner image."
-    },
-    "smallBrandedBannerImageUrl": {
-     "$ref": "LocalizedProperty",
-     "description": "The URL for the 640px by 70px banner image that appears below the video player in the default view of the video watch page."
-    },
-    "trackingImageUrl": {
-     "type": "string",
-     "description": "The URL for a 1px by 1px tracking pixel that can be used to collect statistics for views of the channel or video pages."
-    },
-    "watchIconImageUrl": {
-     "type": "string",
-     "description": "The URL for the image that appears above the top-left corner of the video player. This is a 25-pixel-high image with a flexible width that cannot exceed 170 pixels."
-    }
-   }
-  },
-  "IngestionInfo": {
-   "id": "IngestionInfo",
-   "type": "object",
-   "description": "Describes information necessary for ingesting an RTMP or an HTTP stream.",
-   "properties": {
-    "backupIngestionAddress": {
-     "type": "string",
-     "description": "The backup ingestion URL that you should use to stream video to YouTube. You have the option of simultaneously streaming the content that you are sending to the ingestionAddress to this URL."
-    },
-    "ingestionAddress": {
-     "type": "string",
-     "description": "The primary ingestion URL that you should use to stream video to YouTube. You must stream video to this URL.\n\nDepending on which application or tool you use to encode your video stream, you may need to enter the stream URL and stream name separately or you may need to concatenate them in the following format:\n\nSTREAM_URL/STREAM_NAME"
-    },
-    "streamName": {
-     "type": "string",
-     "description": "The HTTP or RTMP stream name that YouTube assigns to the video stream."
-    }
-   }
-  },
-  "InvideoBranding": {
-   "id": "InvideoBranding",
-   "type": "object",
-   "properties": {
-    "imageBytes": {
-     "type": "string",
-     "format": "byte"
-    },
-    "imageUrl": {
-     "type": "string"
-    },
-    "position": {
-     "$ref": "InvideoPosition"
-    },
-    "targetChannelId": {
-     "type": "string"
-    },
-    "timing": {
-     "$ref": "InvideoTiming"
-    }
-   }
-  },
-  "InvideoPosition": {
-   "id": "InvideoPosition",
-   "type": "object",
-   "description": "Describes the spatial position of a visual widget inside a video. It is a union of various position types, out of which only will be set one.",
-   "properties": {
-    "cornerPosition": {
-     "type": "string",
-     "description": "Describes in which corner of the video the visual widget will appear.",
-     "enum": [
-      "bottomLeft",
-      "bottomRight",
-      "topLeft",
-      "topRight"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "type": {
-     "type": "string",
-     "description": "Defines the position type.",
-     "enum": [
-      "corner"
-     ],
-     "enumDescriptions": [
-      ""
-     ]
-    }
-   }
-  },
-  "InvideoPromotion": {
-   "id": "InvideoPromotion",
-   "type": "object",
-   "description": "Describes an invideo promotion campaign consisting of multiple promoted items. A campaign belongs to a single channel_id.",
-   "properties": {
-    "defaultTiming": {
-     "$ref": "InvideoTiming",
-     "description": "The default temporal position within the video where the promoted item will be displayed. Can be overriden by more specific timing in the item."
-    },
-    "items": {
-     "type": "array",
-     "description": "List of promoted items in decreasing priority.",
-     "items": {
-      "$ref": "PromotedItem"
-     }
-    },
-    "position": {
-     "$ref": "InvideoPosition",
-     "description": "The spatial position within the video where the promoted item will be displayed."
-    },
-    "useSmartTiming": {
-     "type": "boolean",
-     "description": "Indicates whether the channel's promotional campaign uses \"smart timing.\" This feature attempts to show promotions at a point in the video when they are more likely to be clicked and less likely to disrupt the viewing experience. This feature also picks up a single promotion to show on each video."
-    }
-   }
-  },
-  "InvideoTiming": {
-   "id": "InvideoTiming",
-   "type": "object",
-   "description": "Describes a temporal position of a visual widget inside a video.",
-   "properties": {
-    "durationMs": {
-     "type": "string",
-     "description": "Defines the duration in milliseconds for which the promotion should be displayed. If missing, the client should use the default.",
-     "format": "uint64"
-    },
-    "offsetMs": {
-     "type": "string",
-     "description": "Defines the time at which the promotion will appear. Depending on the value of type the value of the offsetMs field will represent a time offset from the start or from the end of the video, expressed in milliseconds.",
-     "format": "uint64"
-    },
-    "type": {
-     "type": "string",
-     "description": "Describes a timing type. If the value is offsetFromStart, then the offsetMs field represents an offset from the start of the video. If the value is offsetFromEnd, then the offsetMs field represents an offset from the end of the video.",
-     "enum": [
-      "offsetFromEnd",
-      "offsetFromStart"
-     ],
-     "enumDescriptions": [
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "LanguageTag": {
-   "id": "LanguageTag",
-   "type": "object",
-   "properties": {
-    "value": {
-     "type": "string"
-    }
-   }
-  },
-  "LiveBroadcast": {
-   "id": "LiveBroadcast",
-   "type": "object",
-   "description": "A liveBroadcast resource represents an event that will be streamed, via live video, on YouTube.",
-   "properties": {
-    "contentDetails": {
-     "$ref": "LiveBroadcastContentDetails",
-     "description": "The contentDetails object contains information about the event's video content, such as whether the content can be shown in an embedded video player or if it will be archived and therefore available for viewing after the event has concluded."
-    },
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube assigns to uniquely identify the broadcast.",
-     "annotations": {
-      "required": [
-       "youtube.liveBroadcasts.update"
-      ]
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#liveBroadcast\".",
-     "default": "youtube#liveBroadcast"
-    },
-    "snippet": {
-     "$ref": "LiveBroadcastSnippet",
-     "description": "The snippet object contains basic details about the event, including its title, description, start time, and end time."
-    },
-    "statistics": {
-     "$ref": "LiveBroadcastStatistics",
-     "description": "The statistics object contains info about the event's current stats. These include concurrent viewers and total chat count. Statistics can change (in either direction) during the lifetime of an event. Statistics are only returned while the event is live."
-    },
-    "status": {
-     "$ref": "LiveBroadcastStatus",
-     "description": "The status object contains information about the event's status."
-    },
-    "topicDetails": {
-     "$ref": "LiveBroadcastTopicDetails"
-    }
-   }
-  },
-  "LiveBroadcastContentDetails": {
-   "id": "LiveBroadcastContentDetails",
-   "type": "object",
-   "description": "Detailed settings of a broadcast.",
-   "properties": {
-    "boundStreamId": {
-     "type": "string",
-     "description": "This value uniquely identifies the live stream bound to the broadcast."
-    },
-    "boundStreamLastUpdateTimeMs": {
-     "type": "string",
-     "description": "The date and time that the live stream referenced by boundStreamId was last updated.",
-     "format": "date-time"
-    },
-    "closedCaptionsType": {
-     "type": "string",
-     "enum": [
-      "closedCaptionsDisabled",
-      "closedCaptionsEmbedded",
-      "closedCaptionsHttpPost"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    },
-    "enableClosedCaptions": {
-     "type": "boolean",
-     "description": "This setting indicates whether HTTP POST closed captioning is enabled for this broadcast. The ingestion URL of the closed captions is returned through the liveStreams API. This is mutually exclusive with using the closed_captions_type property, and is equivalent to setting closed_captions_type to CLOSED_CAPTIONS_HTTP_POST."
-    },
-    "enableContentEncryption": {
-     "type": "boolean",
-     "description": "This setting indicates whether YouTube should enable content encryption for the broadcast.",
-     "annotations": {
-      "required": [
-       "youtube.liveBroadcasts.update"
-      ]
-     }
-    },
-    "enableDvr": {
-     "type": "boolean",
-     "description": "This setting determines whether viewers can access DVR controls while watching the video. DVR controls enable the viewer to control the video playback experience by pausing, rewinding, or fast forwarding content. The default value for this property is true.\n\n\n\nImportant: You must set the value to true and also set the enableArchive property's value to true if you want to make playback available immediately after the broadcast ends.",
-     "annotations": {
-      "required": [
-       "youtube.liveBroadcasts.update"
-      ]
-     }
-    },
-    "enableEmbed": {
-     "type": "boolean",
-     "description": "This setting indicates whether the broadcast video can be played in an embedded player. If you choose to archive the video (using the enableArchive property), this setting will also apply to the archived video.",
-     "annotations": {
-      "required": [
-       "youtube.liveBroadcasts.update"
-      ]
-     }
-    },
-    "enableLowLatency": {
-     "type": "boolean",
-     "description": "Indicates whether this broadcast has low latency enabled."
-    },
-    "monitorStream": {
-     "$ref": "MonitorStreamInfo",
-     "description": "The monitorStream object contains information about the monitor stream, which the broadcaster can use to review the event content before the broadcast stream is shown publicly."
-    },
-    "projection": {
-     "type": "string",
-     "description": "The projection format of this broadcast. This defaults to rectangular.",
-     "enum": [
-      "360",
-      "rectangular"
-     ],
-     "enumDescriptions": [
-      "",
-      ""
-     ]
-    },
-    "recordFromStart": {
-     "type": "boolean",
-     "description": "Automatically start recording after the event goes live. The default value for this property is true.\n\n\n\nImportant: You must also set the enableDvr property's value to true if you want the playback to be available immediately after the broadcast ends. If you set this property's value to true but do not also set the enableDvr property to true, there may be a delay of around one day before the archived video will be available for playback.",
-     "annotations": {
-      "required": [
-       "youtube.liveBroadcasts.update"
-      ]
-     }
-    },
-    "startWithSlate": {
-     "type": "boolean",
-     "description": "This setting indicates whether the broadcast should automatically begin with an in-stream slate when you update the broadcast's status to live. After updating the status, you then need to send a liveCuepoints.insert request that sets the cuepoint's eventState to end to remove the in-stream slate and make your broadcast stream visible to viewers.",
-     "annotations": {
-      "required": [
-       "youtube.liveBroadcasts.update"
-      ]
-     }
-    }
-   }
-  },
-  "LiveBroadcastListResponse": {
-   "id": "LiveBroadcastListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of broadcasts that match the request criteria.",
-     "items": {
-      "$ref": "LiveBroadcast"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#liveBroadcastListResponse\".",
-     "default": "youtube#liveBroadcastListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "prevPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the previous page in the result set."
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "LiveBroadcastSnippet": {
-   "id": "LiveBroadcastSnippet",
-   "type": "object",
-   "properties": {
-    "actualEndTime": {
-     "type": "string",
-     "description": "The date and time that the broadcast actually ended. This information is only available once the broadcast's state is complete. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "actualStartTime": {
-     "type": "string",
-     "description": "The date and time that the broadcast actually started. This information is only available once the broadcast's state is live. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "channelId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the channel that is publishing the broadcast."
-    },
-    "description": {
-     "type": "string",
-     "description": "The broadcast's description. As with the title, you can set this field by modifying the broadcast resource or by setting the description field of the corresponding video resource."
-    },
-    "isDefaultBroadcast": {
-     "type": "boolean"
-    },
-    "liveChatId": {
-     "type": "string",
-     "description": "The id of the live chat for this broadcast."
-    },
-    "publishedAt": {
-     "type": "string",
-     "description": "The date and time that the broadcast was added to YouTube's live broadcast schedule. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "scheduledEndTime": {
-     "type": "string",
-     "description": "The date and time that the broadcast is scheduled to end. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "scheduledStartTime": {
-     "type": "string",
-     "description": "The date and time that the broadcast is scheduled to start. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time",
-     "annotations": {
-      "required": [
-       "youtube.liveBroadcasts.insert",
-       "youtube.liveBroadcasts.update"
-      ]
-     }
-    },
-    "thumbnails": {
-     "$ref": "ThumbnailDetails",
-     "description": "A map of thumbnail images associated with the broadcast. For each nested object in this object, the key is the name of the thumbnail image, and the value is an object that contains other information about the thumbnail."
-    },
-    "title": {
-     "type": "string",
-     "description": "The broadcast's title. Note that the broadcast represents exactly one YouTube video. You can set this field by modifying the broadcast resource or by setting the title field of the corresponding video resource.",
-     "annotations": {
-      "required": [
-       "youtube.liveBroadcasts.insert",
-       "youtube.liveBroadcasts.update"
-      ]
-     }
-    }
-   }
-  },
-  "LiveBroadcastStatistics": {
-   "id": "LiveBroadcastStatistics",
-   "type": "object",
-   "description": "Statistics about the live broadcast. These represent a snapshot of the values at the time of the request. Statistics are only returned for live broadcasts.",
-   "properties": {
-    "concurrentViewers": {
-     "type": "string",
-     "description": "The number of viewers currently watching the broadcast. The property and its value will be present if the broadcast has current viewers and the broadcast owner has not hidden the viewcount for the video. Note that YouTube stops tracking the number of concurrent viewers for a broadcast when the broadcast ends. So, this property would not identify the number of viewers watching an archived video of a live broadcast that already ended.",
-     "format": "uint64"
-    },
-    "totalChatCount": {
-     "type": "string",
-     "description": "The total number of live chat messages currently on the broadcast. The property and its value will be present if the broadcast is public, has the live chat feature enabled, and has at least one message. Note that this field will not be filled after the broadcast ends. So this property would not identify the number of chat messages for an archived video of a completed live broadcast.",
-     "format": "uint64"
-    }
-   }
-  },
-  "LiveBroadcastStatus": {
-   "id": "LiveBroadcastStatus",
-   "type": "object",
-   "properties": {
-    "lifeCycleStatus": {
-     "type": "string",
-     "description": "The broadcast's status. The status can be updated using the API's liveBroadcasts.transition method.",
-     "enum": [
-      "abandoned",
-      "complete",
-      "completeStarting",
-      "created",
-      "live",
-      "liveStarting",
-      "ready",
-      "reclaimed",
-      "revoked",
-      "testStarting",
-      "testing"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "liveBroadcastPriority": {
-     "type": "string",
-     "description": "Priority of the live broadcast event (internal state).",
-     "enum": [
-      "high",
-      "low",
-      "normal"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    },
-    "privacyStatus": {
-     "type": "string",
-     "description": "The broadcast's privacy status. Note that the broadcast represents exactly one YouTube video, so the privacy settings are identical to those supported for videos. In addition, you can set this field by modifying the broadcast resource or by setting the privacyStatus field of the corresponding video resource.",
-     "enum": [
-      "private",
-      "public",
-      "unlisted"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ],
-     "annotations": {
-      "required": [
-       "youtube.liveBroadcasts.insert",
-       "youtube.liveBroadcasts.update"
-      ]
-     }
-    },
-    "recordingStatus": {
-     "type": "string",
-     "description": "The broadcast's recording status.",
-     "enum": [
-      "notRecording",
-      "recorded",
-      "recording"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "LiveBroadcastTopic": {
-   "id": "LiveBroadcastTopic",
-   "type": "object",
-   "properties": {
-    "snippet": {
-     "$ref": "LiveBroadcastTopicSnippet",
-     "description": "Information about the topic matched."
-    },
-    "type": {
-     "type": "string",
-     "description": "The type of the topic.",
-     "enum": [
-      "videoGame"
-     ],
-     "enumDescriptions": [
-      ""
-     ]
-    },
-    "unmatched": {
-     "type": "boolean",
-     "description": "If this flag is set it means that we have not been able to match the topic title and type provided to a known entity."
-    }
-   }
-  },
-  "LiveBroadcastTopicDetails": {
-   "id": "LiveBroadcastTopicDetails",
-   "type": "object",
-   "properties": {
-    "topics": {
-     "type": "array",
-     "items": {
-      "$ref": "LiveBroadcastTopic"
-     }
-    }
-   }
-  },
-  "LiveBroadcastTopicSnippet": {
-   "id": "LiveBroadcastTopicSnippet",
-   "type": "object",
-   "properties": {
-    "name": {
-     "type": "string",
-     "description": "The name of the topic."
-    },
-    "releaseDate": {
-     "type": "string",
-     "description": "The date at which the topic was released. Filled for types: videoGame"
-    }
-   }
-  },
-  "LiveChatBan": {
-   "id": "LiveChatBan",
-   "type": "object",
-   "description": "A liveChatBan resource represents a ban for a YouTube live chat.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube assigns to uniquely identify the ban."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#liveChatBan\".",
-     "default": "youtube#liveChatBan"
-    },
-    "snippet": {
-     "$ref": "LiveChatBanSnippet",
-     "description": "The snippet object contains basic details about the ban."
-    }
-   }
-  },
-  "LiveChatBanSnippet": {
-   "id": "LiveChatBanSnippet",
-   "type": "object",
-   "properties": {
-    "banDurationSeconds": {
-     "type": "string",
-     "description": "The duration of a ban, only filled if the ban has type TEMPORARY.",
-     "format": "uint64"
-    },
-    "bannedUserDetails": {
-     "$ref": "ChannelProfileDetails"
-    },
-    "liveChatId": {
-     "type": "string",
-     "description": "The chat this ban is pertinent to."
-    },
-    "type": {
-     "type": "string",
-     "description": "The type of ban.",
-     "enum": [
-      "permanent",
-      "temporary"
-     ],
-     "enumDescriptions": [
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "LiveChatFanFundingEventDetails": {
-   "id": "LiveChatFanFundingEventDetails",
-   "type": "object",
-   "properties": {
-    "amountDisplayString": {
-     "type": "string",
-     "description": "A rendered string that displays the fund amount and currency to the user."
-    },
-    "amountMicros": {
-     "type": "string",
-     "description": "The amount of the fund.",
-     "format": "uint64"
-    },
-    "currency": {
-     "type": "string",
-     "description": "The currency in which the fund was made."
-    },
-    "userComment": {
-     "type": "string",
-     "description": "The comment added by the user to this fan funding event."
-    }
-   }
-  },
-  "LiveChatMessage": {
-   "id": "LiveChatMessage",
-   "type": "object",
-   "description": "A liveChatMessage resource represents a chat message in a YouTube Live Chat.",
-   "properties": {
-    "authorDetails": {
-     "$ref": "LiveChatMessageAuthorDetails",
-     "description": "The authorDetails object contains basic details about the user that posted this message."
-    },
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube assigns to uniquely identify the message."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#liveChatMessage\".",
-     "default": "youtube#liveChatMessage"
-    },
-    "snippet": {
-     "$ref": "LiveChatMessageSnippet",
-     "description": "The snippet object contains basic details about the message."
-    }
-   }
-  },
-  "LiveChatMessageAuthorDetails": {
-   "id": "LiveChatMessageAuthorDetails",
-   "type": "object",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "description": "The YouTube channel ID."
-    },
-    "channelUrl": {
-     "type": "string",
-     "description": "The channel's URL."
-    },
-    "displayName": {
-     "type": "string",
-     "description": "The channel's display name."
-    },
-    "isChatModerator": {
-     "type": "boolean",
-     "description": "Whether the author is a moderator of the live chat."
-    },
-    "isChatOwner": {
-     "type": "boolean",
-     "description": "Whether the author is the owner of the live chat."
-    },
-    "isChatSponsor": {
-     "type": "boolean",
-     "description": "Whether the author is a sponsor of the live chat."
-    },
-    "isVerified": {
-     "type": "boolean",
-     "description": "Whether the author's identity has been verified by YouTube."
-    },
-    "profileImageUrl": {
-     "type": "string",
-     "description": "The channels's avatar URL."
-    }
-   }
-  },
-  "LiveChatMessageDeletedDetails": {
-   "id": "LiveChatMessageDeletedDetails",
-   "type": "object",
-   "properties": {
-    "deletedMessageId": {
-     "type": "string"
-    }
-   }
-  },
-  "LiveChatMessageListResponse": {
-   "id": "LiveChatMessageListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of live chat messages.",
-     "items": {
-      "$ref": "LiveChatMessage"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#liveChatMessageListResponse\".",
-     "default": "youtube#liveChatMessageListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "offlineAt": {
-     "type": "string",
-     "description": "The date and time when the underlying stream went offline. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "pollingIntervalMillis": {
-     "type": "integer",
-     "description": "The amount of time the client should wait before polling again.",
-     "format": "uint32"
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "LiveChatMessageRetractedDetails": {
-   "id": "LiveChatMessageRetractedDetails",
-   "type": "object",
-   "properties": {
-    "retractedMessageId": {
-     "type": "string"
-    }
-   }
-  },
-  "LiveChatMessageSnippet": {
-   "id": "LiveChatMessageSnippet",
-   "type": "object",
-   "properties": {
-    "authorChannelId": {
-     "type": "string",
-     "description": "The ID of the user that authored this message, this field is not always filled. textMessageEvent - the user that wrote the message fanFundingEvent - the user that funded the broadcast newSponsorEvent - the user that just became a sponsor messageDeletedEvent - the moderator that took the action messageRetractedEvent - the author that retracted their message userBannedEvent - the moderator that took the action superChatEvent - the user that made the purchase"
-    },
-    "displayMessage": {
-     "type": "string",
-     "description": "Contains a string that can be displayed to the user. If this field is not present the message is silent, at the moment only messages of type TOMBSTONE and CHAT_ENDED_EVENT are silent."
-    },
-    "fanFundingEventDetails": {
-     "$ref": "LiveChatFanFundingEventDetails",
-     "description": "Details about the funding event, this is only set if the type is 'fanFundingEvent'."
-    },
-    "hasDisplayContent": {
-     "type": "boolean",
-     "description": "Whether the message has display content that should be displayed to users."
-    },
-    "liveChatId": {
-     "type": "string"
-    },
-    "messageDeletedDetails": {
-     "$ref": "LiveChatMessageDeletedDetails"
-    },
-    "messageRetractedDetails": {
-     "$ref": "LiveChatMessageRetractedDetails"
-    },
-    "pollClosedDetails": {
-     "$ref": "LiveChatPollClosedDetails"
-    },
-    "pollEditedDetails": {
-     "$ref": "LiveChatPollEditedDetails"
-    },
-    "pollOpenedDetails": {
-     "$ref": "LiveChatPollOpenedDetails"
-    },
-    "pollVotedDetails": {
-     "$ref": "LiveChatPollVotedDetails"
-    },
-    "publishedAt": {
-     "type": "string",
-     "description": "The date and time when the message was orignally published. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "superChatDetails": {
-     "$ref": "LiveChatSuperChatDetails",
-     "description": "Details about the Super Chat event, this is only set if the type is 'superChatEvent'."
-    },
-    "textMessageDetails": {
-     "$ref": "LiveChatTextMessageDetails",
-     "description": "Details about the text message, this is only set if the type is 'textMessageEvent'."
-    },
-    "type": {
-     "type": "string",
-     "description": "The type of message, this will always be present, it determines the contents of the message as well as which fields will be present.",
-     "enum": [
-      "chatEndedEvent",
-      "fanFundingEvent",
-      "messageDeletedEvent",
-      "messageRetractedEvent",
-      "newSponsorEvent",
-      "pollClosedEvent",
-      "pollEditedEvent",
-      "pollOpenedEvent",
-      "pollVotedEvent",
-      "sponsorOnlyModeEndedEvent",
-      "sponsorOnlyModeStartedEvent",
-      "superChatEvent",
-      "textMessageEvent",
-      "tombstone",
-      "userBannedEvent"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "userBannedDetails": {
-     "$ref": "LiveChatUserBannedMessageDetails"
-    }
-   }
-  },
-  "LiveChatModerator": {
-   "id": "LiveChatModerator",
-   "type": "object",
-   "description": "A liveChatModerator resource represents a moderator for a YouTube live chat. A chat moderator has the ability to ban/unban users from a chat, remove message, etc.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube assigns to uniquely identify the moderator."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#liveChatModerator\".",
-     "default": "youtube#liveChatModerator"
-    },
-    "snippet": {
-     "$ref": "LiveChatModeratorSnippet",
-     "description": "The snippet object contains basic details about the moderator."
-    }
-   }
-  },
-  "LiveChatModeratorListResponse": {
-   "id": "LiveChatModeratorListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of moderators that match the request criteria.",
-     "items": {
-      "$ref": "LiveChatModerator"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#liveChatModeratorListResponse\".",
-     "default": "youtube#liveChatModeratorListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "prevPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the previous page in the result set."
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "LiveChatModeratorSnippet": {
-   "id": "LiveChatModeratorSnippet",
-   "type": "object",
-   "properties": {
-    "liveChatId": {
-     "type": "string",
-     "description": "The ID of the live chat this moderator can act on."
-    },
-    "moderatorDetails": {
-     "$ref": "ChannelProfileDetails",
-     "description": "Details about the moderator."
-    }
-   }
-  },
-  "LiveChatPollClosedDetails": {
-   "id": "LiveChatPollClosedDetails",
-   "type": "object",
-   "properties": {
-    "pollId": {
-     "type": "string",
-     "description": "The id of the poll that was closed."
-    }
-   }
-  },
-  "LiveChatPollEditedDetails": {
-   "id": "LiveChatPollEditedDetails",
-   "type": "object",
-   "properties": {
-    "id": {
-     "type": "string"
-    },
-    "items": {
-     "type": "array",
-     "items": {
-      "$ref": "LiveChatPollItem"
-     }
-    },
-    "prompt": {
-     "type": "string"
-    }
-   }
-  },
-  "LiveChatPollItem": {
-   "id": "LiveChatPollItem",
-   "type": "object",
-   "properties": {
-    "description": {
-     "type": "string",
-     "description": "Plain text description of the item."
-    },
-    "itemId": {
-     "type": "string"
-    }
-   }
-  },
-  "LiveChatPollOpenedDetails": {
-   "id": "LiveChatPollOpenedDetails",
-   "type": "object",
-   "properties": {
-    "id": {
-     "type": "string"
-    },
-    "items": {
-     "type": "array",
-     "items": {
-      "$ref": "LiveChatPollItem"
-     }
-    },
-    "prompt": {
-     "type": "string"
-    }
-   }
-  },
-  "LiveChatPollVotedDetails": {
-   "id": "LiveChatPollVotedDetails",
-   "type": "object",
-   "properties": {
-    "itemId": {
-     "type": "string",
-     "description": "The poll item the user chose."
-    },
-    "pollId": {
-     "type": "string",
-     "description": "The poll the user voted on."
-    }
-   }
-  },
-  "LiveChatSuperChatDetails": {
-   "id": "LiveChatSuperChatDetails",
-   "type": "object",
-   "properties": {
-    "amountDisplayString": {
-     "type": "string",
-     "description": "A rendered string that displays the fund amount and currency to the user."
-    },
-    "amountMicros": {
-     "type": "string",
-     "description": "The amount purchased by the user, in micros (1,750,000 micros = 1.75).",
-     "format": "uint64"
-    },
-    "currency": {
-     "type": "string",
-     "description": "The currency in which the purchase was made."
-    },
-    "tier": {
-     "type": "integer",
-     "description": "The tier in which the amount belongs to. Lower amounts belong to lower tiers. Starts at 1.",
-     "format": "uint32"
-    },
-    "userComment": {
-     "type": "string",
-     "description": "The comment added by the user to this Super Chat event."
-    }
-   }
-  },
-  "LiveChatTextMessageDetails": {
-   "id": "LiveChatTextMessageDetails",
-   "type": "object",
-   "properties": {
-    "messageText": {
-     "type": "string",
-     "description": "The user's message."
-    }
-   }
-  },
-  "LiveChatUserBannedMessageDetails": {
-   "id": "LiveChatUserBannedMessageDetails",
-   "type": "object",
-   "properties": {
-    "banDurationSeconds": {
-     "type": "string",
-     "description": "The duration of the ban. This property is only present if the banType is temporary.",
-     "format": "uint64"
-    },
-    "banType": {
-     "type": "string",
-     "description": "The type of ban.",
-     "enum": [
-      "permanent",
-      "temporary"
-     ],
-     "enumDescriptions": [
-      "",
-      ""
-     ]
-    },
-    "bannedUserDetails": {
-     "$ref": "ChannelProfileDetails",
-     "description": "The details of the user that was banned."
-    }
-   }
-  },
-  "LiveStream": {
-   "id": "LiveStream",
-   "type": "object",
-   "description": "A live stream describes a live ingestion point.",
-   "properties": {
-    "cdn": {
-     "$ref": "CdnSettings",
-     "description": "The cdn object defines the live stream's content delivery network (CDN) settings. These settings provide details about the manner in which you stream your content to YouTube."
-    },
-    "contentDetails": {
-     "$ref": "LiveStreamContentDetails",
-     "description": "The content_details object contains information about the stream, including the closed captions ingestion URL."
-    },
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube assigns to uniquely identify the stream.",
-     "annotations": {
-      "required": [
-       "youtube.liveStreams.update"
-      ]
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#liveStream\".",
-     "default": "youtube#liveStream"
-    },
-    "snippet": {
-     "$ref": "LiveStreamSnippet",
-     "description": "The snippet object contains basic details about the stream, including its channel, title, and description."
-    },
-    "status": {
-     "$ref": "LiveStreamStatus",
-     "description": "The status object contains information about live stream's status."
-    }
-   }
-  },
-  "LiveStreamConfigurationIssue": {
-   "id": "LiveStreamConfigurationIssue",
-   "type": "object",
-   "properties": {
-    "description": {
-     "type": "string",
-     "description": "The long-form description of the issue and how to resolve it."
-    },
-    "reason": {
-     "type": "string",
-     "description": "The short-form reason for this issue."
-    },
-    "severity": {
-     "type": "string",
-     "description": "How severe this issue is to the stream.",
-     "enum": [
-      "error",
-      "info",
-      "warning"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    },
-    "type": {
-     "type": "string",
-     "description": "The kind of error happening.",
-     "enum": [
-      "audioBitrateHigh",
-      "audioBitrateLow",
-      "audioBitrateMismatch",
-      "audioCodec",
-      "audioCodecMismatch",
-      "audioSampleRate",
-      "audioSampleRateMismatch",
-      "audioStereoMismatch",
-      "audioTooManyChannels",
-      "badContainer",
-      "bitrateHigh",
-      "bitrateLow",
-      "frameRateHigh",
-      "framerateMismatch",
-      "gopMismatch",
-      "gopSizeLong",
-      "gopSizeOver",
-      "gopSizeShort",
-      "interlacedVideo",
-      "multipleAudioStreams",
-      "multipleVideoStreams",
-      "noAudioStream",
-      "noVideoStream",
-      "openGop",
-      "resolutionMismatch",
-      "videoBitrateMismatch",
-      "videoCodec",
-      "videoCodecMismatch",
-      "videoIngestionStarved",
-      "videoInterlaceMismatch",
-      "videoProfileMismatch",
-      "videoResolutionSuboptimal",
-      "videoResolutionUnsupported"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "LiveStreamContentDetails": {
-   "id": "LiveStreamContentDetails",
-   "type": "object",
-   "description": "Detailed settings of a stream.",
-   "properties": {
-    "closedCaptionsIngestionUrl": {
-     "type": "string",
-     "description": "The ingestion URL where the closed captions of this stream are sent."
-    },
-    "isReusable": {
-     "type": "boolean",
-     "description": "Indicates whether the stream is reusable, which means that it can be bound to multiple broadcasts. It is common for broadcasters to reuse the same stream for many different broadcasts if those broadcasts occur at different times.\n\nIf you set this value to false, then the stream will not be reusable, which means that it can only be bound to one broadcast. Non-reusable streams differ from reusable streams in the following ways:  \n- A non-reusable stream can only be bound to one broadcast. \n- A non-reusable stream might be deleted by an automated process after the broadcast ends. \n- The  liveStreams.list method does not list non-reusable streams if you call the method and set the mine parameter to true. The only way to use that method to retrieve the resource for a non-reusable stream is to use the id parameter to identify the stream."
-    }
-   }
-  },
-  "LiveStreamHealthStatus": {
-   "id": "LiveStreamHealthStatus",
-   "type": "object",
-   "properties": {
-    "configurationIssues": {
-     "type": "array",
-     "description": "The configurations issues on this stream",
-     "items": {
-      "$ref": "LiveStreamConfigurationIssue"
-     }
-    },
-    "lastUpdateTimeSeconds": {
-     "type": "string",
-     "description": "The last time this status was updated (in seconds)",
-     "format": "uint64"
-    },
-    "status": {
-     "type": "string",
-     "description": "The status code of this stream",
-     "enum": [
-      "bad",
-      "good",
-      "noData",
-      "ok",
-      "revoked"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "LiveStreamListResponse": {
-   "id": "LiveStreamListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of live streams that match the request criteria.",
-     "items": {
-      "$ref": "LiveStream"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#liveStreamListResponse\".",
-     "default": "youtube#liveStreamListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "prevPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the previous page in the result set."
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "LiveStreamSnippet": {
-   "id": "LiveStreamSnippet",
-   "type": "object",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the channel that is transmitting the stream."
-    },
-    "description": {
-     "type": "string",
-     "description": "The stream's description. The value cannot be longer than 10000 characters."
-    },
-    "isDefaultStream": {
-     "type": "boolean"
-    },
-    "publishedAt": {
-     "type": "string",
-     "description": "The date and time that the stream was created. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "title": {
-     "type": "string",
-     "description": "The stream's title. The value must be between 1 and 128 characters long.",
-     "annotations": {
-      "required": [
-       "youtube.liveStreams.insert",
-       "youtube.liveStreams.update"
-      ]
-     }
-    }
-   }
-  },
-  "LiveStreamStatus": {
-   "id": "LiveStreamStatus",
-   "type": "object",
-   "description": "Brief description of the live stream status.",
-   "properties": {
-    "healthStatus": {
-     "$ref": "LiveStreamHealthStatus",
-     "description": "The health status of the stream."
-    },
-    "streamStatus": {
-     "type": "string",
-     "enum": [
-      "active",
-      "created",
-      "error",
-      "inactive",
-      "ready"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "LocalizedProperty": {
-   "id": "LocalizedProperty",
-   "type": "object",
-   "properties": {
-    "default": {
-     "type": "string"
-    },
-    "defaultLanguage": {
-     "$ref": "LanguageTag",
-     "description": "The language of the default property."
-    },
-    "localized": {
-     "type": "array",
-     "items": {
-      "$ref": "LocalizedString"
-     }
-    }
-   }
-  },
-  "LocalizedString": {
-   "id": "LocalizedString",
-   "type": "object",
-   "properties": {
-    "language": {
-     "type": "string"
-    },
-    "value": {
-     "type": "string"
-    }
-   }
-  },
-  "MonitorStreamInfo": {
-   "id": "MonitorStreamInfo",
-   "type": "object",
-   "description": "Settings and Info of the monitor stream",
-   "properties": {
-    "broadcastStreamDelayMs": {
-     "type": "integer",
-     "description": "If you have set the enableMonitorStream property to true, then this property determines the length of the live broadcast delay.",
-     "format": "uint32",
-     "annotations": {
-      "required": [
-       "youtube.liveBroadcasts.update"
-      ]
-     }
-    },
-    "embedHtml": {
-     "type": "string",
-     "description": "HTML code that embeds a player that plays the monitor stream."
-    },
-    "enableMonitorStream": {
-     "type": "boolean",
-     "description": "This value determines whether the monitor stream is enabled for the broadcast. If the monitor stream is enabled, then YouTube will broadcast the event content on a special stream intended only for the broadcaster's consumption. The broadcaster can use the stream to review the event content and also to identify the optimal times to insert cuepoints.\n\nYou need to set this value to true if you intend to have a broadcast delay for your event.\n\nNote: This property cannot be updated once the broadcast is in the testing or live state.",
-     "annotations": {
-      "required": [
-       "youtube.liveBroadcasts.update"
-      ]
-     }
-    }
-   }
-  },
-  "PageInfo": {
-   "id": "PageInfo",
-   "type": "object",
-   "description": "Paging details for lists of resources, including total number of items available and number of resources returned in a single page.",
-   "properties": {
-    "resultsPerPage": {
-     "type": "integer",
-     "description": "The number of results included in the API response.",
-     "format": "int32"
-    },
-    "totalResults": {
-     "type": "integer",
-     "description": "The total number of results in the result set.",
-     "format": "int32"
-    }
-   }
-  },
-  "Playlist": {
-   "id": "Playlist",
-   "type": "object",
-   "description": "A playlist resource represents a YouTube playlist. A playlist is a collection of videos that can be viewed sequentially and shared with other users. A playlist can contain up to 200 videos, and YouTube does not limit the number of playlists that each user creates. By default, playlists are publicly visible to other users, but playlists can be public or private.\n\nYouTube also uses playlists to identify special collections of videos for a channel, such as:  \n- uploaded videos \n- favorite videos \n- positively rated (liked) videos \n- watch history \n- watch later  To be more specific, these lists are associated with a channel, which is a collection of a person, group, or company's videos, playlists, and other YouTube information. You can retrieve the playlist IDs for each of these lists from the  channel resource for a given channel.\n\nYou can then use the   playlistItems.list method to retrieve any of those lists. You can also add or remove items from those lists by calling the   playlistItems.insert and   playlistItems.delete methods.",
-   "properties": {
-    "contentDetails": {
-     "$ref": "PlaylistContentDetails",
-     "description": "The contentDetails object contains information like video count."
-    },
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the playlist."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#playlist\".",
-     "default": "youtube#playlist"
-    },
-    "localizations": {
-     "type": "object",
-     "description": "Localizations for different languages",
-     "additionalProperties": {
-      "$ref": "PlaylistLocalization",
-      "description": "The language tag, using string since map_key require simple types."
-     }
-    },
-    "player": {
-     "$ref": "PlaylistPlayer",
-     "description": "The player object contains information that you would use to play the playlist in an embedded player."
-    },
-    "snippet": {
-     "$ref": "PlaylistSnippet",
-     "description": "The snippet object contains basic details about the playlist, such as its title and description."
-    },
-    "status": {
-     "$ref": "PlaylistStatus",
-     "description": "The status object contains status information for the playlist."
-    }
-   }
-  },
-  "PlaylistContentDetails": {
-   "id": "PlaylistContentDetails",
-   "type": "object",
-   "properties": {
-    "itemCount": {
-     "type": "integer",
-     "description": "The number of videos in the playlist.",
-     "format": "uint32"
-    }
-   }
-  },
-  "PlaylistItem": {
-   "id": "PlaylistItem",
-   "type": "object",
-   "description": "A playlistItem resource identifies another resource, such as a video, that is included in a playlist. In addition, the playlistItem  resource contains details about the included resource that pertain specifically to how that resource is used in that playlist.\n\nYouTube uses playlists to identify special collections of videos for a channel, such as:  \n- uploaded videos \n- favorite videos \n- positively rated (liked) videos \n- watch history \n- watch later  To be more specific, these lists are associated with a channel, which is a collection of a person, group, or company's videos, playlists, and other YouTube information.\n\nYou can retrieve the playlist IDs for each of these lists from the  channel resource  for a given channel. You can then use the   playlistItems.list method to retrieve any of those lists. You can also add or remove items from those lists by calling the   playlistItems.insert and   playlistItems.delete methods. For example, if a user gives a positive rating to a video, you would insert that video into the liked videos playlist for that user's channel.",
-   "properties": {
-    "contentDetails": {
-     "$ref": "PlaylistItemContentDetails",
-     "description": "The contentDetails object is included in the resource if the included item is a YouTube video. The object contains additional information about the video."
-    },
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the playlist item."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#playlistItem\".",
-     "default": "youtube#playlistItem"
-    },
-    "snippet": {
-     "$ref": "PlaylistItemSnippet",
-     "description": "The snippet object contains basic details about the playlist item, such as its title and position in the playlist."
-    },
-    "status": {
-     "$ref": "PlaylistItemStatus",
-     "description": "The status object contains information about the playlist item's privacy status."
-    }
-   }
-  },
-  "PlaylistItemContentDetails": {
-   "id": "PlaylistItemContentDetails",
-   "type": "object",
-   "properties": {
-    "endAt": {
-     "type": "string",
-     "description": "The time, measured in seconds from the start of the video, when the video should stop playing. (The playlist owner can specify the times when the video should start and stop playing when the video is played in the context of the playlist.) By default, assume that the video.endTime is the end of the video."
-    },
-    "note": {
-     "type": "string",
-     "description": "A user-generated note for this item."
-    },
-    "startAt": {
-     "type": "string",
-     "description": "The time, measured in seconds from the start of the video, when the video should start playing. (The playlist owner can specify the times when the video should start and stop playing when the video is played in the context of the playlist.) The default value is 0."
-    },
-    "videoId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify a video. To retrieve the video resource, set the id query parameter to this value in your API request."
-    },
-    "videoPublishedAt": {
-     "type": "string",
-     "description": "The date and time that the video was published to YouTube. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    }
-   }
-  },
-  "PlaylistItemListResponse": {
-   "id": "PlaylistItemListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of playlist items that match the request criteria.",
-     "items": {
-      "$ref": "PlaylistItem"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#playlistItemListResponse\".",
-     "default": "youtube#playlistItemListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "prevPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the previous page in the result set."
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "PlaylistItemSnippet": {
-   "id": "PlaylistItemSnippet",
-   "type": "object",
-   "description": "Basic details about a playlist, including title, description and thumbnails.",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the user that added the item to the playlist."
-    },
-    "channelTitle": {
-     "type": "string",
-     "description": "Channel title for the channel that the playlist item belongs to."
-    },
-    "description": {
-     "type": "string",
-     "description": "The item's description."
-    },
-    "playlistId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the playlist that the playlist item is in.",
-     "annotations": {
-      "required": [
-       "youtube.playlistItems.insert",
-       "youtube.playlistItems.update"
-      ]
-     }
-    },
-    "position": {
-     "type": "integer",
-     "description": "The order in which the item appears in the playlist. The value uses a zero-based index, so the first item has a position of 0, the second item has a position of 1, and so forth.",
-     "format": "uint32"
-    },
-    "publishedAt": {
-     "type": "string",
-     "description": "The date and time that the item was added to the playlist. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "resourceId": {
-     "$ref": "ResourceId",
-     "description": "The id object contains information that can be used to uniquely identify the resource that is included in the playlist as the playlist item.",
-     "annotations": {
-      "required": [
-       "youtube.playlistItems.insert",
-       "youtube.playlistItems.update"
-      ]
-     }
-    },
-    "thumbnails": {
-     "$ref": "ThumbnailDetails",
-     "description": "A map of thumbnail images associated with the playlist item. For each object in the map, the key is the name of the thumbnail image, and the value is an object that contains other information about the thumbnail."
-    },
-    "title": {
-     "type": "string",
-     "description": "The item's title."
-    }
-   }
-  },
-  "PlaylistItemStatus": {
-   "id": "PlaylistItemStatus",
-   "type": "object",
-   "description": "Information about the playlist item's privacy status.",
-   "properties": {
-    "privacyStatus": {
-     "type": "string",
-     "description": "This resource's privacy status.",
-     "enum": [
-      "private",
-      "public",
-      "unlisted"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "PlaylistListResponse": {
-   "id": "PlaylistListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of playlists that match the request criteria.",
-     "items": {
-      "$ref": "Playlist"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#playlistListResponse\".",
-     "default": "youtube#playlistListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "prevPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the previous page in the result set."
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "PlaylistLocalization": {
-   "id": "PlaylistLocalization",
-   "type": "object",
-   "description": "Playlist localization setting",
-   "properties": {
-    "description": {
-     "type": "string",
-     "description": "The localized strings for playlist's description."
-    },
-    "title": {
-     "type": "string",
-     "description": "The localized strings for playlist's title."
-    }
-   }
-  },
-  "PlaylistPlayer": {
-   "id": "PlaylistPlayer",
-   "type": "object",
-   "properties": {
-    "embedHtml": {
-     "type": "string",
-     "description": "An \u003ciframe\u003e tag that embeds a player that will play the playlist."
-    }
-   }
-  },
-  "PlaylistSnippet": {
-   "id": "PlaylistSnippet",
-   "type": "object",
-   "description": "Basic details about a playlist, including title, description and thumbnails.",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the channel that published the playlist."
-    },
-    "channelTitle": {
-     "type": "string",
-     "description": "The channel title of the channel that the video belongs to."
-    },
-    "defaultLanguage": {
-     "type": "string",
-     "description": "The language of the playlist's default title and description."
-    },
-    "description": {
-     "type": "string",
-     "description": "The playlist's description."
-    },
-    "localized": {
-     "$ref": "PlaylistLocalization",
-     "description": "Localized title and description, read-only."
-    },
-    "publishedAt": {
-     "type": "string",
-     "description": "The date and time that the playlist was created. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "tags": {
-     "type": "array",
-     "description": "Keyword tags associated with the playlist.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "thumbnails": {
-     "$ref": "ThumbnailDetails",
-     "description": "A map of thumbnail images associated with the playlist. For each object in the map, the key is the name of the thumbnail image, and the value is an object that contains other information about the thumbnail."
-    },
-    "title": {
-     "type": "string",
-     "description": "The playlist's title.",
-     "annotations": {
-      "required": [
-       "youtube.playlists.insert",
-       "youtube.playlists.update"
-      ]
-     }
-    }
-   }
-  },
-  "PlaylistStatus": {
-   "id": "PlaylistStatus",
-   "type": "object",
-   "properties": {
-    "privacyStatus": {
-     "type": "string",
-     "description": "The playlist's privacy status.",
-     "enum": [
-      "private",
-      "public",
-      "unlisted"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "PromotedItem": {
-   "id": "PromotedItem",
-   "type": "object",
-   "description": "Describes a single promoted item.",
-   "properties": {
-    "customMessage": {
-     "type": "string",
-     "description": "A custom message to display for this promotion. This field is currently ignored unless the promoted item is a website."
-    },
-    "id": {
-     "$ref": "PromotedItemId",
-     "description": "Identifies the promoted item."
-    },
-    "promotedByContentOwner": {
-     "type": "boolean",
-     "description": "If true, the content owner's name will be used when displaying the promotion. This field can only be set when the update is made on behalf of the content owner."
-    },
-    "timing": {
-     "$ref": "InvideoTiming",
-     "description": "The temporal position within the video where the promoted item will be displayed. If present, it overrides the default timing."
-    }
-   }
-  },
-  "PromotedItemId": {
-   "id": "PromotedItemId",
-   "type": "object",
-   "description": "Describes a single promoted item id. It is a union of various possible types.",
-   "properties": {
-    "recentlyUploadedBy": {
-     "type": "string",
-     "description": "If type is recentUpload, this field identifies the channel from which to take the recent upload. If missing, the channel is assumed to be the same channel for which the invideoPromotion is set."
-    },
-    "type": {
-     "type": "string",
-     "description": "Describes the type of the promoted item.",
-     "enum": [
-      "recentUpload",
-      "video",
-      "website"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    },
-    "videoId": {
-     "type": "string",
-     "description": "If the promoted item represents a video, this field represents the unique YouTube ID identifying it. This field will be present only if type has the value video."
-    },
-    "websiteUrl": {
-     "type": "string",
-     "description": "If the promoted item represents a website, this field represents the url pointing to the website. This field will be present only if type has the value website."
-    }
-   }
-  },
-  "PropertyValue": {
-   "id": "PropertyValue",
-   "type": "object",
-   "description": "A pair Property / Value.",
-   "properties": {
-    "property": {
-     "type": "string",
-     "description": "A property."
-    },
-    "value": {
-     "type": "string",
-     "description": "The property's value."
-    }
-   }
-  },
-  "ResourceId": {
-   "id": "ResourceId",
-   "type": "object",
-   "description": "A resource id is a generic reference that points to another YouTube resource.",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the referred resource, if that resource is a channel. This property is only present if the resourceId.kind value is youtube#channel."
-    },
-    "kind": {
-     "type": "string",
-     "description": "The type of the API resource."
-    },
-    "playlistId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the referred resource, if that resource is a playlist. This property is only present if the resourceId.kind value is youtube#playlist."
-    },
-    "videoId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the referred resource, if that resource is a video. This property is only present if the resourceId.kind value is youtube#video."
-    }
-   }
-  },
-  "SearchListResponse": {
-   "id": "SearchListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of results that match the search criteria.",
-     "items": {
-      "$ref": "SearchResult"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#searchListResponse\".",
-     "default": "youtube#searchListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "prevPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the previous page in the result set."
-    },
-    "regionCode": {
-     "type": "string"
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "SearchResult": {
-   "id": "SearchResult",
-   "type": "object",
-   "description": "A search result contains information about a YouTube video, channel, or playlist that matches the search parameters specified in an API request. While a search result points to a uniquely identifiable resource, like a video, it does not have its own persistent data.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "$ref": "ResourceId",
-     "description": "The id object contains information that can be used to uniquely identify the resource that matches the search request."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#searchResult\".",
-     "default": "youtube#searchResult"
-    },
-    "snippet": {
-     "$ref": "SearchResultSnippet",
-     "description": "The snippet object contains basic details about a search result, such as its title or description. For example, if the search result is a video, then the title will be the video's title and the description will be the video's description."
-    }
-   }
-  },
-  "SearchResultSnippet": {
-   "id": "SearchResultSnippet",
-   "type": "object",
-   "description": "Basic details about a search result, including title, description and thumbnails of the item referenced by the search result.",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "description": "The value that YouTube uses to uniquely identify the channel that published the resource that the search result identifies."
-    },
-    "channelTitle": {
-     "type": "string",
-     "description": "The title of the channel that published the resource that the search result identifies."
-    },
-    "description": {
-     "type": "string",
-     "description": "A description of the search result."
-    },
-    "liveBroadcastContent": {
-     "type": "string",
-     "description": "It indicates if the resource (video or channel) has upcoming/active live broadcast content. Or it's \"none\" if there is not any upcoming/active live broadcasts.",
-     "enum": [
-      "live",
-      "none",
-      "upcoming"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    },
-    "publishedAt": {
-     "type": "string",
-     "description": "The creation date and time of the resource that the search result identifies. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "thumbnails": {
-     "$ref": "ThumbnailDetails",
-     "description": "A map of thumbnail images associated with the search result. For each object in the map, the key is the name of the thumbnail image, and the value is an object that contains other information about the thumbnail."
-    },
-    "title": {
-     "type": "string",
-     "description": "The title of the search result."
-    }
-   }
-  },
-  "Sponsor": {
-   "id": "Sponsor",
-   "type": "object",
-   "description": "A sponsor resource represents a sponsor for a YouTube channel. A sponsor provides recurring monetary support to a creator and receives special benefits.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube assigns to uniquely identify the sponsor."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#sponsor\".",
-     "default": "youtube#sponsor"
-    },
-    "snippet": {
-     "$ref": "SponsorSnippet",
-     "description": "The snippet object contains basic details about the sponsor."
-    }
-   }
-  },
-  "SponsorListResponse": {
-   "id": "SponsorListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of sponsors that match the request criteria.",
-     "items": {
-      "$ref": "Sponsor"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#sponsorListResponse\".",
-     "default": "youtube#sponsorListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "SponsorSnippet": {
-   "id": "SponsorSnippet",
-   "type": "object",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "description": "The id of the channel being sponsored."
-    },
-    "sponsorDetails": {
-     "$ref": "ChannelProfileDetails",
-     "description": "Details about the sponsor."
-    },
-    "sponsorSince": {
-     "type": "string",
-     "description": "The date and time when the user became a sponsor. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    }
-   }
-  },
-  "Subscription": {
-   "id": "Subscription",
-   "type": "object",
-   "description": "A subscription resource contains information about a YouTube user subscription. A subscription notifies a user when new videos are added to a channel or when another user takes one of several actions on YouTube, such as uploading a video, rating a video, or commenting on a video.",
-   "properties": {
-    "contentDetails": {
-     "$ref": "SubscriptionContentDetails",
-     "description": "The contentDetails object contains basic statistics about the subscription."
-    },
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the subscription."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#subscription\".",
-     "default": "youtube#subscription"
-    },
-    "snippet": {
-     "$ref": "SubscriptionSnippet",
-     "description": "The snippet object contains basic details about the subscription, including its title and the channel that the user subscribed to."
-    },
-    "subscriberSnippet": {
-     "$ref": "SubscriptionSubscriberSnippet",
-     "description": "The subscriberSnippet object contains basic details about the sbuscriber."
-    }
-   }
-  },
-  "SubscriptionContentDetails": {
-   "id": "SubscriptionContentDetails",
-   "type": "object",
-   "description": "Details about the content to witch a subscription refers.",
-   "properties": {
-    "activityType": {
-     "type": "string",
-     "description": "The type of activity this subscription is for (only uploads, everything).",
-     "enum": [
-      "all",
-      "uploads"
-     ],
-     "enumDescriptions": [
-      "",
-      ""
-     ]
-    },
-    "newItemCount": {
-     "type": "integer",
-     "description": "The number of new items in the subscription since its content was last read.",
-     "format": "uint32"
-    },
-    "totalItemCount": {
-     "type": "integer",
-     "description": "The approximate number of items that the subscription points to.",
-     "format": "uint32"
-    }
-   }
-  },
-  "SubscriptionListResponse": {
-   "id": "SubscriptionListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of subscriptions that match the request criteria.",
-     "items": {
-      "$ref": "Subscription"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#subscriptionListResponse\".",
-     "default": "youtube#subscriptionListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "prevPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the previous page in the result set."
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "SubscriptionSnippet": {
-   "id": "SubscriptionSnippet",
-   "type": "object",
-   "description": "Basic details about a subscription, including title, description and thumbnails of the subscribed item.",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the subscriber's channel."
-    },
-    "channelTitle": {
-     "type": "string",
-     "description": "Channel title for the channel that the subscription belongs to."
-    },
-    "description": {
-     "type": "string",
-     "description": "The subscription's details."
-    },
-    "publishedAt": {
-     "type": "string",
-     "description": "The date and time that the subscription was created. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "resourceId": {
-     "$ref": "ResourceId",
-     "description": "The id object contains information about the channel that the user subscribed to.",
-     "annotations": {
-      "required": [
-       "youtube.subscriptions.insert"
-      ]
-     }
-    },
-    "thumbnails": {
-     "$ref": "ThumbnailDetails",
-     "description": "A map of thumbnail images associated with the video. For each object in the map, the key is the name of the thumbnail image, and the value is an object that contains other information about the thumbnail."
-    },
-    "title": {
-     "type": "string",
-     "description": "The subscription's title."
-    }
-   }
-  },
-  "SubscriptionSubscriberSnippet": {
-   "id": "SubscriptionSubscriberSnippet",
-   "type": "object",
-   "description": "Basic details about a subscription's subscriber including title, description, channel ID and thumbnails.",
-   "properties": {
-    "channelId": {
-     "type": "string",
-     "description": "The channel ID of the subscriber."
-    },
-    "description": {
-     "type": "string",
-     "description": "The description of the subscriber."
-    },
-    "thumbnails": {
-     "$ref": "ThumbnailDetails",
-     "description": "Thumbnails for this subscriber."
-    },
-    "title": {
-     "type": "string",
-     "description": "The title of the subscriber."
-    }
-   }
-  },
-  "SuperChatEvent": {
-   "id": "SuperChatEvent",
-   "type": "object",
-   "description": "A superChatEvent resource represents a Super Chat purchase on a YouTube channel.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube assigns to uniquely identify the Super Chat event."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#superChatEvent\".",
-     "default": "youtube#superChatEvent"
-    },
-    "snippet": {
-     "$ref": "SuperChatEventSnippet",
-     "description": "The snippet object contains basic details about the Super Chat event."
-    }
-   }
-  },
-  "SuperChatEventListResponse": {
-   "id": "SuperChatEventListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of Super Chat purchases that match the request criteria.",
-     "items": {
-      "$ref": "SuperChatEvent"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#superChatEventListResponse\".",
-     "default": "youtube#superChatEventListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "SuperChatEventSnippet": {
-   "id": "SuperChatEventSnippet",
-   "type": "object",
-   "properties": {
-    "amountMicros": {
-     "type": "string",
-     "description": "The purchase amount, in micros of the purchase currency. e.g., 1 is represented as 1000000.",
-     "format": "uint64"
-    },
-    "channelId": {
-     "type": "string",
-     "description": "Channel id where the event occurred."
-    },
-    "commentText": {
-     "type": "string",
-     "description": "The text contents of the comment left by the user."
-    },
-    "createdAt": {
-     "type": "string",
-     "description": "The date and time when the event occurred. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "currency": {
-     "type": "string",
-     "description": "The currency in which the purchase was made. ISO 4217."
-    },
-    "displayString": {
-     "type": "string",
-     "description": "A rendered string that displays the purchase amount and currency (e.g., \"$1.00\"). The string is rendered for the given language."
-    },
-    "messageType": {
-     "type": "integer",
-     "description": "The tier for the paid message, which is based on the amount of money spent to purchase the message.",
-     "format": "uint32"
-    },
-    "supporterDetails": {
-     "$ref": "ChannelProfileDetails",
-     "description": "Details about the supporter."
-    }
-   }
-  },
-  "Thumbnail": {
-   "id": "Thumbnail",
-   "type": "object",
-   "description": "A thumbnail is an image representing a YouTube resource.",
-   "properties": {
-    "height": {
-     "type": "integer",
-     "description": "(Optional) Height of the thumbnail image.",
-     "format": "uint32"
-    },
-    "url": {
-     "type": "string",
-     "description": "The thumbnail image's URL."
-    },
-    "width": {
-     "type": "integer",
-     "description": "(Optional) Width of the thumbnail image.",
-     "format": "uint32"
-    }
-   }
-  },
-  "ThumbnailDetails": {
-   "id": "ThumbnailDetails",
-   "type": "object",
-   "description": "Internal representation of thumbnails for a YouTube resource.",
-   "properties": {
-    "default": {
-     "$ref": "Thumbnail",
-     "description": "The default image for this resource."
-    },
-    "high": {
-     "$ref": "Thumbnail",
-     "description": "The high quality image for this resource."
-    },
-    "maxres": {
-     "$ref": "Thumbnail",
-     "description": "The maximum resolution quality image for this resource."
-    },
-    "medium": {
-     "$ref": "Thumbnail",
-     "description": "The medium quality image for this resource."
-    },
-    "standard": {
-     "$ref": "Thumbnail",
-     "description": "The standard quality image for this resource."
-    }
-   }
-  },
-  "ThumbnailSetResponse": {
-   "id": "ThumbnailSetResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of thumbnails.",
-     "items": {
-      "$ref": "ThumbnailDetails"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#thumbnailSetResponse\".",
-     "default": "youtube#thumbnailSetResponse"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "TokenPagination": {
-   "id": "TokenPagination",
-   "type": "object",
-   "description": "Stub token pagination template to suppress results."
-  },
-  "Video": {
-   "id": "Video",
-   "type": "object",
-   "description": "A video resource represents a YouTube video.",
-   "properties": {
-    "ageGating": {
-     "$ref": "VideoAgeGating",
-     "description": "Age restriction details related to a video. This data can only be retrieved by the video owner."
-    },
-    "contentDetails": {
-     "$ref": "VideoContentDetails",
-     "description": "The contentDetails object contains information about the video content, including the length of the video and its aspect ratio."
-    },
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "fileDetails": {
-     "$ref": "VideoFileDetails",
-     "description": "The fileDetails object encapsulates information about the video file that was uploaded to YouTube, including the file's resolution, duration, audio and video codecs, stream bitrates, and more. This data can only be retrieved by the video owner."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the video.",
-     "annotations": {
-      "required": [
-       "youtube.videos.update"
-      ]
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#video\".",
-     "default": "youtube#video"
-    },
-    "liveStreamingDetails": {
-     "$ref": "VideoLiveStreamingDetails",
-     "description": "The liveStreamingDetails object contains metadata about a live video broadcast. The object will only be present in a video resource if the video is an upcoming, live, or completed live broadcast."
-    },
-    "localizations": {
-     "type": "object",
-     "description": "List with all localizations.",
-     "additionalProperties": {
-      "$ref": "VideoLocalization",
-      "description": "The language tag, using string since map_key require simple types."
-     }
-    },
-    "monetizationDetails": {
-     "$ref": "VideoMonetizationDetails",
-     "description": "The monetizationDetails object encapsulates information about the monetization status of the video."
-    },
-    "player": {
-     "$ref": "VideoPlayer",
-     "description": "The player object contains information that you would use to play the video in an embedded player."
-    },
-    "processingDetails": {
-     "$ref": "VideoProcessingDetails",
-     "description": "The processingProgress object encapsulates information about YouTube's progress in processing the uploaded video file. The properties in the object identify the current processing status and an estimate of the time remaining until YouTube finishes processing the video. This part also indicates whether different types of data or content, such as file details or thumbnail images, are available for the video.\n\nThe processingProgress object is designed to be polled so that the video uploaded can track the progress that YouTube has made in processing the uploaded video file. This data can only be retrieved by the video owner."
-    },
-    "projectDetails": {
-     "$ref": "VideoProjectDetails",
-     "description": "The projectDetails object contains information about the project specific video metadata."
-    },
-    "recordingDetails": {
-     "$ref": "VideoRecordingDetails",
-     "description": "The recordingDetails object encapsulates information about the location, date and address where the video was recorded."
-    },
-    "snippet": {
-     "$ref": "VideoSnippet",
-     "description": "The snippet object contains basic details about the video, such as its title, description, and category."
-    },
-    "statistics": {
-     "$ref": "VideoStatistics",
-     "description": "The statistics object contains statistics about the video."
-    },
-    "status": {
-     "$ref": "VideoStatus",
-     "description": "The status object contains information about the video's uploading, processing, and privacy statuses."
-    },
-    "suggestions": {
-     "$ref": "VideoSuggestions",
-     "description": "The suggestions object encapsulates suggestions that identify opportunities to improve the video quality or the metadata for the uploaded video. This data can only be retrieved by the video owner."
-    },
-    "topicDetails": {
-     "$ref": "VideoTopicDetails",
-     "description": "The topicDetails object encapsulates information about Freebase topics associated with the video."
-    }
-   }
-  },
-  "VideoAbuseReport": {
-   "id": "VideoAbuseReport",
-   "type": "object",
-   "properties": {
-    "comments": {
-     "type": "string",
-     "description": "Additional comments regarding the abuse report."
-    },
-    "language": {
-     "type": "string",
-     "description": "The language that the content was viewed in."
-    },
-    "reasonId": {
-     "type": "string",
-     "description": "The high-level, or primary, reason that the content is abusive. The value is an abuse report reason ID."
-    },
-    "secondaryReasonId": {
-     "type": "string",
-     "description": "The specific, or secondary, reason that this content is abusive (if available). The value is an abuse report reason ID that is a valid secondary reason for the primary reason."
-    },
-    "videoId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the video."
-    }
-   }
-  },
-  "VideoAbuseReportReason": {
-   "id": "VideoAbuseReportReason",
-   "type": "object",
-   "description": "A videoAbuseReportReason resource identifies a reason that a video could be reported as abusive. Video abuse report reasons are used with video.ReportAbuse.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID of this abuse report reason."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#videoAbuseReportReason\".",
-     "default": "youtube#videoAbuseReportReason"
-    },
-    "snippet": {
-     "$ref": "VideoAbuseReportReasonSnippet",
-     "description": "The snippet object contains basic details about the abuse report reason."
-    }
-   }
-  },
-  "VideoAbuseReportReasonListResponse": {
-   "id": "VideoAbuseReportReasonListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of valid abuse reasons that are used with video.ReportAbuse.",
-     "items": {
-      "$ref": "VideoAbuseReportReason"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#videoAbuseReportReasonListResponse\".",
-     "default": "youtube#videoAbuseReportReasonListResponse"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "VideoAbuseReportReasonSnippet": {
-   "id": "VideoAbuseReportReasonSnippet",
-   "type": "object",
-   "description": "Basic details about a video category, such as its localized title.",
-   "properties": {
-    "label": {
-     "type": "string",
-     "description": "The localized label belonging to this abuse report reason."
-    },
-    "secondaryReasons": {
-     "type": "array",
-     "description": "The secondary reasons associated with this reason, if any are available. (There might be 0 or more.)",
-     "items": {
-      "$ref": "VideoAbuseReportSecondaryReason"
-     }
-    }
-   }
-  },
-  "VideoAbuseReportSecondaryReason": {
-   "id": "VideoAbuseReportSecondaryReason",
-   "type": "object",
-   "properties": {
-    "id": {
-     "type": "string",
-     "description": "The ID of this abuse report secondary reason."
-    },
-    "label": {
-     "type": "string",
-     "description": "The localized label for this abuse report secondary reason."
-    }
-   }
-  },
-  "VideoAgeGating": {
-   "id": "VideoAgeGating",
-   "type": "object",
-   "properties": {
-    "alcoholContent": {
-     "type": "boolean",
-     "description": "Indicates whether or not the video has alcoholic beverage content. Only users of legal purchasing age in a particular country, as identified by ICAP, can view the content."
-    },
-    "restricted": {
-     "type": "boolean",
-     "description": "Age-restricted trailers. For redband trailers and adult-rated video-games. Only users aged 18+ can view the content. The the field is true the content is restricted to viewers aged 18+. Otherwise The field won't be present."
-    },
-    "videoGameRating": {
-     "type": "string",
-     "description": "Video game rating, if any.",
-     "enum": [
-      "anyone",
-      "m15Plus",
-      "m16Plus",
-      "m17Plus"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "VideoCategory": {
-   "id": "VideoCategory",
-   "type": "object",
-   "description": "A videoCategory resource identifies a category that has been or could be associated with uploaded videos.",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "id": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the video category."
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#videoCategory\".",
-     "default": "youtube#videoCategory"
-    },
-    "snippet": {
-     "$ref": "VideoCategorySnippet",
-     "description": "The snippet object contains basic details about the video category, including its title."
-    }
-   }
-  },
-  "VideoCategoryListResponse": {
-   "id": "VideoCategoryListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of video categories that can be associated with YouTube videos. In this map, the video category ID is the map key, and its value is the corresponding videoCategory resource.",
-     "items": {
-      "$ref": "VideoCategory"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#videoCategoryListResponse\".",
-     "default": "youtube#videoCategoryListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "prevPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the previous page in the result set."
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "VideoCategorySnippet": {
-   "id": "VideoCategorySnippet",
-   "type": "object",
-   "description": "Basic details about a video category, such as its localized title.",
-   "properties": {
-    "assignable": {
-     "type": "boolean"
-    },
-    "channelId": {
-     "type": "string",
-     "description": "The YouTube channel that created the video category.",
-     "default": "UCBR8-60-B28hp2BmDPdntcQ"
-    },
-    "title": {
-     "type": "string",
-     "description": "The video category's title."
-    }
-   }
-  },
-  "VideoContentDetails": {
-   "id": "VideoContentDetails",
-   "type": "object",
-   "description": "Details about the content of a YouTube Video.",
-   "properties": {
-    "caption": {
-     "type": "string",
-     "description": "The value of captions indicates whether the video has captions or not.",
-     "enum": [
-      "false",
-      "true"
-     ],
-     "enumDescriptions": [
-      "",
-      ""
-     ]
-    },
-    "contentRating": {
-     "$ref": "ContentRating",
-     "description": "Specifies the ratings that the video received under various rating schemes."
-    },
-    "countryRestriction": {
-     "$ref": "AccessPolicy",
-     "description": "The countryRestriction object contains information about the countries where a video is (or is not) viewable."
-    },
-    "definition": {
-     "type": "string",
-     "description": "The value of definition indicates whether the video is available in high definition or only in standard definition.",
-     "enum": [
-      "hd",
-      "sd"
-     ],
-     "enumDescriptions": [
-      "",
-      ""
-     ]
-    },
-    "dimension": {
-     "type": "string",
-     "description": "The value of dimension indicates whether the video is available in 3D or in 2D."
-    },
-    "duration": {
-     "type": "string",
-     "description": "The length of the video. The tag value is an ISO 8601 duration in the format PT#M#S, in which the letters PT indicate that the value specifies a period of time, and the letters M and S refer to length in minutes and seconds, respectively. The # characters preceding the M and S letters are both integers that specify the number of minutes (or seconds) of the video. For example, a value of PT15M51S indicates that the video is 15 minutes and 51 seconds long."
-    },
-    "hasCustomThumbnail": {
-     "type": "boolean",
-     "description": "Indicates whether the video uploader has provided a custom thumbnail image for the video. This property is only visible to the video uploader."
-    },
-    "licensedContent": {
-     "type": "boolean",
-     "description": "The value of is_license_content indicates whether the video is licensed content."
-    },
-    "projection": {
-     "type": "string",
-     "description": "Specifies the projection format of the video.",
-     "enum": [
-      "360",
-      "rectangular"
-     ],
-     "enumDescriptions": [
-      "",
-      ""
-     ]
-    },
-    "regionRestriction": {
-     "$ref": "VideoContentDetailsRegionRestriction",
-     "description": "The regionRestriction object contains information about the countries where a video is (or is not) viewable. The object will contain either the contentDetails.regionRestriction.allowed property or the contentDetails.regionRestriction.blocked property."
-    }
-   }
-  },
-  "VideoContentDetailsRegionRestriction": {
-   "id": "VideoContentDetailsRegionRestriction",
-   "type": "object",
-   "description": "DEPRECATED Region restriction of the video.",
-   "properties": {
-    "allowed": {
-     "type": "array",
-     "description": "A list of region codes that identify countries where the video is viewable. If this property is present and a country is not listed in its value, then the video is blocked from appearing in that country. If this property is present and contains an empty list, the video is blocked in all countries.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "blocked": {
-     "type": "array",
-     "description": "A list of region codes that identify countries where the video is blocked. If this property is present and a country is not listed in its value, then the video is viewable in that country. If this property is present and contains an empty list, the video is viewable in all countries.",
-     "items": {
-      "type": "string"
-     }
-    }
-   }
-  },
-  "VideoFileDetails": {
-   "id": "VideoFileDetails",
-   "type": "object",
-   "description": "Describes original video file properties, including technical details about audio and video streams, but also metadata information like content length, digitization time, or geotagging information.",
-   "properties": {
-    "audioStreams": {
-     "type": "array",
-     "description": "A list of audio streams contained in the uploaded video file. Each item in the list contains detailed metadata about an audio stream.",
-     "items": {
-      "$ref": "VideoFileDetailsAudioStream"
-     }
-    },
-    "bitrateBps": {
-     "type": "string",
-     "description": "The uploaded video file's combined (video and audio) bitrate in bits per second.",
-     "format": "uint64"
-    },
-    "container": {
-     "type": "string",
-     "description": "The uploaded video file's container format."
-    },
-    "creationTime": {
-     "type": "string",
-     "description": "The date and time when the uploaded video file was created. The value is specified in ISO 8601 format. Currently, the following ISO 8601 formats are supported:  \n- Date only: YYYY-MM-DD \n- Naive time: YYYY-MM-DDTHH:MM:SS \n- Time with timezone: YYYY-MM-DDTHH:MM:SS+HH:MM"
-    },
-    "durationMs": {
-     "type": "string",
-     "description": "The length of the uploaded video in milliseconds.",
-     "format": "uint64"
-    },
-    "fileName": {
-     "type": "string",
-     "description": "The uploaded file's name. This field is present whether a video file or another type of file was uploaded."
-    },
-    "fileSize": {
-     "type": "string",
-     "description": "The uploaded file's size in bytes. This field is present whether a video file or another type of file was uploaded.",
-     "format": "uint64"
-    },
-    "fileType": {
-     "type": "string",
-     "description": "The uploaded file's type as detected by YouTube's video processing engine. Currently, YouTube only processes video files, but this field is present whether a video file or another type of file was uploaded.",
-     "enum": [
-      "archive",
-      "audio",
-      "document",
-      "image",
-      "other",
-      "project",
-      "video"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "videoStreams": {
-     "type": "array",
-     "description": "A list of video streams contained in the uploaded video file. Each item in the list contains detailed metadata about a video stream.",
-     "items": {
-      "$ref": "VideoFileDetailsVideoStream"
-     }
-    }
-   }
-  },
-  "VideoFileDetailsAudioStream": {
-   "id": "VideoFileDetailsAudioStream",
-   "type": "object",
-   "description": "Information about an audio stream.",
-   "properties": {
-    "bitrateBps": {
-     "type": "string",
-     "description": "The audio stream's bitrate, in bits per second.",
-     "format": "uint64"
-    },
-    "channelCount": {
-     "type": "integer",
-     "description": "The number of audio channels that the stream contains.",
-     "format": "uint32"
-    },
-    "codec": {
-     "type": "string",
-     "description": "The audio codec that the stream uses."
-    },
-    "vendor": {
-     "type": "string",
-     "description": "A value that uniquely identifies a video vendor. Typically, the value is a four-letter vendor code."
-    }
-   }
-  },
-  "VideoFileDetailsVideoStream": {
-   "id": "VideoFileDetailsVideoStream",
-   "type": "object",
-   "description": "Information about a video stream.",
-   "properties": {
-    "aspectRatio": {
-     "type": "number",
-     "description": "The video content's display aspect ratio, which specifies the aspect ratio in which the video should be displayed.",
-     "format": "double"
-    },
-    "bitrateBps": {
-     "type": "string",
-     "description": "The video stream's bitrate, in bits per second.",
-     "format": "uint64"
-    },
-    "codec": {
-     "type": "string",
-     "description": "The video codec that the stream uses."
-    },
-    "frameRateFps": {
-     "type": "number",
-     "description": "The video stream's frame rate, in frames per second.",
-     "format": "double"
-    },
-    "heightPixels": {
-     "type": "integer",
-     "description": "The encoded video content's height in pixels.",
-     "format": "uint32"
-    },
-    "rotation": {
-     "type": "string",
-     "description": "The amount that YouTube needs to rotate the original source content to properly display the video.",
-     "enum": [
-      "clockwise",
-      "counterClockwise",
-      "none",
-      "other",
-      "upsideDown"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "vendor": {
-     "type": "string",
-     "description": "A value that uniquely identifies a video vendor. Typically, the value is a four-letter vendor code."
-    },
-    "widthPixels": {
-     "type": "integer",
-     "description": "The encoded video content's width in pixels. You can calculate the video's encoding aspect ratio as width_pixels / height_pixels.",
-     "format": "uint32"
-    }
-   }
-  },
-  "VideoGetRatingResponse": {
-   "id": "VideoGetRatingResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of ratings that match the request criteria.",
-     "items": {
-      "$ref": "VideoRating"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#videoGetRatingResponse\".",
-     "default": "youtube#videoGetRatingResponse"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "VideoListResponse": {
-   "id": "VideoListResponse",
-   "type": "object",
-   "properties": {
-    "etag": {
-     "type": "string",
-     "description": "Etag of this resource."
-    },
-    "eventId": {
-     "type": "string",
-     "description": "Serialized EventId of the request which produced this response."
-    },
-    "items": {
-     "type": "array",
-     "description": "A list of videos that match the request criteria.",
-     "items": {
-      "$ref": "Video"
-     }
-    },
-    "kind": {
-     "type": "string",
-     "description": "Identifies what kind of resource this is. Value: the fixed string \"youtube#videoListResponse\".",
-     "default": "youtube#videoListResponse"
-    },
-    "nextPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the next page in the result set."
-    },
-    "pageInfo": {
-     "$ref": "PageInfo"
-    },
-    "prevPageToken": {
-     "type": "string",
-     "description": "The token that can be used as the value of the pageToken parameter to retrieve the previous page in the result set."
-    },
-    "tokenPagination": {
-     "$ref": "TokenPagination"
-    },
-    "visitorId": {
-     "type": "string",
-     "description": "The visitorId identifies the visitor."
-    }
-   }
-  },
-  "VideoLiveStreamingDetails": {
-   "id": "VideoLiveStreamingDetails",
-   "type": "object",
-   "description": "Details about the live streaming metadata.",
-   "properties": {
-    "activeLiveChatId": {
-     "type": "string",
-     "description": "The ID of the currently active live chat attached to this video. This field is filled only if the video is a currently live broadcast that has live chat. Once the broadcast transitions to complete this field will be removed and the live chat closed down. For persistent broadcasts that live chat id will no longer be tied to this video but rather to the new video being displayed at the persistent page."
-    },
-    "actualEndTime": {
-     "type": "string",
-     "description": "The time that the broadcast actually ended. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format. This value will not be available until the broadcast is over.",
-     "format": "date-time"
-    },
-    "actualStartTime": {
-     "type": "string",
-     "description": "The time that the broadcast actually started. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format. This value will not be available until the broadcast begins.",
-     "format": "date-time"
-    },
-    "concurrentViewers": {
-     "type": "string",
-     "description": "The number of viewers currently watching the broadcast. The property and its value will be present if the broadcast has current viewers and the broadcast owner has not hidden the viewcount for the video. Note that YouTube stops tracking the number of concurrent viewers for a broadcast when the broadcast ends. So, this property would not identify the number of viewers watching an archived video of a live broadcast that already ended.",
-     "format": "uint64"
-    },
-    "scheduledEndTime": {
-     "type": "string",
-     "description": "The time that the broadcast is scheduled to end. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format. If the value is empty or the property is not present, then the broadcast is scheduled to continue indefinitely.",
-     "format": "date-time"
-    },
-    "scheduledStartTime": {
-     "type": "string",
-     "description": "The time that the broadcast is scheduled to begin. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    }
-   }
-  },
-  "VideoLocalization": {
-   "id": "VideoLocalization",
-   "type": "object",
-   "description": "Localized versions of certain video properties (e.g. title).",
-   "properties": {
-    "description": {
-     "type": "string",
-     "description": "Localized version of the video's description."
-    },
-    "title": {
-     "type": "string",
-     "description": "Localized version of the video's title."
-    }
-   }
-  },
-  "VideoMonetizationDetails": {
-   "id": "VideoMonetizationDetails",
-   "type": "object",
-   "description": "Details about monetization of a YouTube Video.",
-   "properties": {
-    "access": {
-     "$ref": "AccessPolicy",
-     "description": "The value of access indicates whether the video can be monetized or not."
-    }
-   }
-  },
-  "VideoPlayer": {
-   "id": "VideoPlayer",
-   "type": "object",
-   "description": "Player to be used for a video playback.",
-   "properties": {
-    "embedHeight": {
-     "type": "string",
-     "format": "int64"
-    },
-    "embedHtml": {
-     "type": "string",
-     "description": "An \u003ciframe\u003e tag that embeds a player that will play the video."
-    },
-    "embedWidth": {
-     "type": "string",
-     "description": "The embed width",
-     "format": "int64"
-    }
-   }
-  },
-  "VideoProcessingDetails": {
-   "id": "VideoProcessingDetails",
-   "type": "object",
-   "description": "Describes processing status and progress and availability of some other Video resource parts.",
-   "properties": {
-    "editorSuggestionsAvailability": {
-     "type": "string",
-     "description": "This value indicates whether video editing suggestions, which might improve video quality or the playback experience, are available for the video. You can retrieve these suggestions by requesting the suggestions part in your videos.list() request."
-    },
-    "fileDetailsAvailability": {
-     "type": "string",
-     "description": "This value indicates whether file details are available for the uploaded video. You can retrieve a video's file details by requesting the fileDetails part in your videos.list() request."
-    },
-    "processingFailureReason": {
-     "type": "string",
-     "description": "The reason that YouTube failed to process the video. This property will only have a value if the processingStatus property's value is failed.",
-     "enum": [
-      "other",
-      "streamingFailed",
-      "transcodeFailed",
-      "uploadFailed"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "processingIssuesAvailability": {
-     "type": "string",
-     "description": "This value indicates whether the video processing engine has generated suggestions that might improve YouTube's ability to process the the video, warnings that explain video processing problems, or errors that cause video processing problems. You can retrieve these suggestions by requesting the suggestions part in your videos.list() request."
-    },
-    "processingProgress": {
-     "$ref": "VideoProcessingDetailsProcessingProgress",
-     "description": "The processingProgress object contains information about the progress YouTube has made in processing the video. The values are really only relevant if the video's processing status is processing."
-    },
-    "processingStatus": {
-     "type": "string",
-     "description": "The video's processing status. This value indicates whether YouTube was able to process the video or if the video is still being processed.",
-     "enum": [
-      "failed",
-      "processing",
-      "succeeded",
-      "terminated"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "tagSuggestionsAvailability": {
-     "type": "string",
-     "description": "This value indicates whether keyword (tag) suggestions are available for the video. Tags can be added to a video's metadata to make it easier for other users to find the video. You can retrieve these suggestions by requesting the suggestions part in your videos.list() request."
-    },
-    "thumbnailsAvailability": {
-     "type": "string",
-     "description": "This value indicates whether thumbnail images have been generated for the video."
-    }
-   }
-  },
-  "VideoProcessingDetailsProcessingProgress": {
-   "id": "VideoProcessingDetailsProcessingProgress",
-   "type": "object",
-   "description": "Video processing progress and completion time estimate.",
-   "properties": {
-    "partsProcessed": {
-     "type": "string",
-     "description": "The number of parts of the video that YouTube has already processed. You can estimate the percentage of the video that YouTube has already processed by calculating:\n100 * parts_processed / parts_total\n\nNote that since the estimated number of parts could increase without a corresponding increase in the number of parts that have already been processed, it is possible that the calculated progress could periodically decrease while YouTube processes a video.",
-     "format": "uint64"
-    },
-    "partsTotal": {
-     "type": "string",
-     "description": "An estimate of the total number of parts that need to be processed for the video. The number may be updated with more precise estimates while YouTube processes the video.",
-     "format": "uint64"
-    },
-    "timeLeftMs": {
-     "type": "string",
-     "description": "An estimate of the amount of time, in millseconds, that YouTube needs to finish processing the video.",
-     "format": "uint64"
-    }
-   }
-  },
-  "VideoProjectDetails": {
-   "id": "VideoProjectDetails",
-   "type": "object",
-   "description": "Project specific details about the content of a YouTube Video.",
-   "properties": {
-    "tags": {
-     "type": "array",
-     "description": "A list of project tags associated with the video during the upload.",
-     "items": {
-      "type": "string"
-     }
-    }
-   }
-  },
-  "VideoRating": {
-   "id": "VideoRating",
-   "type": "object",
-   "properties": {
-    "rating": {
-     "type": "string",
-     "enum": [
-      "dislike",
-      "like",
-      "none",
-      "unspecified"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "videoId": {
-     "type": "string"
-    }
-   }
-  },
-  "VideoRecordingDetails": {
-   "id": "VideoRecordingDetails",
-   "type": "object",
-   "description": "Recording information associated with the video.",
-   "properties": {
-    "location": {
-     "$ref": "GeoPoint",
-     "description": "The geolocation information associated with the video."
-    },
-    "locationDescription": {
-     "type": "string",
-     "description": "The text description of the location where the video was recorded."
-    },
-    "recordingDate": {
-     "type": "string",
-     "description": "The date and time when the video was recorded. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sssZ) format.",
-     "format": "date-time"
-    }
-   }
-  },
-  "VideoSnippet": {
-   "id": "VideoSnippet",
-   "type": "object",
-   "description": "Basic details about a video, including title, description, uploader, thumbnails and category.",
-   "properties": {
-    "categoryId": {
-     "type": "string",
-     "description": "The YouTube video category associated with the video."
-    },
-    "channelId": {
-     "type": "string",
-     "description": "The ID that YouTube uses to uniquely identify the channel that the video was uploaded to."
-    },
-    "channelTitle": {
-     "type": "string",
-     "description": "Channel title for the channel that the video belongs to."
-    },
-    "defaultAudioLanguage": {
-     "type": "string",
-     "description": "The default_audio_language property specifies the language spoken in the video's default audio track."
-    },
-    "defaultLanguage": {
-     "type": "string",
-     "description": "The language of the videos's default snippet."
-    },
-    "description": {
-     "type": "string",
-     "description": "The video's description."
-    },
-    "liveBroadcastContent": {
-     "type": "string",
-     "description": "Indicates if the video is an upcoming/active live broadcast. Or it's \"none\" if the video is not an upcoming/active live broadcast.",
-     "enum": [
-      "live",
-      "none",
-      "upcoming"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    },
-    "localized": {
-     "$ref": "VideoLocalization",
-     "description": "Localized snippet selected with the hl parameter. If no such localization exists, this field is populated with the default snippet. (Read-only)"
-    },
-    "publishedAt": {
-     "type": "string",
-     "description": "The date and time that the video was uploaded. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "tags": {
-     "type": "array",
-     "description": "A list of keyword tags associated with the video. Tags may contain spaces.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "thumbnails": {
-     "$ref": "ThumbnailDetails",
-     "description": "A map of thumbnail images associated with the video. For each object in the map, the key is the name of the thumbnail image, and the value is an object that contains other information about the thumbnail."
-    },
-    "title": {
-     "type": "string",
-     "description": "The video's title."
-    }
-   }
-  },
-  "VideoStatistics": {
-   "id": "VideoStatistics",
-   "type": "object",
-   "description": "Statistics about the video, such as the number of times the video was viewed or liked.",
-   "properties": {
-    "commentCount": {
-     "type": "string",
-     "description": "The number of comments for the video.",
-     "format": "uint64"
-    },
-    "dislikeCount": {
-     "type": "string",
-     "description": "The number of users who have indicated that they disliked the video by giving it a negative rating.",
-     "format": "uint64"
-    },
-    "favoriteCount": {
-     "type": "string",
-     "description": "The number of users who currently have the video marked as a favorite video.",
-     "format": "uint64"
-    },
-    "likeCount": {
-     "type": "string",
-     "description": "The number of users who have indicated that they liked the video by giving it a positive rating.",
-     "format": "uint64"
-    },
-    "viewCount": {
-     "type": "string",
-     "description": "The number of times the video has been viewed.",
-     "format": "uint64"
-    }
-   }
-  },
-  "VideoStatus": {
-   "id": "VideoStatus",
-   "type": "object",
-   "description": "Basic details about a video category, such as its localized title.",
-   "properties": {
-    "embeddable": {
-     "type": "boolean",
-     "description": "This value indicates if the video can be embedded on another website."
-    },
-    "failureReason": {
-     "type": "string",
-     "description": "This value explains why a video failed to upload. This property is only present if the uploadStatus property indicates that the upload failed.",
-     "enum": [
-      "codec",
-      "conversion",
-      "emptyFile",
-      "invalidFile",
-      "tooSmall",
-      "uploadAborted"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "license": {
-     "type": "string",
-     "description": "The video's license.",
-     "enum": [
-      "creativeCommon",
-      "youtube"
-     ],
-     "enumDescriptions": [
-      "",
-      ""
-     ]
-    },
-    "privacyStatus": {
-     "type": "string",
-     "description": "The video's privacy status.",
-     "enum": [
-      "private",
-      "public",
-      "unlisted"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      ""
-     ]
-    },
-    "publicStatsViewable": {
-     "type": "boolean",
-     "description": "This value indicates if the extended video statistics on the watch page can be viewed by everyone. Note that the view count, likes, etc will still be visible if this is disabled."
-    },
-    "publishAt": {
-     "type": "string",
-     "description": "The date and time when the video is scheduled to publish. It can be set only if the privacy status of the video is private. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-     "format": "date-time"
-    },
-    "rejectionReason": {
-     "type": "string",
-     "description": "This value explains why YouTube rejected an uploaded video. This property is only present if the uploadStatus property indicates that the upload was rejected.",
-     "enum": [
-      "claim",
-      "copyright",
-      "duplicate",
-      "inappropriate",
-      "legal",
-      "length",
-      "termsOfUse",
-      "trademark",
-      "uploaderAccountClosed",
-      "uploaderAccountSuspended"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    },
-    "uploadStatus": {
-     "type": "string",
-     "description": "The status of the uploaded video.",
-     "enum": [
-      "deleted",
-      "failed",
-      "processed",
-      "rejected",
-      "uploaded"
-     ],
-     "enumDescriptions": [
-      "",
-      "",
-      "",
-      "",
-      ""
-     ]
-    }
-   }
-  },
-  "VideoSuggestions": {
-   "id": "VideoSuggestions",
-   "type": "object",
-   "description": "Specifies suggestions on how to improve video content, including encoding hints, tag suggestions, and editor suggestions.",
-   "properties": {
-    "editorSuggestions": {
-     "type": "array",
-     "description": "A list of video editing operations that might improve the video quality or playback experience of the uploaded video.",
-     "items": {
-      "type": "string",
-      "enum": [
-       "audioQuietAudioSwap",
-       "videoAutoLevels",
-       "videoCrop",
-       "videoStabilize"
-      ],
-      "enumDescriptions": [
-       "",
-       "",
-       "",
-       ""
-      ]
-     }
-    },
-    "processingErrors": {
-     "type": "array",
-     "description": "A list of errors that will prevent YouTube from successfully processing the uploaded video video. These errors indicate that, regardless of the video's current processing status, eventually, that status will almost certainly be failed.",
-     "items": {
-      "type": "string",
-      "enum": [
-       "archiveFile",
-       "audioFile",
-       "docFile",
-       "imageFile",
-       "notAVideoFile",
-       "projectFile",
-       "unsupportedSpatialAudioLayout"
-      ],
-      "enumDescriptions": [
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       ""
-      ]
-     }
-    },
-    "processingHints": {
-     "type": "array",
-     "description": "A list of suggestions that may improve YouTube's ability to process the video.",
-     "items": {
-      "type": "string",
-      "enum": [
-       "nonStreamableMov",
-       "sendBestQualityVideo",
-       "spatialAudio",
-       "sphericalVideo",
-       "vrVideo"
-      ],
-      "enumDescriptions": [
-       "",
-       "",
-       "",
-       "",
-       ""
-      ]
-     }
-    },
-    "processingWarnings": {
-     "type": "array",
-     "description": "A list of reasons why YouTube may have difficulty transcoding the uploaded video or that might result in an erroneous transcoding. These warnings are generated before YouTube actually processes the uploaded video file. In addition, they identify issues that are unlikely to cause the video processing to fail but that might cause problems such as sync issues, video artifacts, or a missing audio track.",
-     "items": {
-      "type": "string",
-      "enum": [
-       "hasEditlist",
-       "inconsistentResolution",
-       "problematicAudioCodec",
-       "problematicVideoCodec",
-       "unknownAudioCodec",
-       "unknownContainer",
-       "unknownVideoCodec",
-       "unsupportedSphericalProjectionType",
-       "unsupportedVrStereoMode"
-      ],
-      "enumDescriptions": [
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       "",
-       ""
-      ]
-     }
-    },
-    "tagSuggestions": {
-     "type": "array",
-     "description": "A list of keyword tags that could be added to the video's metadata to increase the likelihood that users will locate your video when searching or browsing on YouTube.",
-     "items": {
-      "$ref": "VideoSuggestionsTagSuggestion"
-     }
-    }
-   }
-  },
-  "VideoSuggestionsTagSuggestion": {
-   "id": "VideoSuggestionsTagSuggestion",
-   "type": "object",
-   "description": "A single tag suggestion with it's relevance information.",
-   "properties": {
-    "categoryRestricts": {
-     "type": "array",
-     "description": "A set of video categories for which the tag is relevant. You can use this information to display appropriate tag suggestions based on the video category that the video uploader associates with the video. By default, tag suggestions are relevant for all categories if there are no restricts defined for the keyword.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "tag": {
-     "type": "string",
-     "description": "The keyword tag suggested for the video."
-    }
-   }
-  },
-  "VideoTopicDetails": {
-   "id": "VideoTopicDetails",
-   "type": "object",
-   "description": "Freebase topic information related to the video.",
-   "properties": {
-    "relevantTopicIds": {
-     "type": "array",
-     "description": "Similar to topic_id, except that these topics are merely relevant to the video. These are topics that may be mentioned in, or appear in the video. You can retrieve information about each topic using Freebase Topic API.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "topicCategories": {
-     "type": "array",
-     "description": "A list of Wikipedia URLs that provide a high-level description of the video's content.",
-     "items": {
-      "type": "string"
-     }
-    },
-    "topicIds": {
-     "type": "array",
-     "description": "A list of Freebase topic IDs that are centrally associated with the video. These are topics that are centrally featured in the video, and it can be said that the video is mainly about each of these. You can retrieve information about each topic using the Freebase Topic API.",
-     "items": {
-      "type": "string"
-     }
-    }
-   }
-  },
-  "WatchSettings": {
-   "id": "WatchSettings",
-   "type": "object",
-   "description": "Branding properties for the watch. All deprecated.",
-   "properties": {
-    "backgroundColor": {
-     "type": "string",
-     "description": "The text color for the video watch page's branded area."
-    },
-    "featuredPlaylistId": {
-     "type": "string",
-     "description": "An ID that uniquely identifies a playlist that displays next to the video player."
-    },
-    "textColor": {
-     "type": "string",
-     "description": "The background color for the video watch page's branded area."
-    }
-   }
-  }
- },
- "resources": {
-  "activities": {
-   "methods": {
-    "insert": {
-     "id": "youtube.activities.insert",
-     "path": "activities",
-     "httpMethod": "POST",
-     "description": "Posts a bulletin for a specific channel. (The user submitting the request must be authorized to act on the channel's behalf.)\n\nNote: Even though an activity resource can contain information about actions like a user rating a video or marking a video as a favorite, you need to use other API methods to generate those activity resources. For example, you would use the API's videos.rate() method to rate a video and the playlistItems.insert() method to mark a video as a favorite.",
-     "parameters": {
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "Activity"
-     },
-     "response": {
-      "$ref": "Activity"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "list": {
-     "id": "youtube.activities.list",
-     "path": "activities",
-     "httpMethod": "GET",
-     "description": "Returns a list of channel activity events that match the request criteria. For example, you can retrieve events associated with a particular channel, events associated with the user's subscriptions and Google+ friends, or the YouTube home page feed, which is customized for each user.",
-     "parameters": {
-      "channelId": {
-       "type": "string",
-       "description": "The channelId parameter specifies a unique YouTube channel ID. The API will then return a list of that channel's activities.",
-       "location": "query"
-      },
-      "home": {
-       "type": "boolean",
-       "description": "Set this parameter's value to true to retrieve the activity feed that displays on the YouTube home page for the currently authenticated user.",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "0",
-       "maximum": "50",
-       "location": "query"
-      },
-      "mine": {
-       "type": "boolean",
-       "description": "Set this parameter's value to true to retrieve a feed of the authenticated user's activities.",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more activity resource properties that the API response will include.\n\nIf the parameter identifies a property that contains child properties, the child properties will be included in the response. For example, in an activity resource, the snippet property contains other properties that identify the type of activity, a display title for the activity, and so forth. If you set part=snippet, the API response will also contain all of those nested properties.",
-       "required": true,
-       "location": "query"
-      },
-      "publishedAfter": {
-       "type": "string",
-       "description": "The publishedAfter parameter specifies the earliest date and time that an activity could have occurred for that activity to be included in the API response. If the parameter value specifies a day, but not a time, then any activities that occurred that day will be included in the result set. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-       "format": "date-time",
-       "location": "query"
-      },
-      "publishedBefore": {
-       "type": "string",
-       "description": "The publishedBefore parameter specifies the date and time before which an activity must have occurred for that activity to be included in the API response. If the parameter value specifies a day, but not a time, then any activities that occurred that day will be excluded from the result set. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sZ) format.",
-       "format": "date-time",
-       "location": "query"
-      },
-      "regionCode": {
-       "type": "string",
-       "description": "The regionCode parameter instructs the API to return results for the specified country. The parameter value is an ISO 3166-1 alpha-2 country code. YouTube uses this value when the authorized user's previous activity on YouTube does not provide enough information to generate the activity feed.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "ActivityListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly"
-     ]
-    }
-   }
-  },
-  "captions": {
-   "methods": {
-    "delete": {
-     "id": "youtube.captions.delete",
-     "path": "captions",
-     "httpMethod": "DELETE",
-     "description": "Deletes a specified caption track.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter identifies the caption track that is being deleted. The value is a caption track ID as identified by the id property in a caption resource.",
-       "required": true,
-       "location": "query"
-      },
-      "onBehalfOf": {
-       "type": "string",
-       "description": "ID of the Google+ Page for the channel that the request is be on behalf of",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The actual CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "download": {
-     "id": "youtube.captions.download",
-     "path": "captions/{id}",
-     "httpMethod": "GET",
-     "description": "Downloads a caption track. The caption track is returned in its original format unless the request specifies a value for the tfmt parameter and in its original language unless the request specifies a value for the tlang parameter.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter identifies the caption track that is being retrieved. The value is a caption track ID as identified by the id property in a caption resource.",
-       "required": true,
-       "location": "path"
-      },
-      "onBehalfOf": {
-       "type": "string",
-       "description": "ID of the Google+ Page for the channel that the request is be on behalf of",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The actual CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "tfmt": {
-       "type": "string",
-       "description": "The tfmt parameter specifies that the caption track should be returned in a specific format. If the parameter is not included in the request, the track is returned in its original format.",
-       "enum": [
-        "sbv",
-        "scc",
-        "srt",
-        "ttml",
-        "vtt"
-       ],
-       "enumDescriptions": [
-        "SubViewer subtitle.",
-        "Scenarist Closed Caption format.",
-        "SubRip subtitle.",
-        "Timed Text Markup Language caption.",
-        "Web Video Text Tracks caption."
-       ],
-       "location": "query"
-      },
-      "tlang": {
-       "type": "string",
-       "description": "The tlang parameter specifies that the API response should return a translation of the specified caption track. The parameter value is an ISO 639-1 two-letter language code that identifies the desired caption language. The translation is generated by using machine translation, such as Google Translate.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ],
-     "supportsMediaDownload": true
-    },
-    "insert": {
-     "id": "youtube.captions.insert",
-     "path": "captions",
-     "httpMethod": "POST",
-     "description": "Uploads a caption track.",
-     "parameters": {
-      "onBehalfOf": {
-       "type": "string",
-       "description": "ID of the Google+ Page for the channel that the request is be on behalf of",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The actual CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies the caption resource parts that the API response will include. Set the parameter value to snippet.",
-       "required": true,
-       "location": "query"
-      },
-      "sync": {
-       "type": "boolean",
-       "description": "The sync parameter indicates whether YouTube should automatically synchronize the caption file with the audio track of the video. If you set the value to true, YouTube will disregard any time codes that are in the uploaded caption file and generate new time codes for the captions.\n\nYou should set the sync parameter to true if you are uploading a transcript, which has no time codes, or if you suspect the time codes in your file are incorrect and want YouTube to try to fix them.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "Caption"
-     },
-     "response": {
-      "$ref": "Caption"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ],
-     "supportsMediaUpload": true,
-     "mediaUpload": {
-      "accept": [
-       "*/*",
-       "application/octet-stream",
-       "text/xml"
-      ],
-      "maxSize": "100MB",
-      "protocols": {
-       "simple": {
-        "multipart": true,
-        "path": "/upload/youtube/v3/captions"
-       },
-       "resumable": {
-        "multipart": true,
-        "path": "/resumable/upload/youtube/v3/captions"
-       }
-      }
-     }
-    },
-    "list": {
-     "id": "youtube.captions.list",
-     "path": "captions",
-     "httpMethod": "GET",
-     "description": "Returns a list of caption tracks that are associated with a specified video. Note that the API response does not contain the actual captions and that the captions.download method provides the ability to retrieve a caption track.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of IDs that identify the caption resources that should be retrieved. Each ID must identify a caption track associated with the specified video.",
-       "location": "query"
-      },
-      "onBehalfOf": {
-       "type": "string",
-       "description": "ID of the Google+ Page for the channel that the request is on behalf of.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The actual CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more caption resource parts that the API response will include. The part names that you can include in the parameter value are id and snippet.",
-       "required": true,
-       "location": "query"
-      },
-      "videoId": {
-       "type": "string",
-       "description": "The videoId parameter specifies the YouTube video ID of the video for which the API should return caption tracks.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part",
-      "videoId"
-     ],
-     "response": {
-      "$ref": "CaptionListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "update": {
-     "id": "youtube.captions.update",
-     "path": "captions",
-     "httpMethod": "PUT",
-     "description": "Updates a caption track. When updating a caption track, you can change the track's draft status, upload a new caption file for the track, or both.",
-     "parameters": {
-      "onBehalfOf": {
-       "type": "string",
-       "description": "ID of the Google+ Page for the channel that the request is be on behalf of",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The actual CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include. Set the property value to snippet if you are updating the track's draft status. Otherwise, set the property value to id.",
-       "required": true,
-       "location": "query"
-      },
-      "sync": {
-       "type": "boolean",
-       "description": "Note: The API server only processes the parameter value if the request contains an updated caption file.\n\nThe sync parameter indicates whether YouTube should automatically synchronize the caption file with the audio track of the video. If you set the value to true, YouTube will automatically synchronize the caption track with the audio track.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "Caption"
-     },
-     "response": {
-      "$ref": "Caption"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ],
-     "supportsMediaUpload": true,
-     "mediaUpload": {
-      "accept": [
-       "*/*",
-       "application/octet-stream",
-       "text/xml"
-      ],
-      "maxSize": "100MB",
-      "protocols": {
-       "simple": {
-        "multipart": true,
-        "path": "/upload/youtube/v3/captions"
-       },
-       "resumable": {
-        "multipart": true,
-        "path": "/resumable/upload/youtube/v3/captions"
-       }
-      }
-     }
-    }
-   }
-  },
-  "channelBanners": {
-   "methods": {
-    "insert": {
-     "id": "youtube.channelBanners.insert",
-     "path": "channelBanners/insert",
-     "httpMethod": "POST",
-     "description": "Uploads a channel banner image to YouTube. This method represents the first two steps in a three-step process to update the banner image for a channel:\n\n- Call the channelBanners.insert method to upload the binary image data to YouTube. The image must have a 16:9 aspect ratio and be at least 2120x1192 pixels.\n- Extract the url property's value from the response that the API returns for step 1.\n- Call the channels.update method to update the channel's branding settings. Set the brandingSettings.image.bannerExternalUrl property's value to the URL obtained in step 2.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      }
-     },
-     "request": {
-      "$ref": "ChannelBannerResource"
-     },
-     "response": {
-      "$ref": "ChannelBannerResource"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.upload"
-     ],
-     "supportsMediaUpload": true,
-     "mediaUpload": {
-      "accept": [
-       "application/octet-stream",
-       "image/jpeg",
-       "image/png"
-      ],
-      "maxSize": "6MB",
-      "protocols": {
-       "simple": {
-        "multipart": true,
-        "path": "/upload/youtube/v3/channelBanners/insert"
-       },
-       "resumable": {
-        "multipart": true,
-        "path": "/resumable/upload/youtube/v3/channelBanners/insert"
-       }
-      }
-     }
-    }
-   }
-  },
-  "channelSections": {
-   "methods": {
-    "delete": {
-     "id": "youtube.channelSections.delete",
-     "path": "channelSections",
-     "httpMethod": "DELETE",
-     "description": "Deletes a channelSection.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the YouTube channelSection ID for the resource that is being deleted. In a channelSection resource, the id property specifies the YouTube channelSection ID.",
-       "required": true,
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "insert": {
-     "id": "youtube.channelSections.insert",
-     "path": "channelSections",
-     "httpMethod": "POST",
-     "description": "Adds a channelSection for the authenticated user's channel.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.\n\nThe part names that you can include in the parameter value are snippet and contentDetails.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "ChannelSection"
-     },
-     "response": {
-      "$ref": "ChannelSection"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "list": {
-     "id": "youtube.channelSections.list",
-     "path": "channelSections",
-     "httpMethod": "GET",
-     "description": "Returns channelSection resources that match the API request criteria.",
-     "parameters": {
-      "channelId": {
-       "type": "string",
-       "description": "The channelId parameter specifies a YouTube channel ID. The API will only return that channel's channelSections.",
-       "location": "query"
-      },
-      "hl": {
-       "type": "string",
-       "description": "The hl parameter indicates that the snippet.localized property values in the returned channelSection resources should be in the specified language if localized values for that language are available. For example, if the API request specifies hl=de, the snippet.localized properties in the API response will contain German titles if German titles are available. Channel owners can provide localized channel section titles using either the channelSections.insert or channelSections.update method.",
-       "location": "query"
-      },
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of the YouTube channelSection ID(s) for the resource(s) that are being retrieved. In a channelSection resource, the id property specifies the YouTube channelSection ID.",
-       "location": "query"
-      },
-      "mine": {
-       "type": "boolean",
-       "description": "Set this parameter's value to true to retrieve a feed of the authenticated user's channelSections.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more channelSection resource properties that the API response will include. The part names that you can include in the parameter value are id, snippet, and contentDetails.\n\nIf the parameter identifies a property that contains child properties, the child properties will be included in the response. For example, in a channelSection resource, the snippet property contains other properties, such as a display title for the channelSection. If you set part=snippet, the API response will also contain all of those nested properties.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "ChannelSectionListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "update": {
-     "id": "youtube.channelSections.update",
-     "path": "channelSections",
-     "httpMethod": "PUT",
-     "description": "Update a channelSection.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.\n\nThe part names that you can include in the parameter value are snippet and contentDetails.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "ChannelSection"
-     },
-     "response": {
-      "$ref": "ChannelSection"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    }
-   }
-  },
-  "channels": {
-   "methods": {
-    "list": {
-     "id": "youtube.channels.list",
-     "path": "channels",
-     "httpMethod": "GET",
-     "description": "Returns a collection of zero or more channel resources that match the request criteria.",
-     "parameters": {
-      "categoryId": {
-       "type": "string",
-       "description": "The categoryId parameter specifies a YouTube guide category, thereby requesting YouTube channels associated with that category.",
-       "location": "query"
-      },
-      "forUsername": {
-       "type": "string",
-       "description": "The forUsername parameter specifies a YouTube username, thereby requesting the channel associated with that username.",
-       "location": "query"
-      },
-      "hl": {
-       "type": "string",
-       "description": "The hl parameter should be used for filter out the properties that are not in the given language. Used for the brandingSettings part.",
-       "location": "query"
-      },
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of the YouTube channel ID(s) for the resource(s) that are being retrieved. In a channel resource, the id property specifies the channel's YouTube channel ID.",
-       "location": "query"
-      },
-      "managedByMe": {
-       "type": "boolean",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nSet this parameter's value to true to instruct the API to only return channels managed by the content owner that the onBehalfOfContentOwner parameter specifies. The user must be authenticated as a CMS account linked to the specified content owner and onBehalfOfContentOwner must be provided.",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "0",
-       "maximum": "50",
-       "location": "query"
-      },
-      "mine": {
-       "type": "boolean",
-       "description": "Set this parameter's value to true to instruct the API to only return channels owned by the authenticated user.",
-       "location": "query"
-      },
-      "mySubscribers": {
-       "type": "boolean",
-       "description": "Use the subscriptions.list method and its mySubscribers parameter to retrieve a list of subscribers to the authenticated user's channel.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more channel resource properties that the API response will include.\n\nIf the parameter identifies a property that contains child properties, the child properties will be included in the response. For example, in a channel resource, the contentDetails property contains other properties, such as the uploads properties. As such, if you set part=contentDetails, the API response will also contain all of those nested properties.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "ChannelListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly",
-      "https://www.googleapis.com/auth/youtubepartner",
-      "https://www.googleapis.com/auth/youtubepartner-channel-audit"
-     ]
-    },
-    "update": {
-     "id": "youtube.channels.update",
-     "path": "channels",
-     "httpMethod": "PUT",
-     "description": "Updates a channel's metadata. Note that this method currently only supports updates to the channel resource's brandingSettings and invideoPromotion objects and their child properties.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "The onBehalfOfContentOwner parameter indicates that the authenticated user is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The actual CMS account that the user authenticates with needs to be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.\n\nThe API currently only allows the parameter value to be set to either brandingSettings or invideoPromotion. (You cannot update both of those parts with a single request.)\n\nNote that this method overrides the existing values for all of the mutable properties that are contained in any parts that the parameter value specifies.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "Channel"
-     },
-     "response": {
-      "$ref": "Channel"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    }
-   }
-  },
-  "commentThreads": {
-   "methods": {
-    "insert": {
-     "id": "youtube.commentThreads.insert",
-     "path": "commentThreads",
-     "httpMethod": "POST",
-     "description": "Creates a new top-level comment. To add a reply to an existing comment, use the comments.insert method instead.",
-     "parameters": {
-      "part": {
-       "type": "string",
-       "description": "The part parameter identifies the properties that the API response will include. Set the parameter value to snippet. The snippet part has a quota cost of 2 units.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "CommentThread"
-     },
-     "response": {
-      "$ref": "CommentThread"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "list": {
-     "id": "youtube.commentThreads.list",
-     "path": "commentThreads",
-     "httpMethod": "GET",
-     "description": "Returns a list of comment threads that match the API request parameters.",
-     "parameters": {
-      "allThreadsRelatedToChannelId": {
-       "type": "string",
-       "description": "The allThreadsRelatedToChannelId parameter instructs the API to return all comment threads associated with the specified channel. The response can include comments about the channel or about the channel's videos.",
-       "location": "query"
-      },
-      "channelId": {
-       "type": "string",
-       "description": "The channelId parameter instructs the API to return comment threads containing comments about the specified channel. (The response will not include comments left on videos that the channel uploaded.)",
-       "location": "query"
-      },
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of comment thread IDs for the resources that should be retrieved.",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.\n\nNote: This parameter is not supported for use in conjunction with the id parameter.",
-       "default": "20",
-       "format": "uint32",
-       "minimum": "1",
-       "maximum": "100",
-       "location": "query"
-      },
-      "moderationStatus": {
-       "type": "string",
-       "description": "Set this parameter to limit the returned comment threads to a particular moderation state.\n\nNote: This parameter is not supported for use in conjunction with the id parameter.",
-       "default": "MODERATION_STATUS_PUBLISHED",
-       "enum": [
-        "heldForReview",
-        "likelySpam",
-        "published"
-       ],
-       "enumDescriptions": [
-        "Retrieve comment threads that are awaiting review by a moderator. A comment thread can be included in the response if the top-level comment or at least one of the replies to that comment are awaiting review.",
-        "Retrieve comment threads classified as likely to be spam. A comment thread can be included in the response if the top-level comment or at least one of the replies to that comment is considered likely to be spam.",
-        "Retrieve threads of published comments. This is the default value. A comment thread can be included in the response if its top-level comment has been published."
-       ],
-       "location": "query"
-      },
-      "order": {
-       "type": "string",
-       "description": "The order parameter specifies the order in which the API response should list comment threads. Valid values are: \n- time - Comment threads are ordered by time. This is the default behavior.\n- relevance - Comment threads are ordered by relevance.Note: This parameter is not supported for use in conjunction with the id parameter.",
-       "default": "true",
-       "enum": [
-        "relevance",
-        "time"
-       ],
-       "enumDescriptions": [
-        "Order by relevance.",
-        "Order by time."
-       ],
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken property identifies the next page of the result that can be retrieved.\n\nNote: This parameter is not supported for use in conjunction with the id parameter.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more commentThread resource properties that the API response will include.",
-       "required": true,
-       "location": "query"
-      },
-      "searchTerms": {
-       "type": "string",
-       "description": "The searchTerms parameter instructs the API to limit the API response to only contain comments that contain the specified search terms.\n\nNote: This parameter is not supported for use in conjunction with the id parameter.",
-       "location": "query"
-      },
-      "textFormat": {
-       "type": "string",
-       "description": "Set this parameter's value to html or plainText to instruct the API to return the comments left by users in html formatted or in plain text.",
-       "default": "FORMAT_HTML",
-       "enum": [
-        "html",
-        "plainText"
-       ],
-       "enumDescriptions": [
-        "Returns the comments in HTML format. This is the default value.",
-        "Returns the comments in plain text format."
-       ],
-       "location": "query"
-      },
-      "videoId": {
-       "type": "string",
-       "description": "The videoId parameter instructs the API to return comment threads associated with the specified video ID.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "CommentThreadListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "update": {
-     "id": "youtube.commentThreads.update",
-     "path": "commentThreads",
-     "httpMethod": "PUT",
-     "description": "Modifies the top-level comment in a comment thread.",
-     "parameters": {
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of commentThread resource properties that the API response will include. You must at least include the snippet part in the parameter value since that part contains all of the properties that the API request can update.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "CommentThread"
-     },
-     "response": {
-      "$ref": "CommentThread"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    }
-   }
-  },
-  "comments": {
-   "methods": {
-    "delete": {
-     "id": "youtube.comments.delete",
-     "path": "comments",
-     "httpMethod": "DELETE",
-     "description": "Deletes a comment.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the comment ID for the resource that is being deleted.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "insert": {
-     "id": "youtube.comments.insert",
-     "path": "comments",
-     "httpMethod": "POST",
-     "description": "Creates a reply to an existing comment. Note: To create a top-level comment, use the commentThreads.insert method.",
-     "parameters": {
-      "part": {
-       "type": "string",
-       "description": "The part parameter identifies the properties that the API response will include. Set the parameter value to snippet. The snippet part has a quota cost of 2 units.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "Comment"
-     },
-     "response": {
-      "$ref": "Comment"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "list": {
-     "id": "youtube.comments.list",
-     "path": "comments",
-     "httpMethod": "GET",
-     "description": "Returns a list of comments that match the API request parameters.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of comment IDs for the resources that are being retrieved. In a comment resource, the id property specifies the comment's ID.",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.\n\nNote: This parameter is not supported for use in conjunction with the id parameter.",
-       "default": "20",
-       "format": "uint32",
-       "minimum": "1",
-       "maximum": "100",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken property identifies the next page of the result that can be retrieved.\n\nNote: This parameter is not supported for use in conjunction with the id parameter.",
-       "location": "query"
-      },
-      "parentId": {
-       "type": "string",
-       "description": "The parentId parameter specifies the ID of the comment for which replies should be retrieved.\n\nNote: YouTube currently supports replies only for top-level comments. However, replies to replies may be supported in the future.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more comment resource properties that the API response will include.",
-       "required": true,
-       "location": "query"
-      },
-      "textFormat": {
-       "type": "string",
-       "description": "This parameter indicates whether the API should return comments formatted as HTML or as plain text.",
-       "default": "FORMAT_HTML",
-       "enum": [
-        "html",
-        "plainText"
-       ],
-       "enumDescriptions": [
-        "Returns the comments in HTML format. This is the default value.",
-        "Returns the comments in plain text format."
-       ],
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "CommentListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "markAsSpam": {
-     "id": "youtube.comments.markAsSpam",
-     "path": "comments/markAsSpam",
-     "httpMethod": "POST",
-     "description": "Expresses the caller's opinion that one or more comments should be flagged as spam.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of IDs of comments that the caller believes should be classified as spam.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "setModerationStatus": {
-     "id": "youtube.comments.setModerationStatus",
-     "path": "comments/setModerationStatus",
-     "httpMethod": "POST",
-     "description": "Sets the moderation status of one or more comments. The API request must be authorized by the owner of the channel or video associated with the comments.",
-     "parameters": {
-      "banAuthor": {
-       "type": "boolean",
-       "description": "The banAuthor parameter lets you indicate that you want to automatically reject any additional comments written by the comment's author. Set the parameter value to true to ban the author.\n\nNote: This parameter is only valid if the moderationStatus parameter is also set to rejected.",
-       "default": "false",
-       "location": "query"
-      },
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of IDs that identify the comments for which you are updating the moderation status.",
-       "required": true,
-       "location": "query"
-      },
-      "moderationStatus": {
-       "type": "string",
-       "description": "Identifies the new moderation status of the specified comments.",
-       "required": true,
-       "enum": [
-        "heldForReview",
-        "published",
-        "rejected"
-       ],
-       "enumDescriptions": [
-        "Marks a comment as awaiting review by a moderator.",
-        "Clears a comment for public display.",
-        "Rejects a comment as being unfit for display. This action also effectively hides all replies to the rejected comment.\n\nNote: The API does not currently provide a way to list or otherwise discover rejected comments. However, you can change the moderation status of a rejected comment if you still know its ID. If you were to change the moderation status of a rejected comment, the comment replies would subsequently be discoverable again as well."
-       ],
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id",
-      "moderationStatus"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "update": {
-     "id": "youtube.comments.update",
-     "path": "comments",
-     "httpMethod": "PUT",
-     "description": "Modifies a comment.",
-     "parameters": {
-      "part": {
-       "type": "string",
-       "description": "The part parameter identifies the properties that the API response will include. You must at least include the snippet part in the parameter value since that part contains all of the properties that the API request can update.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "Comment"
-     },
-     "response": {
-      "$ref": "Comment"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    }
-   }
-  },
-  "fanFundingEvents": {
-   "methods": {
-    "list": {
-     "id": "youtube.fanFundingEvents.list",
-     "path": "fanFundingEvents",
-     "httpMethod": "GET",
-     "description": "Lists fan funding events for a channel.",
-     "parameters": {
-      "hl": {
-       "type": "string",
-       "description": "The hl parameter instructs the API to retrieve localized resource metadata for a specific application language that the YouTube website supports. The parameter value must be a language code included in the list returned by the i18nLanguages.list method.\n\nIf localized resource details are available in that language, the resource's snippet.localized object will contain the localized values. However, if localized details are not available, the snippet.localized object will contain resource details in the resource's default language.",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "0",
-       "maximum": "50",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies the fanFundingEvent resource parts that the API response will include. Supported values are id and snippet.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "FanFundingEventListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly"
-     ]
-    }
-   }
-  },
-  "guideCategories": {
-   "methods": {
-    "list": {
-     "id": "youtube.guideCategories.list",
-     "path": "guideCategories",
-     "httpMethod": "GET",
-     "description": "Returns a list of categories that can be associated with YouTube channels.",
-     "parameters": {
-      "hl": {
-       "type": "string",
-       "description": "The hl parameter specifies the language that will be used for text values in the API response.",
-       "default": "en-US",
-       "location": "query"
-      },
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of the YouTube channel category ID(s) for the resource(s) that are being retrieved. In a guideCategory resource, the id property specifies the YouTube channel category ID.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies the guideCategory resource properties that the API response will include. Set the parameter value to snippet.",
-       "required": true,
-       "location": "query"
-      },
-      "regionCode": {
-       "type": "string",
-       "description": "The regionCode parameter instructs the API to return the list of guide categories available in the specified country. The parameter value is an ISO 3166-1 alpha-2 country code.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "GuideCategoryListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    }
-   }
-  },
-  "i18nLanguages": {
-   "methods": {
-    "list": {
-     "id": "youtube.i18nLanguages.list",
-     "path": "i18nLanguages",
-     "httpMethod": "GET",
-     "description": "Returns a list of application languages that the YouTube website supports.",
-     "parameters": {
-      "hl": {
-       "type": "string",
-       "description": "The hl parameter specifies the language that should be used for text values in the API response.",
-       "default": "en_US",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies the i18nLanguage resource properties that the API response will include. Set the parameter value to snippet.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "I18nLanguageListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    }
-   }
-  },
-  "i18nRegions": {
-   "methods": {
-    "list": {
-     "id": "youtube.i18nRegions.list",
-     "path": "i18nRegions",
-     "httpMethod": "GET",
-     "description": "Returns a list of content regions that the YouTube website supports.",
-     "parameters": {
-      "hl": {
-       "type": "string",
-       "description": "The hl parameter specifies the language that should be used for text values in the API response.",
-       "default": "en_US",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies the i18nRegion resource properties that the API response will include. Set the parameter value to snippet.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "I18nRegionListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    }
-   }
-  },
-  "liveBroadcasts": {
-   "methods": {
-    "bind": {
-     "id": "youtube.liveBroadcasts.bind",
-     "path": "liveBroadcasts/bind",
-     "httpMethod": "POST",
-     "description": "Binds a YouTube broadcast to a stream or removes an existing binding between a broadcast and a stream. A broadcast can only be bound to one video stream, though a video stream may be bound to more than one broadcast.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the unique ID of the broadcast that is being bound to a video stream.",
-       "required": true,
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more liveBroadcast resource properties that the API response will include. The part names that you can include in the parameter value are id, snippet, contentDetails, and status.",
-       "required": true,
-       "location": "query"
-      },
-      "streamId": {
-       "type": "string",
-       "description": "The streamId parameter specifies the unique ID of the video stream that is being bound to a broadcast. If this parameter is omitted, the API will remove any existing binding between the broadcast and a video stream.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id",
-      "part"
-     ],
-     "response": {
-      "$ref": "LiveBroadcast"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "control": {
-     "id": "youtube.liveBroadcasts.control",
-     "path": "liveBroadcasts/control",
-     "httpMethod": "POST",
-     "description": "Controls the settings for a slate that can be displayed in the broadcast stream.",
-     "parameters": {
-      "displaySlate": {
-       "type": "boolean",
-       "description": "The displaySlate parameter specifies whether the slate is being enabled or disabled.",
-       "location": "query"
-      },
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the YouTube live broadcast ID that uniquely identifies the broadcast in which the slate is being updated.",
-       "required": true,
-       "location": "query"
-      },
-      "offsetTimeMs": {
-       "type": "string",
-       "description": "The offsetTimeMs parameter specifies a positive time offset when the specified slate change will occur. The value is measured in milliseconds from the beginning of the broadcast's monitor stream, which is the time that the testing phase for the broadcast began. Even though it is specified in milliseconds, the value is actually an approximation, and YouTube completes the requested action as closely as possible to that time.\n\nIf you do not specify a value for this parameter, then YouTube performs the action as soon as possible. See the Getting started guide for more details.\n\nImportant: You should only specify a value for this parameter if your broadcast stream is delayed.",
-       "format": "uint64",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more liveBroadcast resource properties that the API response will include. The part names that you can include in the parameter value are id, snippet, contentDetails, and status.",
-       "required": true,
-       "location": "query"
-      },
-      "walltime": {
-       "type": "string",
-       "description": "The walltime parameter specifies the wall clock time at which the specified slate change will occur. The value is specified in ISO 8601 (YYYY-MM-DDThh:mm:ss.sssZ) format.",
-       "format": "date-time",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id",
-      "part"
-     ],
-     "response": {
-      "$ref": "LiveBroadcast"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "delete": {
-     "id": "youtube.liveBroadcasts.delete",
-     "path": "liveBroadcasts",
-     "httpMethod": "DELETE",
-     "description": "Deletes a broadcast.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the YouTube live broadcast ID for the resource that is being deleted.",
-       "required": true,
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "insert": {
-     "id": "youtube.liveBroadcasts.insert",
-     "path": "liveBroadcasts",
-     "httpMethod": "POST",
-     "description": "Creates a broadcast.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.\n\nThe part properties that you can include in the parameter value are id, snippet, contentDetails, and status.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "LiveBroadcast"
-     },
-     "response": {
-      "$ref": "LiveBroadcast"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "list": {
-     "id": "youtube.liveBroadcasts.list",
-     "path": "liveBroadcasts",
-     "httpMethod": "GET",
-     "description": "Returns a list of YouTube broadcasts that match the API request parameters.",
-     "parameters": {
-      "broadcastStatus": {
-       "type": "string",
-       "description": "The broadcastStatus parameter filters the API response to only include broadcasts with the specified status.",
-       "enum": [
-        "active",
-        "all",
-        "completed",
-        "upcoming"
-       ],
-       "enumDescriptions": [
-        "Return current live broadcasts.",
-        "Return all broadcasts.",
-        "Return broadcasts that have already ended.",
-        "Return broadcasts that have not yet started."
-       ],
-       "location": "query"
-      },
-      "broadcastType": {
-       "type": "string",
-       "description": "The broadcastType parameter filters the API response to only include broadcasts with the specified type. This is only compatible with the mine filter for now.",
-       "default": "BROADCAST_TYPE_FILTER_EVENT",
-       "enum": [
-        "all",
-        "event",
-        "persistent"
-       ],
-       "enumDescriptions": [
-        "Return all broadcasts.",
-        "Return only scheduled event broadcasts.",
-        "Return only persistent broadcasts."
-       ],
-       "location": "query"
-      },
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of YouTube broadcast IDs that identify the broadcasts being retrieved. In a liveBroadcast resource, the id property specifies the broadcast's ID.",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "0",
-       "maximum": "50",
-       "location": "query"
-      },
-      "mine": {
-       "type": "boolean",
-       "description": "The mine parameter can be used to instruct the API to only return broadcasts owned by the authenticated user. Set the parameter value to true to only retrieve your own broadcasts.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more liveBroadcast resource properties that the API response will include. The part names that you can include in the parameter value are id, snippet, contentDetails, and status.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "LiveBroadcastListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly"
-     ]
-    },
-    "transition": {
-     "id": "youtube.liveBroadcasts.transition",
-     "path": "liveBroadcasts/transition",
-     "httpMethod": "POST",
-     "description": "Changes the status of a YouTube live broadcast and initiates any processes associated with the new status. For example, when you transition a broadcast's status to testing, YouTube starts to transmit video to that broadcast's monitor stream. Before calling this method, you should confirm that the value of the status.streamStatus property for the stream bound to your broadcast is active.",
-     "parameters": {
-      "broadcastStatus": {
-       "type": "string",
-       "description": "The broadcastStatus parameter identifies the state to which the broadcast is changing. Note that to transition a broadcast to either the testing or live state, the status.streamStatus must be active for the stream that the broadcast is bound to.",
-       "required": true,
-       "enum": [
-        "complete",
-        "live",
-        "testing"
-       ],
-       "enumDescriptions": [
-        "The broadcast is over. YouTube stops transmitting video.",
-        "The broadcast is visible to its audience. YouTube transmits video to the broadcast's monitor stream and its broadcast stream.",
-        "Start testing the broadcast. YouTube transmits video to the broadcast's monitor stream. Note that you can only transition a broadcast to the testing state if its contentDetails.monitorStream.enableMonitorStream property is set to true."
-       ],
-       "location": "query"
-      },
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the unique ID of the broadcast that is transitioning to another status.",
-       "required": true,
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more liveBroadcast resource properties that the API response will include. The part names that you can include in the parameter value are id, snippet, contentDetails, and status.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "broadcastStatus",
-      "id",
-      "part"
-     ],
-     "response": {
-      "$ref": "LiveBroadcast"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "update": {
-     "id": "youtube.liveBroadcasts.update",
-     "path": "liveBroadcasts",
-     "httpMethod": "PUT",
-     "description": "Updates a broadcast. For example, you could modify the broadcast settings defined in the liveBroadcast resource's contentDetails object.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.\n\nThe part properties that you can include in the parameter value are id, snippet, contentDetails, and status.\n\nNote that this method will override the existing values for all of the mutable properties that are contained in any parts that the parameter value specifies. For example, a broadcast's privacy status is defined in the status part. As such, if your request is updating a private or unlisted broadcast, and the request's part parameter value includes the status part, the broadcast's privacy setting will be updated to whatever value the request body specifies. If the request body does not specify a value, the existing privacy setting will be removed and the broadcast will revert to the default privacy setting.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "LiveBroadcast"
-     },
-     "response": {
-      "$ref": "LiveBroadcast"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    }
-   }
-  },
-  "liveChatBans": {
-   "methods": {
-    "delete": {
-     "id": "youtube.liveChatBans.delete",
-     "path": "liveChat/bans",
-     "httpMethod": "DELETE",
-     "description": "Removes a chat ban.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter identifies the chat ban to remove. The value uniquely identifies both the ban and the chat.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "insert": {
-     "id": "youtube.liveChatBans.insert",
-     "path": "liveChat/bans",
-     "httpMethod": "POST",
-     "description": "Adds a new ban to the chat.",
-     "parameters": {
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response returns. Set the parameter value to snippet.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "LiveChatBan"
-     },
-     "response": {
-      "$ref": "LiveChatBan"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    }
-   }
-  },
-  "liveChatMessages": {
-   "methods": {
-    "delete": {
-     "id": "youtube.liveChatMessages.delete",
-     "path": "liveChat/messages",
-     "httpMethod": "DELETE",
-     "description": "Deletes a chat message.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the YouTube chat message ID of the resource that is being deleted.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "insert": {
-     "id": "youtube.liveChatMessages.insert",
-     "path": "liveChat/messages",
-     "httpMethod": "POST",
-     "description": "Adds a message to a live chat.",
-     "parameters": {
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes. It identifies the properties that the write operation will set as well as the properties that the API response will include. Set the parameter value to snippet.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "LiveChatMessage"
-     },
-     "response": {
-      "$ref": "LiveChatMessage"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "list": {
-     "id": "youtube.liveChatMessages.list",
-     "path": "liveChat/messages",
-     "httpMethod": "GET",
-     "description": "Lists live chat messages for a specific chat.",
-     "parameters": {
-      "hl": {
-       "type": "string",
-       "description": "The hl parameter instructs the API to retrieve localized resource metadata for a specific application language that the YouTube website supports. The parameter value must be a language code included in the list returned by the i18nLanguages.list method.\n\nIf localized resource details are available in that language, the resource's snippet.localized object will contain the localized values. However, if localized details are not available, the snippet.localized object will contain resource details in the resource's default language.",
-       "location": "query"
-      },
-      "liveChatId": {
-       "type": "string",
-       "description": "The liveChatId parameter specifies the ID of the chat whose messages will be returned.",
-       "required": true,
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of messages that should be returned in the result set.",
-       "default": "500",
-       "format": "uint32",
-       "minimum": "200",
-       "maximum": "2000",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken property identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies the liveChatComment resource parts that the API response will include. Supported values are id and snippet.",
-       "required": true,
-       "location": "query"
-      },
-      "profileImageSize": {
-       "type": "integer",
-       "description": "The profileImageSize parameter specifies the size of the user profile pictures that should be returned in the result set. Default: 88.",
-       "format": "uint32",
-       "minimum": "16",
-       "maximum": "720",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "liveChatId",
-      "part"
-     ],
-     "response": {
-      "$ref": "LiveChatMessageListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly"
-     ]
-    }
-   }
-  },
-  "liveChatModerators": {
-   "methods": {
-    "delete": {
-     "id": "youtube.liveChatModerators.delete",
-     "path": "liveChat/moderators",
-     "httpMethod": "DELETE",
-     "description": "Removes a chat moderator.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter identifies the chat moderator to remove. The value uniquely identifies both the moderator and the chat.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "insert": {
-     "id": "youtube.liveChatModerators.insert",
-     "path": "liveChat/moderators",
-     "httpMethod": "POST",
-     "description": "Adds a new moderator for the chat.",
-     "parameters": {
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response returns. Set the parameter value to snippet.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "LiveChatModerator"
-     },
-     "response": {
-      "$ref": "LiveChatModerator"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "list": {
-     "id": "youtube.liveChatModerators.list",
-     "path": "liveChat/moderators",
-     "httpMethod": "GET",
-     "description": "Lists moderators for a live chat.",
-     "parameters": {
-      "liveChatId": {
-       "type": "string",
-       "description": "The liveChatId parameter specifies the YouTube live chat for which the API should return moderators.",
-       "required": true,
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "0",
-       "maximum": "50",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies the liveChatModerator resource parts that the API response will include. Supported values are id and snippet.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "liveChatId",
-      "part"
-     ],
-     "response": {
-      "$ref": "LiveChatModeratorListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly"
-     ]
-    }
-   }
-  },
-  "liveStreams": {
-   "methods": {
-    "delete": {
-     "id": "youtube.liveStreams.delete",
-     "path": "liveStreams",
-     "httpMethod": "DELETE",
-     "description": "Deletes a video stream.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the YouTube live stream ID for the resource that is being deleted.",
-       "required": true,
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "insert": {
-     "id": "youtube.liveStreams.insert",
-     "path": "liveStreams",
-     "httpMethod": "POST",
-     "description": "Creates a video stream. The stream enables you to send your video to YouTube, which can then broadcast the video to your audience.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.\n\nThe part properties that you can include in the parameter value are id, snippet, cdn, and status.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "LiveStream"
-     },
-     "response": {
-      "$ref": "LiveStream"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    },
-    "list": {
-     "id": "youtube.liveStreams.list",
-     "path": "liveStreams",
-     "httpMethod": "GET",
-     "description": "Returns a list of video streams that match the API request parameters.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of YouTube stream IDs that identify the streams being retrieved. In a liveStream resource, the id property specifies the stream's ID.",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "0",
-       "maximum": "50",
-       "location": "query"
-      },
-      "mine": {
-       "type": "boolean",
-       "description": "The mine parameter can be used to instruct the API to only return streams owned by the authenticated user. Set the parameter value to true to only retrieve your own streams.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more liveStream resource properties that the API response will include. The part names that you can include in the parameter value are id, snippet, cdn, and status.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "LiveStreamListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly"
-     ]
-    },
-    "update": {
-     "id": "youtube.liveStreams.update",
-     "path": "liveStreams",
-     "httpMethod": "PUT",
-     "description": "Updates a video stream. If the properties that you want to change cannot be updated, then you need to create a new stream with the proper settings.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.\n\nThe part properties that you can include in the parameter value are id, snippet, cdn, and status.\n\nNote that this method will override the existing values for all of the mutable properties that are contained in any parts that the parameter value specifies. If the request body does not specify a value for a mutable property, the existing value for that property will be removed.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "LiveStream"
-     },
-     "response": {
-      "$ref": "LiveStream"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl"
-     ]
-    }
-   }
-  },
-  "playlistItems": {
-   "methods": {
-    "delete": {
-     "id": "youtube.playlistItems.delete",
-     "path": "playlistItems",
-     "httpMethod": "DELETE",
-     "description": "Deletes a playlist item.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the YouTube playlist item ID for the playlist item that is being deleted. In a playlistItem resource, the id property specifies the playlist item's ID.",
-       "required": true,
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "insert": {
-     "id": "youtube.playlistItems.insert",
-     "path": "playlistItems",
-     "httpMethod": "POST",
-     "description": "Adds a resource to a playlist.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "PlaylistItem"
-     },
-     "response": {
-      "$ref": "PlaylistItem"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "list": {
-     "id": "youtube.playlistItems.list",
-     "path": "playlistItems",
-     "httpMethod": "GET",
-     "description": "Returns a collection of playlist items that match the API request parameters. You can retrieve all of the playlist items in a specified playlist or retrieve one or more playlist items by their unique IDs.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of one or more unique playlist item IDs.",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "0",
-       "maximum": "50",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more playlistItem resource properties that the API response will include.\n\nIf the parameter identifies a property that contains child properties, the child properties will be included in the response. For example, in a playlistItem resource, the snippet property contains numerous fields, including the title, description, position, and resourceId properties. As such, if you set part=snippet, the API response will contain all of those properties.",
-       "required": true,
-       "location": "query"
-      },
-      "playlistId": {
-       "type": "string",
-       "description": "The playlistId parameter specifies the unique ID of the playlist for which you want to retrieve playlist items. Note that even though this is an optional parameter, every request to retrieve playlist items must specify a value for either the id parameter or the playlistId parameter.",
-       "location": "query"
-      },
-      "videoId": {
-       "type": "string",
-       "description": "The videoId parameter specifies that the request should return only the playlist items that contain the specified video.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "PlaylistItemListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ],
-     "supportsSubscription": true
-    },
-    "update": {
-     "id": "youtube.playlistItems.update",
-     "path": "playlistItems",
-     "httpMethod": "PUT",
-     "description": "Modifies a playlist item. For example, you could update the item's position in the playlist.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.\n\nNote that this method will override the existing values for all of the mutable properties that are contained in any parts that the parameter value specifies. For example, a playlist item can specify a start time and end time, which identify the times portion of the video that should play when users watch the video in the playlist. If your request is updating a playlist item that sets these values, and the request's part parameter value includes the contentDetails part, the playlist item's start and end times will be updated to whatever value the request body specifies. If the request body does not specify values, the existing start and end times will be removed and replaced with the default settings.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "PlaylistItem"
-     },
-     "response": {
-      "$ref": "PlaylistItem"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    }
-   }
-  },
-  "playlists": {
-   "methods": {
-    "delete": {
-     "id": "youtube.playlists.delete",
-     "path": "playlists",
-     "httpMethod": "DELETE",
-     "description": "Deletes a playlist.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the YouTube playlist ID for the playlist that is being deleted. In a playlist resource, the id property specifies the playlist's ID.",
-       "required": true,
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "insert": {
-     "id": "youtube.playlists.insert",
-     "path": "playlists",
-     "httpMethod": "POST",
-     "description": "Creates a playlist.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "Playlist"
-     },
-     "response": {
-      "$ref": "Playlist"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "list": {
-     "id": "youtube.playlists.list",
-     "path": "playlists",
-     "httpMethod": "GET",
-     "description": "Returns a collection of playlists that match the API request parameters. For example, you can retrieve all playlists that the authenticated user owns, or you can retrieve one or more playlists by their unique IDs.",
-     "parameters": {
-      "channelId": {
-       "type": "string",
-       "description": "This value indicates that the API should only return the specified channel's playlists.",
-       "location": "query"
-      },
-      "hl": {
-       "type": "string",
-       "description": "The hl parameter should be used for filter out the properties that are not in the given language. Used for the snippet part.",
-       "location": "query"
-      },
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of the YouTube playlist ID(s) for the resource(s) that are being retrieved. In a playlist resource, the id property specifies the playlist's YouTube playlist ID.",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "0",
-       "maximum": "50",
-       "location": "query"
-      },
-      "mine": {
-       "type": "boolean",
-       "description": "Set this parameter's value to true to instruct the API to only return playlists owned by the authenticated user.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more playlist resource properties that the API response will include.\n\nIf the parameter identifies a property that contains child properties, the child properties will be included in the response. For example, in a playlist resource, the snippet property contains properties like author, title, description, tags, and timeCreated. As such, if you set part=snippet, the API response will contain all of those properties.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "PlaylistListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "update": {
-     "id": "youtube.playlists.update",
-     "path": "playlists",
-     "httpMethod": "PUT",
-     "description": "Modifies a playlist. For example, you could change a playlist's title, description, or privacy status.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.\n\nNote that this method will override the existing values for mutable properties that are contained in any parts that the request body specifies. For example, a playlist's description is contained in the snippet part, which must be included in the request body. If the request does not specify a value for the snippet.description property, the playlist's existing description will be deleted.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "Playlist"
-     },
-     "response": {
-      "$ref": "Playlist"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    }
-   }
-  },
-  "search": {
-   "methods": {
-    "list": {
-     "id": "youtube.search.list",
-     "path": "search",
-     "httpMethod": "GET",
-     "description": "Returns a collection of search results that match the query parameters specified in the API request. By default, a search result set identifies matching video, channel, and playlist resources, but you can also configure queries to only retrieve a specific type of resource.",
-     "parameters": {
-      "channelId": {
-       "type": "string",
-       "description": "The channelId parameter indicates that the API response should only contain resources created by the channel",
-       "location": "query"
-      },
-      "channelType": {
-       "type": "string",
-       "description": "The channelType parameter lets you restrict a search to a particular type of channel.",
-       "enum": [
-        "any",
-        "show"
-       ],
-       "enumDescriptions": [
-        "Return all channels.",
-        "Only retrieve shows."
-       ],
-       "location": "query"
-      },
-      "eventType": {
-       "type": "string",
-       "description": "The eventType parameter restricts a search to broadcast events. If you specify a value for this parameter, you must also set the type parameter's value to video.",
-       "enum": [
-        "completed",
-        "live",
-        "upcoming"
-       ],
-       "enumDescriptions": [
-        "Only include completed broadcasts.",
-        "Only include active broadcasts.",
-        "Only include upcoming broadcasts."
-       ],
-       "location": "query"
-      },
-      "forContentOwner": {
-       "type": "boolean",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe forContentOwner parameter restricts the search to only retrieve resources owned by the content owner specified by the onBehalfOfContentOwner parameter. The user must be authenticated using a CMS account linked to the specified content owner and onBehalfOfContentOwner must be provided.",
-       "location": "query"
-      },
-      "forDeveloper": {
-       "type": "boolean",
-       "description": "The forDeveloper parameter restricts the search to only retrieve videos uploaded via the developer's application or website. The API server uses the request's authorization credentials to identify the developer. Therefore, a developer can restrict results to videos uploaded through the developer's own app or website but not to videos uploaded through other apps or sites.",
-       "location": "query"
-      },
-      "forMine": {
-       "type": "boolean",
-       "description": "The forMine parameter restricts the search to only retrieve videos owned by the authenticated user. If you set this parameter to true, then the type parameter's value must also be set to video.",
-       "location": "query"
-      },
-      "location": {
-       "type": "string",
-       "description": "The location parameter, in conjunction with the locationRadius parameter, defines a circular geographic area and also restricts a search to videos that specify, in their metadata, a geographic location that falls within that area. The parameter value is a string that specifies latitude/longitude coordinates e.g. (37.42307,-122.08427).\n\n\n- The location parameter value identifies the point at the center of the area.\n- The locationRadius parameter specifies the maximum distance that the location associated with a video can be from that point for the video to still be included in the search results.The API returns an error if your request specifies a value for the location parameter but does not also specify a value for the locationRadius parameter.",
-       "location": "query"
-      },
-      "locationRadius": {
-       "type": "string",
-       "description": "The locationRadius parameter, in conjunction with the location parameter, defines a circular geographic area.\n\nThe parameter value must be a floating point number followed by a measurement unit. Valid measurement units are m, km, ft, and mi. For example, valid parameter values include 1500m, 5km, 10000ft, and 0.75mi. The API does not support locationRadius parameter values larger than 1000 kilometers.\n\nNote: See the definition of the location parameter for more information.",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "0",
-       "maximum": "50",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "order": {
-       "type": "string",
-       "description": "The order parameter specifies the method that will be used to order resources in the API response.",
-       "default": "SEARCH_SORT_RELEVANCE",
-       "enum": [
-        "date",
-        "rating",
-        "relevance",
-        "title",
-        "videoCount",
-        "viewCount"
-       ],
-       "enumDescriptions": [
-        "Resources are sorted in reverse chronological order based on the date they were created.",
-        "Resources are sorted from highest to lowest rating.",
-        "Resources are sorted based on their relevance to the search query. This is the default value for this parameter.",
-        "Resources are sorted alphabetically by title.",
-        "Channels are sorted in descending order of their number of uploaded videos.",
-        "Resources are sorted from highest to lowest number of views."
-       ],
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more search resource properties that the API response will include. Set the parameter value to snippet.",
-       "required": true,
-       "location": "query"
-      },
-      "publishedAfter": {
-       "type": "string",
-       "description": "The publishedAfter parameter indicates that the API response should only contain resources created after the specified time. The value is an RFC 3339 formatted date-time value (1970-01-01T00:00:00Z).",
-       "format": "date-time",
-       "location": "query"
-      },
-      "publishedBefore": {
-       "type": "string",
-       "description": "The publishedBefore parameter indicates that the API response should only contain resources created before the specified time. The value is an RFC 3339 formatted date-time value (1970-01-01T00:00:00Z).",
-       "format": "date-time",
-       "location": "query"
-      },
-      "q": {
-       "type": "string",
-       "description": "The q parameter specifies the query term to search for.\n\nYour request can also use the Boolean NOT (-) and OR (|) operators to exclude videos or to find videos that are associated with one of several search terms. For example, to search for videos matching either \"boating\" or \"sailing\", set the q parameter value to boating|sailing. Similarly, to search for videos matching either \"boating\" or \"sailing\" but not \"fishing\", set the q parameter value to boating|sailing -fishing. Note that the pipe character must be URL-escaped when it is sent in your API request. The URL-escaped value for the pipe character is %7C.",
-       "location": "query"
-      },
-      "regionCode": {
-       "type": "string",
-       "description": "The regionCode parameter instructs the API to return search results for the specified country. The parameter value is an ISO 3166-1 alpha-2 country code.",
-       "location": "query"
-      },
-      "relatedToVideoId": {
-       "type": "string",
-       "description": "The relatedToVideoId parameter retrieves a list of videos that are related to the video that the parameter value identifies. The parameter value must be set to a YouTube video ID and, if you are using this parameter, the type parameter must be set to video.",
-       "location": "query"
-      },
-      "relevanceLanguage": {
-       "type": "string",
-       "description": "The relevanceLanguage parameter instructs the API to return search results that are most relevant to the specified language. The parameter value is typically an ISO 639-1 two-letter language code. However, you should use the values zh-Hans for simplified Chinese and zh-Hant for traditional Chinese. Please note that results in other languages will still be returned if they are highly relevant to the search query term.",
-       "location": "query"
-      },
-      "safeSearch": {
-       "type": "string",
-       "description": "The safeSearch parameter indicates whether the search results should include restricted content as well as standard content.",
-       "enum": [
-        "moderate",
-        "none",
-        "strict"
-       ],
-       "enumDescriptions": [
-        "YouTube will filter some content from search results and, at the least, will filter content that is restricted in your locale. Based on their content, search results could be removed from search results or demoted in search results. This is the default parameter value.",
-        "YouTube will not filter the search result set.",
-        "YouTube will try to exclude all restricted content from the search result set. Based on their content, search results could be removed from search results or demoted in search results."
-       ],
-       "location": "query"
-      },
-      "topicId": {
-       "type": "string",
-       "description": "The topicId parameter indicates that the API response should only contain resources associated with the specified topic. The value identifies a Freebase topic ID.",
-       "location": "query"
-      },
-      "type": {
-       "type": "string",
-       "description": "The type parameter restricts a search query to only retrieve a particular type of resource. The value is a comma-separated list of resource types.",
-       "default": "video,channel,playlist",
-       "location": "query"
-      },
-      "videoCaption": {
-       "type": "string",
-       "description": "The videoCaption parameter indicates whether the API should filter video search results based on whether they have captions. If you specify a value for this parameter, you must also set the type parameter's value to video.",
-       "enum": [
-        "any",
-        "closedCaption",
-        "none"
-       ],
-       "enumDescriptions": [
-        "Do not filter results based on caption availability.",
-        "Only include videos that have captions.",
-        "Only include videos that do not have captions."
-       ],
-       "location": "query"
-      },
-      "videoCategoryId": {
-       "type": "string",
-       "description": "The videoCategoryId parameter filters video search results based on their category. If you specify a value for this parameter, you must also set the type parameter's value to video.",
-       "location": "query"
-      },
-      "videoDefinition": {
-       "type": "string",
-       "description": "The videoDefinition parameter lets you restrict a search to only include either high definition (HD) or standard definition (SD) videos. HD videos are available for playback in at least 720p, though higher resolutions, like 1080p, might also be available. If you specify a value for this parameter, you must also set the type parameter's value to video.",
-       "enum": [
-        "any",
-        "high",
-        "standard"
-       ],
-       "enumDescriptions": [
-        "Return all videos, regardless of their resolution.",
-        "Only retrieve HD videos.",
-        "Only retrieve videos in standard definition."
-       ],
-       "location": "query"
-      },
-      "videoDimension": {
-       "type": "string",
-       "description": "The videoDimension parameter lets you restrict a search to only retrieve 2D or 3D videos. If you specify a value for this parameter, you must also set the type parameter's value to video.",
-       "enum": [
-        "2d",
-        "3d",
-        "any"
-       ],
-       "enumDescriptions": [
-        "Restrict search results to exclude 3D videos.",
-        "Restrict search results to only include 3D videos.",
-        "Include both 3D and non-3D videos in returned results. This is the default value."
-       ],
-       "location": "query"
-      },
-      "videoDuration": {
-       "type": "string",
-       "description": "The videoDuration parameter filters video search results based on their duration. If you specify a value for this parameter, you must also set the type parameter's value to video.",
-       "enum": [
-        "any",
-        "long",
-        "medium",
-        "short"
-       ],
-       "enumDescriptions": [
-        "Do not filter video search results based on their duration. This is the default value.",
-        "Only include videos longer than 20 minutes.",
-        "Only include videos that are between four and 20 minutes long (inclusive).",
-        "Only include videos that are less than four minutes long."
-       ],
-       "location": "query"
-      },
-      "videoEmbeddable": {
-       "type": "string",
-       "description": "The videoEmbeddable parameter lets you to restrict a search to only videos that can be embedded into a webpage. If you specify a value for this parameter, you must also set the type parameter's value to video.",
-       "enum": [
-        "any",
-        "true"
-       ],
-       "enumDescriptions": [
-        "Return all videos, embeddable or not.",
-        "Only retrieve embeddable videos."
-       ],
-       "location": "query"
-      },
-      "videoLicense": {
-       "type": "string",
-       "description": "The videoLicense parameter filters search results to only include videos with a particular license. YouTube lets video uploaders choose to attach either the Creative Commons license or the standard YouTube license to each of their videos. If you specify a value for this parameter, you must also set the type parameter's value to video.",
-       "enum": [
-        "any",
-        "creativeCommon",
-        "youtube"
-       ],
-       "enumDescriptions": [
-        "Return all videos, regardless of which license they have, that match the query parameters.",
-        "Only return videos that have a Creative Commons license. Users can reuse videos with this license in other videos that they create. Learn more.",
-        "Only return videos that have the standard YouTube license."
-       ],
-       "location": "query"
-      },
-      "videoSyndicated": {
-       "type": "string",
-       "description": "The videoSyndicated parameter lets you to restrict a search to only videos that can be played outside youtube.com. If you specify a value for this parameter, you must also set the type parameter's value to video.",
-       "enum": [
-        "any",
-        "true"
-       ],
-       "enumDescriptions": [
-        "Return all videos, syndicated or not.",
-        "Only retrieve syndicated videos."
-       ],
-       "location": "query"
-      },
-      "videoType": {
-       "type": "string",
-       "description": "The videoType parameter lets you restrict a search to a particular type of videos. If you specify a value for this parameter, you must also set the type parameter's value to video.",
-       "enum": [
-        "any",
-        "episode",
-        "movie"
-       ],
-       "enumDescriptions": [
-        "Return all videos.",
-        "Only retrieve episodes of shows.",
-        "Only retrieve movies."
-       ],
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "SearchListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    }
-   }
-  },
-  "sponsors": {
-   "methods": {
-    "list": {
-     "id": "youtube.sponsors.list",
-     "path": "sponsors",
-     "httpMethod": "GET",
-     "description": "Lists sponsors for a channel.",
-     "parameters": {
-      "filter": {
-       "type": "string",
-       "description": "The filter parameter specifies which channel sponsors to return.",
-       "default": "POLL_NEWEST",
-       "enum": [
-        "all",
-        "newest"
-       ],
-       "enumDescriptions": [
-        "Return all sponsors, from newest to oldest.",
-        "Return the most recent sponsors, from newest to oldest."
-       ],
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "0",
-       "maximum": "50",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies the sponsor resource parts that the API response will include. Supported values are id and snippet.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "SponsorListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly"
-     ]
-    }
-   }
-  },
-  "subscriptions": {
-   "methods": {
-    "delete": {
-     "id": "youtube.subscriptions.delete",
-     "path": "subscriptions",
-     "httpMethod": "DELETE",
-     "description": "Deletes a subscription.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the YouTube subscription ID for the resource that is being deleted. In a subscription resource, the id property specifies the YouTube subscription ID.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "insert": {
-     "id": "youtube.subscriptions.insert",
-     "path": "subscriptions",
-     "httpMethod": "POST",
-     "description": "Adds a subscription for the authenticated user's channel.",
-     "parameters": {
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "Subscription"
-     },
-     "response": {
-      "$ref": "Subscription"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "list": {
-     "id": "youtube.subscriptions.list",
-     "path": "subscriptions",
-     "httpMethod": "GET",
-     "description": "Returns subscription resources that match the API request criteria.",
-     "parameters": {
-      "channelId": {
-       "type": "string",
-       "description": "The channelId parameter specifies a YouTube channel ID. The API will only return that channel's subscriptions.",
-       "location": "query"
-      },
-      "forChannelId": {
-       "type": "string",
-       "description": "The forChannelId parameter specifies a comma-separated list of channel IDs. The API response will then only contain subscriptions matching those channels.",
-       "location": "query"
-      },
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of the YouTube subscription ID(s) for the resource(s) that are being retrieved. In a subscription resource, the id property specifies the YouTube subscription ID.",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "0",
-       "maximum": "50",
-       "location": "query"
-      },
-      "mine": {
-       "type": "boolean",
-       "description": "Set this parameter's value to true to retrieve a feed of the authenticated user's subscriptions.",
-       "location": "query"
-      },
-      "myRecentSubscribers": {
-       "type": "boolean",
-       "description": "Set this parameter's value to true to retrieve a feed of the subscribers of the authenticated user in reverse chronological order (newest first).",
-       "location": "query"
-      },
-      "mySubscribers": {
-       "type": "boolean",
-       "description": "Set this parameter's value to true to retrieve a feed of the subscribers of the authenticated user in no particular order.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "order": {
-       "type": "string",
-       "description": "The order parameter specifies the method that will be used to sort resources in the API response.",
-       "default": "SUBSCRIPTION_ORDER_RELEVANCE",
-       "enum": [
-        "alphabetical",
-        "relevance",
-        "unread"
-       ],
-       "enumDescriptions": [
-        "Sort alphabetically.",
-        "Sort by relevance.",
-        "Sort by order of activity."
-       ],
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more subscription resource properties that the API response will include.\n\nIf the parameter identifies a property that contains child properties, the child properties will be included in the response. For example, in a subscription resource, the snippet property contains other properties, such as a display title for the subscription. If you set part=snippet, the API response will also contain all of those nested properties.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "SubscriptionListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    }
-   }
-  },
-  "superChatEvents": {
-   "methods": {
-    "list": {
-     "id": "youtube.superChatEvents.list",
-     "path": "superChatEvents",
-     "httpMethod": "GET",
-     "description": "Lists Super Chat events for a channel.",
-     "parameters": {
-      "hl": {
-       "type": "string",
-       "description": "The hl parameter instructs the API to retrieve localized resource metadata for a specific application language that the YouTube website supports. The parameter value must be a language code included in the list returned by the i18nLanguages.list method.\n\nIf localized resource details are available in that language, the resource's snippet.localized object will contain the localized values. However, if localized details are not available, the snippet.localized object will contain resource details in the resource's default language.",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "0",
-       "maximum": "50",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies the superChatEvent resource parts that the API response will include. Supported values are id and snippet.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "SuperChatEventListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly"
-     ]
-    }
-   }
-  },
-  "thumbnails": {
-   "methods": {
-    "set": {
-     "id": "youtube.thumbnails.set",
-     "path": "thumbnails/set",
-     "httpMethod": "POST",
-     "description": "Uploads a custom video thumbnail to YouTube and sets it for a video.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The actual CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "videoId": {
-       "type": "string",
-       "description": "The videoId parameter specifies a YouTube video ID for which the custom video thumbnail is being provided.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "videoId"
-     ],
-     "response": {
-      "$ref": "ThumbnailSetResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.upload",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ],
-     "supportsMediaUpload": true,
-     "mediaUpload": {
-      "accept": [
-       "application/octet-stream",
-       "image/jpeg",
-       "image/png"
-      ],
-      "maxSize": "2MB",
-      "protocols": {
-       "simple": {
-        "multipart": true,
-        "path": "/upload/youtube/v3/thumbnails/set"
-       },
-       "resumable": {
-        "multipart": true,
-        "path": "/resumable/upload/youtube/v3/thumbnails/set"
-       }
-      }
-     }
-    }
-   }
-  },
-  "videoAbuseReportReasons": {
-   "methods": {
-    "list": {
-     "id": "youtube.videoAbuseReportReasons.list",
-     "path": "videoAbuseReportReasons",
-     "httpMethod": "GET",
-     "description": "Returns a list of abuse reasons that can be used for reporting abusive videos.",
-     "parameters": {
-      "hl": {
-       "type": "string",
-       "description": "The hl parameter specifies the language that should be used for text values in the API response.",
-       "default": "en_US",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies the videoCategory resource parts that the API response will include. Supported values are id and snippet.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "VideoAbuseReportReasonListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly"
-     ]
-    }
-   }
-  },
-  "videoCategories": {
-   "methods": {
-    "list": {
-     "id": "youtube.videoCategories.list",
-     "path": "videoCategories",
-     "httpMethod": "GET",
-     "description": "Returns a list of categories that can be associated with YouTube videos.",
-     "parameters": {
-      "hl": {
-       "type": "string",
-       "description": "The hl parameter specifies the language that should be used for text values in the API response.",
-       "default": "en_US",
-       "location": "query"
-      },
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of video category IDs for the resources that you are retrieving.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies the videoCategory resource properties that the API response will include. Set the parameter value to snippet.",
-       "required": true,
-       "location": "query"
-      },
-      "regionCode": {
-       "type": "string",
-       "description": "The regionCode parameter instructs the API to return the list of video categories available in the specified country. The parameter value is an ISO 3166-1 alpha-2 country code.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "VideoCategoryListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    }
-   }
-  },
-  "videos": {
-   "methods": {
-    "delete": {
-     "id": "youtube.videos.delete",
-     "path": "videos",
-     "httpMethod": "DELETE",
-     "description": "Deletes a YouTube video.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the YouTube video ID for the resource that is being deleted. In a video resource, the id property specifies the video's ID.",
-       "required": true,
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The actual CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "getRating": {
-     "id": "youtube.videos.getRating",
-     "path": "videos/getRating",
-     "httpMethod": "GET",
-     "description": "Retrieves the ratings that the authorized user gave to a list of specified videos.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of the YouTube video ID(s) for the resource(s) for which you are retrieving rating data. In a video resource, the id property specifies the video's ID.",
-       "required": true,
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id"
-     ],
-     "response": {
-      "$ref": "VideoGetRatingResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "insert": {
-     "id": "youtube.videos.insert",
-     "path": "videos",
-     "httpMethod": "POST",
-     "description": "Uploads a video to YouTube and optionally sets the video's metadata.",
-     "parameters": {
-      "autoLevels": {
-       "type": "boolean",
-       "description": "The autoLevels parameter indicates whether YouTube should automatically enhance the video's lighting and color.",
-       "location": "query"
-      },
-      "notifySubscribers": {
-       "type": "boolean",
-       "description": "The notifySubscribers parameter indicates whether YouTube should send a notification about the new video to users who subscribe to the video's channel. A parameter value of True indicates that subscribers will be notified of newly uploaded videos. However, a channel owner who is uploading many videos might prefer to set the value to False to avoid sending a notification about each new video to the channel's subscribers.",
-       "default": "true",
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "onBehalfOfContentOwnerChannel": {
-       "type": "string",
-       "description": "This parameter can only be used in a properly authorized request. Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwnerChannel parameter specifies the YouTube channel ID of the channel to which a video is being added. This parameter is required when a request specifies a value for the onBehalfOfContentOwner parameter, and it can only be used in conjunction with that parameter. In addition, the request must be authorized using a CMS account that is linked to the content owner that the onBehalfOfContentOwner parameter specifies. Finally, the channel that the onBehalfOfContentOwnerChannel parameter value specifies must be linked to the content owner that the onBehalfOfContentOwner parameter specifies.\n\nThis parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and perform actions on behalf of the channel specified in the parameter value, without having to provide authentication credentials for each separate channel.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.\n\nNote that not all parts contain properties that can be set when inserting or updating a video. For example, the statistics object encapsulates statistics that YouTube calculates for a video and does not contain values that you can set or modify. If the parameter value specifies a part that does not contain mutable values, that part will still be included in the API response.",
-       "required": true,
-       "location": "query"
-      },
-      "stabilize": {
-       "type": "boolean",
-       "description": "The stabilize parameter indicates whether YouTube should adjust the video to remove shaky camera motions.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "Video"
-     },
-     "response": {
-      "$ref": "Video"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.upload",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ],
-     "supportsMediaUpload": true,
-     "mediaUpload": {
-      "accept": [
-       "application/octet-stream",
-       "video/*"
-      ],
-      "maxSize": "64GB",
-      "protocols": {
-       "simple": {
-        "multipart": true,
-        "path": "/upload/youtube/v3/videos"
-       },
-       "resumable": {
-        "multipart": true,
-        "path": "/resumable/upload/youtube/v3/videos"
-       }
-      }
-     }
-    },
-    "list": {
-     "id": "youtube.videos.list",
-     "path": "videos",
-     "httpMethod": "GET",
-     "description": "Returns a list of videos that match the API request parameters.",
-     "parameters": {
-      "chart": {
-       "type": "string",
-       "description": "The chart parameter identifies the chart that you want to retrieve.",
-       "enum": [
-        "mostPopular"
-       ],
-       "enumDescriptions": [
-        "Return the most popular videos for the specified content region and video category."
-       ],
-       "location": "query"
-      },
-      "hl": {
-       "type": "string",
-       "description": "The hl parameter instructs the API to retrieve localized resource metadata for a specific application language that the YouTube website supports. The parameter value must be a language code included in the list returned by the i18nLanguages.list method.\n\nIf localized resource details are available in that language, the resource's snippet.localized object will contain the localized values. However, if localized details are not available, the snippet.localized object will contain resource details in the resource's default language.",
-       "location": "query"
-      },
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies a comma-separated list of the YouTube video ID(s) for the resource(s) that are being retrieved. In a video resource, the id property specifies the video's ID.",
-       "location": "query"
-      },
-      "locale": {
-       "type": "string",
-       "description": "DEPRECATED",
-       "location": "query"
-      },
-      "maxHeight": {
-       "type": "integer",
-       "description": "The maxHeight parameter specifies a maximum height of the embedded player. If maxWidth is provided, maxHeight may not be reached in order to not violate the width request.",
-       "format": "uint32",
-       "minimum": "72",
-       "maximum": "8192",
-       "location": "query"
-      },
-      "maxResults": {
-       "type": "integer",
-       "description": "The maxResults parameter specifies the maximum number of items that should be returned in the result set.\n\nNote: This parameter is supported for use in conjunction with the myRating and chart parameters, but it is not supported for use in conjunction with the id parameter.",
-       "default": "5",
-       "format": "uint32",
-       "minimum": "1",
-       "maximum": "50",
-       "location": "query"
-      },
-      "maxWidth": {
-       "type": "integer",
-       "description": "The maxWidth parameter specifies a maximum width of the embedded player. If maxHeight is provided, maxWidth may not be reached in order to not violate the height request.",
-       "format": "uint32",
-       "minimum": "72",
-       "maximum": "8192",
-       "location": "query"
-      },
-      "myRating": {
-       "type": "string",
-       "description": "Set this parameter's value to like or dislike to instruct the API to only return videos liked or disliked by the authenticated user.",
-       "enum": [
-        "dislike",
-        "like"
-       ],
-       "enumDescriptions": [
-        "Returns only videos disliked by the authenticated user.",
-        "Returns only video liked by the authenticated user."
-       ],
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "pageToken": {
-       "type": "string",
-       "description": "The pageToken parameter identifies a specific page in the result set that should be returned. In an API response, the nextPageToken and prevPageToken properties identify other pages that could be retrieved.\n\nNote: This parameter is supported for use in conjunction with the myRating and chart parameters, but it is not supported for use in conjunction with the id parameter.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter specifies a comma-separated list of one or more video resource properties that the API response will include.\n\nIf the parameter identifies a property that contains child properties, the child properties will be included in the response. For example, in a video resource, the snippet property contains the channelId, title, description, tags, and categoryId properties. As such, if you set part=snippet, the API response will contain all of those properties.",
-       "required": true,
-       "location": "query"
-      },
-      "regionCode": {
-       "type": "string",
-       "description": "The regionCode parameter instructs the API to select a video chart available in the specified region. This parameter can only be used in conjunction with the chart parameter. The parameter value is an ISO 3166-1 alpha-2 country code.",
-       "location": "query"
-      },
-      "videoCategoryId": {
-       "type": "string",
-       "description": "The videoCategoryId parameter identifies the video category for which the chart should be retrieved. This parameter can only be used in conjunction with the chart parameter. By default, charts are not restricted to a particular category.",
-       "default": "0",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "response": {
-      "$ref": "VideoListResponse"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.readonly",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "rate": {
-     "id": "youtube.videos.rate",
-     "path": "videos/rate",
-     "httpMethod": "POST",
-     "description": "Add a like or dislike rating to a video or remove a rating from a video.",
-     "parameters": {
-      "id": {
-       "type": "string",
-       "description": "The id parameter specifies the YouTube video ID of the video that is being rated or having its rating removed.",
-       "required": true,
-       "location": "query"
-      },
-      "rating": {
-       "type": "string",
-       "description": "Specifies the rating to record.",
-       "required": true,
-       "enum": [
-        "dislike",
-        "like",
-        "none"
-       ],
-       "enumDescriptions": [
-        "Records that the authenticated user disliked the video.",
-        "Records that the authenticated user liked the video.",
-        "Removes any rating that the authenticated user had previously set for the video."
-       ],
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "id",
-      "rating"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "reportAbuse": {
-     "id": "youtube.videos.reportAbuse",
-     "path": "videos/reportAbuse",
-     "httpMethod": "POST",
-     "description": "Report abuse for a video.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      }
-     },
-     "request": {
-      "$ref": "VideoAbuseReport"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    },
-    "update": {
-     "id": "youtube.videos.update",
-     "path": "videos",
-     "httpMethod": "PUT",
-     "description": "Updates a video's metadata.",
-     "parameters": {
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The actual CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      },
-      "part": {
-       "type": "string",
-       "description": "The part parameter serves two purposes in this operation. It identifies the properties that the write operation will set as well as the properties that the API response will include.\n\nNote that this method will override the existing values for all of the mutable properties that are contained in any parts that the parameter value specifies. For example, a video's privacy setting is contained in the status part. As such, if your request is updating a private video, and the request's part parameter value includes the status part, the video's privacy setting will be updated to whatever value the request body specifies. If the request body does not specify a value, the existing privacy setting will be removed and the video will revert to the default privacy setting.\n\nIn addition, not all parts contain properties that can be set when inserting or updating a video. For example, the statistics object encapsulates statistics that YouTube calculates for a video and does not contain values that you can set or modify. If the parameter value specifies a part that does not contain mutable values, that part will still be included in the API response.",
-       "required": true,
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "part"
-     ],
-     "request": {
-      "$ref": "Video"
-     },
-     "response": {
-      "$ref": "Video"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    }
-   }
-  },
-  "watermarks": {
-   "methods": {
-    "set": {
-     "id": "youtube.watermarks.set",
-     "path": "watermarks/set",
-     "httpMethod": "POST",
-     "description": "Uploads a watermark image to YouTube and sets it for a channel.",
-     "parameters": {
-      "channelId": {
-       "type": "string",
-       "description": "The channelId parameter specifies the YouTube channel ID for which the watermark is being provided.",
-       "required": true,
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "channelId"
-     ],
-     "request": {
-      "$ref": "InvideoBranding"
-     },
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtube.upload",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ],
-     "supportsMediaUpload": true,
-     "mediaUpload": {
-      "accept": [
-       "application/octet-stream",
-       "image/jpeg",
-       "image/png"
-      ],
-      "maxSize": "10MB",
-      "protocols": {
-       "simple": {
-        "multipart": true,
-        "path": "/upload/youtube/v3/watermarks/set"
-       },
-       "resumable": {
-        "multipart": true,
-        "path": "/resumable/upload/youtube/v3/watermarks/set"
-       }
-      }
-     }
-    },
-    "unset": {
-     "id": "youtube.watermarks.unset",
-     "path": "watermarks/unset",
-     "httpMethod": "POST",
-     "description": "Deletes a channel's watermark image.",
-     "parameters": {
-      "channelId": {
-       "type": "string",
-       "description": "The channelId parameter specifies the YouTube channel ID for which the watermark is being unset.",
-       "required": true,
-       "location": "query"
-      },
-      "onBehalfOfContentOwner": {
-       "type": "string",
-       "description": "Note: This parameter is intended exclusively for YouTube content partners.\n\nThe onBehalfOfContentOwner parameter indicates that the request's authorization credentials identify a YouTube CMS user who is acting on behalf of the content owner specified in the parameter value. This parameter is intended for YouTube content partners that own and manage many different YouTube channels. It allows content owners to authenticate once and get access to all their video and channel data, without having to provide authentication credentials for each individual channel. The CMS account that the user authenticates with must be linked to the specified YouTube content owner.",
-       "location": "query"
-      }
-     },
-     "parameterOrder": [
-      "channelId"
-     ],
-     "scopes": [
-      "https://www.googleapis.com/auth/youtube",
-      "https://www.googleapis.com/auth/youtube.force-ssl",
-      "https://www.googleapis.com/auth/youtubepartner"
-     ]
-    }
-   }
-  }
- }
-}
diff --git a/test/data/youtube-playlistid.json b/test/data/youtube-playlistid.json
deleted file mode 100644
index b69205f434..0000000000
--- a/test/data/youtube-playlistid.json
+++ /dev/null
@@ -1 +0,0 @@
-{"regionCode": "US", "kind": "youtube#searchListResponse", "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/eoV8llUEbIu5LXnwqBaLOkOK0Hg\"", "pageInfo": {"resultsPerPage": 1, "totalResults": 1}, "items": [{"snippet": {"thumbnails": {"default": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/default.jpg", "width": 120, "height": 90}, "high": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/hqdefault.jpg", "width": 480, "height": 360}, "medium": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/mqdefault.jpg", "width": 320, "height": 180}}, "title": "IETF98", "channelId": "UC8dtK9njBLdFnBahHFp0eZQ", "publishedAt": "2017-03-30T12:41:04.000Z", "liveBroadcastContent": "none", "channelTitle": "IETF - Internet Engineering Task Force", "description": "Videos from the IETF 98 Meeting held in Chicago, Illinois, United States 26-31 March 2017."}, "kind": "youtube#searchResult", "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/X3dbZGRvgpvedtOP0KLGhZLg5UI\"", "id": {"kind": "youtube#playlist", "playlistId": "PLC86T-test"}}]}
diff --git a/test/data/youtube-playlistitems.json b/test/data/youtube-playlistitems.json
deleted file mode 100644
index 4b42e3d21b..0000000000
--- a/test/data/youtube-playlistitems.json
+++ /dev/null
@@ -1 +0,0 @@
-{"items": [{"snippet": {"playlistId": "PLC86T-6ZTP5jo6kIuqdyeYYhsKv9sUwG1", "thumbnails": {"default": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/default.jpg", "width": 120, "height": 90}, "high": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/hqdefault.jpg", "width": 480, "height": 360}, "medium": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/mqdefault.jpg", "width": 320, "height": 180}, "maxres": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/maxresdefault.jpg", "width": 1280, "height": 720}, "standard": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/sddefault.jpg", "width": 640, "height": 480}}, "title": "IETF98 Wrap Up", "resourceId": {"kind": "youtube#video", "videoId": "lhYWB5FFkg4"}, "channelId": "UC8dtK9njBLdFnBahHFp0eZQ", "publishedAt": "2017-04-06T13:32:39.000Z", "channelTitle": "IETF - Internet Engineering Task Force", "position": 0, "description": "Jari Arkko and Alissa Cooper recap some highlights the IETF 98 meeting held 26-31 March 2017 in Chicago, Illinois, United States"}, "kind": "youtube#playlistItem", "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/eW_De3gQF2fRzN_rPBbX-kY7oBI\"", "id": "UExDODZULTZaVFA1am82a0l1cWR5ZVlZaHNLdjlzVXdHMS40OTQ5QjlEMDgzN0FBNUIw"}, {"snippet": {"playlistId": "PLC86T-6ZTP5jo6kIuqdyeYYhsKv9sUwG1", "thumbnails": {"default": {"url": "https://i.ytimg.com/vi/lPSTcBITbvs/default.jpg", "width": 120, "height": 90}, "high": {"url": "https://i.ytimg.com/vi/lPSTcBITbvs/hqdefault.jpg", "width": 480, "height": 360}, "medium": {"url": "https://i.ytimg.com/vi/lPSTcBITbvs/mqdefault.jpg", "width": 320, "height": 180}}, "title": "IETF 98 - QUIC Tutorial", "resourceId": {"kind": "youtube#video", "videoId": "lPSTcBITbvs"}, "channelId": "UC8dtK9njBLdFnBahHFp0eZQ", "publishedAt": "2017-03-30T12:41:35.000Z", "channelTitle": "IETF - Internet Engineering Task Force", "position": 1, "description": "A tutorial about the new QUIC protocol"}, "kind": "youtube#playlistItem", "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/GhKVt6zTuEpFavgtf9GWlWuzX9s\"", "id": "UExDODZULTZaVFA1am82a0l1cWR5ZVlZaHNLdjlzVXdHMS41NkI0NEY2RDEwNTU3Q0M2"}], "kind": "youtube#playlistItemListResponse", "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/jlFue-jZVpFMOuLUXQZH4Y0Lh3Y\"", "pageInfo": {"resultsPerPage": 2, "totalResults": 110}}
diff --git a/test/lib/.gitignore b/test/lib/.gitignore
deleted file mode 100644
index 330b0c2b64..0000000000
--- a/test/lib/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-/django
diff --git a/test/lib/README b/test/lib/README
deleted file mode 100644
index 2dda507f05..0000000000
--- a/test/lib/README
+++ /dev/null
@@ -1,9 +0,0 @@
-This directory will be used to set up packages used for testing if they need any
-special handling which should not be applied to the system-wide setup.
-
-For instance, many of the tests to be run on the Django application should be run
-with a standard Django environment; but there are some Django test features which
-are broken in 0.9.6, and need patching in order to do the testing, like the ability
-to create a test database to run unit tests, according to this issue and patch:
-http://code.djangoproject.com/changeset/5106
-
diff --git a/test/media/floor/.gitignore b/test/media/floor/.gitignore
deleted file mode 100644
index 33662f5545..0000000000
--- a/test/media/floor/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-/*
diff --git a/test/media/photo/.gitignore b/test/media/photo/.gitignore
deleted file mode 100644
index 33662f5545..0000000000
--- a/test/media/photo/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-/*

From 32057f335aa711095fb0907612496430fa61412a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 18 Sep 2024 18:37:02 -0300
Subject: [PATCH 078/656] feat: allow blanket IPR disclosures (#7934)

* refactor: avoid shadowing type()

* style: Black

* feat: is_blanket_disclosure field

* feat: add field to form

* feat: js to mark field required/not required

* feat: blanket disclosure = royalty-free license

* feat: manage licensing radio buttons

* fix: adjust wording/format of disclosure page

* fix: point at RFC 8179 in checkbox label

* test: test blanket disclosure licensing restrictions

* fix: conditionally render is_blanket_disclosure

* test: refactor test case

* test: patent details optional for blanket ipr
---
 ietf/ipr/forms.py                             |  21 ++++
 ...lderiprdisclosure_is_blanket_disclosure.py |  16 +++
 ietf/ipr/models.py                            |  34 ++++--
 ietf/ipr/tests.py                             |  78 +++++++++++--
 ietf/ipr/urls.py                              |   2 +-
 ietf/ipr/views.py                             | 110 +++++++++++-------
 ietf/static/js/ipr-edit.js                    |  67 ++++++++++-
 ietf/templates/ipr/details_edit.html          |  16 ++-
 8 files changed, 275 insertions(+), 69 deletions(-)
 create mode 100644 ietf/ipr/migrations/0004_holderiprdisclosure_is_blanket_disclosure.py

diff --git a/ietf/ipr/forms.py b/ietf/ipr/forms.py
index 62d3f9c216..dac34bddf6 100644
--- a/ietf/ipr/forms.py
+++ b/ietf/ipr/forms.py
@@ -338,7 +338,19 @@ def clean(self):
 
         return cleaned_data
 
+
 class HolderIprDisclosureForm(IprDisclosureFormBase):
+    is_blanket_disclosure = forms.BooleanField(
+        label=mark_safe(
+            'This is a blanket IPR disclosure '
+            '(see Section 5.4.3 of <a href="https://www.ietf.org/rfc/rfc8179.txt">RFC 8179</a>)'
+        ),
+        help_text="In satisfaction of its disclosure obligations, Patent Holder commits to license all of "
+                  "IPR (as defined in RFC 8179) that would have required disclosure under RFC 8179 on a "
+                  "royalty-free (and otherwise reasonable and non-discriminatory) basis. Patent Holder "
+                  "confirms that all other terms and conditions are described in this IPR disclosure.",
+        required=False,
+    )
     licensing = CustomModelChoiceField(IprLicenseTypeName.objects.all(),
         widget=forms.RadioSelect,empty_label=None)
 
@@ -356,6 +368,15 @@ def __init__(self, *args, **kwargs):
         else:
             # entering new disclosure
             self.fields['licensing'].queryset = IprLicenseTypeName.objects.exclude(slug='none-selected')
+        
+        if self.data.get("is_blanket_disclosure", False):
+            # for a blanket disclosure, patent details are not required
+            self.fields["patent_number"].required = False
+            self.fields["patent_inventor"].required = False
+            self.fields["patent_title"].required = False
+            self.fields["patent_date"].required = False
+            # n.b., self.fields["patent_notes"] is never required
+
             
     def clean(self):
         cleaned_data = super(HolderIprDisclosureForm, self).clean()
diff --git a/ietf/ipr/migrations/0004_holderiprdisclosure_is_blanket_disclosure.py b/ietf/ipr/migrations/0004_holderiprdisclosure_is_blanket_disclosure.py
new file mode 100644
index 0000000000..66282b3cd5
--- /dev/null
+++ b/ietf/ipr/migrations/0004_holderiprdisclosure_is_blanket_disclosure.py
@@ -0,0 +1,16 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ipr", "0003_alter_iprdisclosurebase_docs"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="holderiprdisclosure",
+            name="is_blanket_disclosure",
+            field=models.BooleanField(default=False),
+        ),
+    ]
diff --git a/ietf/ipr/models.py b/ietf/ipr/models.py
index 693f19abe9..2d81eb4b42 100644
--- a/ietf/ipr/models.py
+++ b/ietf/ipr/models.py
@@ -3,6 +3,7 @@
 
 
 from django.conf import settings
+from django.core.exceptions import ValidationError
 from django.db import models
 from django.urls import reverse
 from django.utils import timezone
@@ -124,17 +125,30 @@ def is_thirdparty(self):
 
 
 class HolderIprDisclosure(IprDisclosureBase):
-    ietfer_name              = models.CharField(max_length=255, blank=True) # "Whose Personal Belief Triggered..."
-    ietfer_contact_email     = models.EmailField(blank=True)
-    ietfer_contact_info      = models.TextField(blank=True)
-    patent_info              = models.TextField()
-    has_patent_pending       = models.BooleanField(default=False)
-    holder_contact_email     = models.EmailField()
-    holder_contact_name      = models.CharField(max_length=255)
-    holder_contact_info      = models.TextField(blank=True, help_text="Address, phone, etc.")
-    licensing                = ForeignKey(IprLicenseTypeName)
-    licensing_comments       = models.TextField(blank=True)
+    ietfer_name = models.CharField(
+        max_length=255, blank=True
+    )  # "Whose Personal Belief Triggered..."
+    ietfer_contact_email = models.EmailField(blank=True)
+    ietfer_contact_info = models.TextField(blank=True)
+    patent_info = models.TextField()
+    has_patent_pending = models.BooleanField(default=False)
+    holder_contact_email = models.EmailField()
+    holder_contact_name = models.CharField(max_length=255)
+    holder_contact_info = models.TextField(blank=True, help_text="Address, phone, etc.")
+    licensing = ForeignKey(IprLicenseTypeName)
+    licensing_comments = models.TextField(blank=True)
     submitter_claims_all_terms_disclosed = models.BooleanField(default=False)
+    is_blanket_disclosure = models.BooleanField(default=False)
+    
+    def clean(self):
+        if self.is_blanket_disclosure:
+            # If the IprLicenseTypeName does not exist, we have a serious problem and a 500 response is ok,
+            # so not handling failure of the `get()`
+            royalty_free_licensing = IprLicenseTypeName.objects.get(slug="royalty-free")
+            if self.licensing_id != royalty_free_licensing.pk:
+                raise ValidationError(
+                    f'Must select "{royalty_free_licensing.desc}" for a blanket IPR disclosure.')
+
 
 class ThirdPartyIprDisclosure(IprDisclosureBase):
     ietfer_name              = models.CharField(max_length=255) # "Whose Personal Belief Triggered..."
diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py
index b08e359462..3c70567fd8 100644
--- a/ietf/ipr/tests.py
+++ b/ietf/ipr/tests.py
@@ -33,7 +33,7 @@
     IprDocRelFactory,
     IprEventFactory
 )
-from ietf.ipr.forms import DraftForm
+from ietf.ipr.forms import DraftForm, HolderIprDisclosureForm
 from ietf.ipr.mail import (process_response_email, get_reply_to, get_update_submitter_emails,
     get_pseudo_submitter, get_holders, get_update_cc_addrs)
 from ietf.ipr.models import (IprDisclosureBase,GenericIprDisclosure,HolderIprDisclosure,
@@ -272,16 +272,16 @@ def test_sitemap(self):
 
     def test_new_generic(self):
         """Ensure new-generic redirects to new-general"""
-        url = urlreverse("ietf.ipr.views.new", kwargs={ "type": "generic" })
+        url = urlreverse("ietf.ipr.views.new", kwargs={ "_type": "generic" })
         r = self.client.get(url)
         self.assertEqual(r.status_code,302)
-        self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.ipr.views.new", kwargs={ "type": "general"}))
+        self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.ipr.views.new", kwargs={ "_type": "general"}))
 
 
     def test_new_general(self):
         """Add a new general disclosure.  Note: submitter does not need to be logged in.
         """
-        url = urlreverse("ietf.ipr.views.new", kwargs={ "type": "general" })
+        url = urlreverse("ietf.ipr.views.new", kwargs={ "_type": "general" })
 
         # invalid post
         r = self.client.post(url, {
@@ -319,7 +319,7 @@ def test_new_specific(self):
         """
         draft = WgDraftFactory()
         rfc = WgRfcFactory()
-        url = urlreverse("ietf.ipr.views.new", kwargs={ "type": "specific" })
+        url = urlreverse("ietf.ipr.views.new", kwargs={ "_type": "specific" })
 
         # successful post
         empty_outbox()
@@ -375,7 +375,7 @@ def test_new_specific(self):
     def test_new_specific_no_revision(self):
         draft = WgDraftFactory()
         rfc = WgRfcFactory()
-        url = urlreverse("ietf.ipr.views.new", kwargs={ "type": "specific" })
+        url = urlreverse("ietf.ipr.views.new", kwargs={ "_type": "specific" })
 
         # successful post
         empty_outbox()
@@ -409,7 +409,7 @@ def test_new_thirdparty(self):
         """
         draft = WgDraftFactory()
         rfc = WgRfcFactory()
-        url = urlreverse("ietf.ipr.views.new", kwargs={ "type": "third-party" })
+        url = urlreverse("ietf.ipr.views.new", kwargs={ "_type": "third-party" })
 
         # successful post
         empty_outbox()
@@ -456,7 +456,7 @@ def test_edit(self):
         r = self.client.get(url)
         self.assertContains(r, original_ipr.holder_legal_name)
 
-        #url = urlreverse("ietf.ipr.views.new", kwargs={ "type": "specific" })
+        #url = urlreverse("ietf.ipr.views.new", kwargs={ "_type": "specific" })
         # successful post
         empty_outbox()
         post_data = {
@@ -503,7 +503,7 @@ def test_update(self):
         r = self.client.get(url)
         self.assertContains(r, original_ipr.title)
 
-        #url = urlreverse("ietf.ipr.views.new", kwargs={ "type": "specific" })
+        #url = urlreverse("ietf.ipr.views.new", kwargs={ "_type": "specific" })
         # successful post
         empty_outbox()
         r = self.client.post(url, {
@@ -543,7 +543,7 @@ def test_update(self):
 
     def test_update_bad_post(self):
         draft = WgDraftFactory()
-        url = urlreverse("ietf.ipr.views.new", kwargs={ "type": "specific" })
+        url = urlreverse("ietf.ipr.views.new", kwargs={ "_type": "specific" })
 
         empty_outbox()
         r = self.client.post(url, {
@@ -1022,3 +1022,61 @@ def test_revisions_invalid(self):
             "revisions",
             null_char_error_msg,
         )
+
+
+class HolderIprDisclosureFormTests(TestCase):
+    def setUp(self):
+        super().setUp()
+        # Checkboxes that are False are left out of the Form data, not sent back at all. These are
+        # commented out - if they were checked, their value would be "on".
+        self.data = {
+            "holder_legal_name": "Test Legal",
+            "holder_contact_name": "Test Holder",
+            "holder_contact_email": "test@holder.com",
+            "holder_contact_info": "555-555-0100",
+            "ietfer_name": "Test Participant",
+            "ietfer_contact_info": "555-555-0101",
+            "iprdocrel_set-TOTAL_FORMS": 2,
+            "iprdocrel_set-INITIAL_FORMS": 0,
+            "iprdocrel_set-0-document": "1234",  # fake id - validates but won't save()
+            "iprdocrel_set-0-revisions": '00',
+            "iprdocrel_set-1-document": "4567",  # fake id - validates but won't save()
+            # "is_blanket_disclosure": "on", 
+            "patent_number": "SE12345678901",
+            "patent_inventor": "A. Nonymous",
+            "patent_title": "A method of transferring bits",
+            "patent_date": "2000-01-01",
+            # "has_patent_pending": "on",
+            "licensing": "reasonable",
+            "submitter_name": "Test Holder",
+            "submitter_email": "test@holder.com",
+        }
+        
+    def test_blanket_disclosure_licensing_restrictions(self):
+        """when is_blanket_disclosure is True only royalty-free licensing is valid
+        
+        Most of the form functionality is tested via the views in IprTests above. More thorough testing
+        of validation ought to move here so we don't have to exercise the whole Django plumbing repeatedly.
+        """       
+        self.assertTrue(HolderIprDisclosureForm(data=self.data).is_valid())       
+        self.data["is_blanket_disclosure"] = "on"
+        self.assertFalse(HolderIprDisclosureForm(data=self.data).is_valid())       
+        self.data["licensing"] = "royalty-free"
+        self.assertTrue(HolderIprDisclosureForm(data=self.data).is_valid())       
+
+    def test_patent_details_required_unless_blanket(self):
+        self.assertTrue(HolderIprDisclosureForm(data=self.data).is_valid())
+        patent_fields = ["patent_number", "patent_inventor", "patent_title", "patent_date"]
+        # any of the fields being missing should invalidate the form
+        for pf in patent_fields:
+            val = self.data.pop(pf)
+            self.assertFalse(HolderIprDisclosureForm(data=self.data).is_valid())
+            self.data[pf] = val
+
+        # should be optional if is_blanket_disclosure is True
+        self.data["is_blanket_disclosure"] = "on"
+        self.data["licensing"] = "royalty-free"  # also needed for a blanket disclosure
+        for pf in patent_fields:
+            val = self.data.pop(pf)
+            self.assertTrue(HolderIprDisclosureForm(data=self.data).is_valid())
+            self.data[pf] = val
diff --git a/ietf/ipr/urls.py b/ietf/ipr/urls.py
index 6f7b2d4080..2b6abee31c 100644
--- a/ietf/ipr/urls.py
+++ b/ietf/ipr/urls.py
@@ -25,6 +25,6 @@
     url(r'^(?P<id>\d+)/state/$', views.state),
     url(r'^update/$', RedirectView.as_view(url=reverse_lazy('ietf.ipr.views.showlist'), permanent=True)),
     url(r'^update/(?P<id>\d+)/$', views.update),
-    url(r'^new-(?P<type>(specific|generic|general|third-party))/$', views.new),
+    url(r'^new-(?P<_type>(specific|generic|general|third-party))/$', views.new),
     url(r'^search/$', views.search),
 ]
diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py
index 45fad9a2cc..0347c4d78d 100644
--- a/ietf/ipr/views.py
+++ b/ietf/ipr/views.py
@@ -475,28 +475,34 @@ def by_draft_recursive_txt(request):
     return HttpResponse(content, content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET)
 
 
-def new(request, type, updates=None):
+def new(request, _type, updates=None):
     """Submit a new IPR Disclosure.  If the updates field != None, this disclosure
     updates one or more other disclosures."""
     # Note that URL patterns won't ever send updates - updates is only non-null when called from code
 
     # This odd construct flipping generic and general allows the URLs to say 'general' while having a minimal impact on the code.
     # A cleanup to change the code to switch on type 'general' should follow.
-    if type == 'generic' and updates: # Only happens when called directly from the updates view
+    if (
+        _type == "generic" and updates
+    ):  # Only happens when called directly from the updates view
         pass
-    elif type == 'generic':
-        return HttpResponseRedirect(urlreverse('ietf.ipr.views.new',kwargs=dict(type='general')))
-    elif type == 'general':
-        type = 'generic'
+    elif _type == "generic":
+        return HttpResponseRedirect(
+            urlreverse("ietf.ipr.views.new", kwargs=dict(_type="general"))
+        )
+    elif _type == "general":
+        _type = "generic"
     else:
         pass
 
     # 1 to show initially + the template
-    DraftFormset = inlineformset_factory(IprDisclosureBase, IprDocRel, form=DraftForm, can_delete=False, extra=1 + 1)
+    DraftFormset = inlineformset_factory(
+        IprDisclosureBase, IprDocRel, form=DraftForm, can_delete=False, extra=1 + 1
+    )
 
-    if request.method == 'POST':
-        form = ipr_form_mapping[type](request.POST)
-        if type != 'generic':
+    if request.method == "POST":
+        form = ipr_form_mapping[_type](request.POST)
+        if _type != "generic":
             draft_formset = DraftFormset(request.POST, instance=IprDisclosureBase())
         else:
             draft_formset = None
@@ -505,72 +511,92 @@ def new(request, type, updates=None):
             person = Person.objects.get(name="(System)")
         else:
             person = request.user.person
-            
+
         # check formset validity
-        if type != 'generic':
+        if _type != "generic":
             valid_formsets = draft_formset.is_valid()
         else:
             valid_formsets = True
-            
+
         if form.is_valid() and valid_formsets:
-            if 'updates' in form.cleaned_data:
-                updates = form.cleaned_data['updates']
-                del form.cleaned_data['updates']
+            if "updates" in form.cleaned_data:
+                updates = form.cleaned_data["updates"]
+                del form.cleaned_data["updates"]
             disclosure = form.save(commit=False)
             disclosure.by = person
-            disclosure.state = IprDisclosureStateName.objects.get(slug='pending')
+            disclosure.state = IprDisclosureStateName.objects.get(slug="pending")
             disclosure.save()
-            
-            if type != 'generic':
+
+            if _type != "generic":
                 draft_formset = DraftFormset(request.POST, instance=disclosure)
                 draft_formset.save()
 
             set_disclosure_title(disclosure)
             disclosure.save()
-            
+
             if updates:
                 for ipr in updates:
-                    RelatedIpr.objects.create(source=disclosure,target=ipr,relationship_id='updates')
-                
+                    RelatedIpr.objects.create(
+                        source=disclosure, target=ipr, relationship_id="updates"
+                    )
+
             # create IprEvent
             IprEvent.objects.create(
-                type_id='submitted',
+                type_id="submitted",
                 by=person,
                 disclosure=disclosure,
-                desc="Disclosure Submitted")
+                desc="Disclosure Submitted",
+            )
 
             # send email notification
-            (to, cc) = gather_address_lists('ipr_disclosure_submitted')
-            send_mail(request, to, ('IPR Submitter App', 'ietf-ipr@ietf.org'),
-                'New IPR Submission Notification',
+            (to, cc) = gather_address_lists("ipr_disclosure_submitted")
+            send_mail(
+                request,
+                to,
+                ("IPR Submitter App", "ietf-ipr@ietf.org"),
+                "New IPR Submission Notification",
                 "ipr/new_update_email.txt",
-                {"ipr": disclosure,},
-                cc=cc)
-            
+                {
+                    "ipr": disclosure,
+                },
+                cc=cc,
+            )
+
             return render(request, "ipr/submitted.html")
 
     else:
         if updates:
             original = IprDisclosureBase(id=updates).get_child()
             initial = model_to_dict(original)
-            initial.update({'updates':str(updates), })
-            patent_info = text_to_dict(initial.get('patent_info', ''))
+            initial.update(
+                {
+                    "updates": str(updates),
+                }
+            )
+            patent_info = text_to_dict(initial.get("patent_info", ""))
             if list(patent_info.keys()):
-                patent_dict = dict([ ('patent_'+k.lower(), v) for k,v in list(patent_info.items()) ])
+                patent_dict = dict(
+                    [("patent_" + k.lower(), v) for k, v in list(patent_info.items())]
+                )
             else:
-                patent_dict = {'patent_notes': initial.get('patent_info', '')}
+                patent_dict = {"patent_notes": initial.get("patent_info", "")}
             initial.update(patent_dict)
-            form = ipr_form_mapping[type](initial=initial)
+            form = ipr_form_mapping[_type](initial=initial)
         else:
-            form = ipr_form_mapping[type]()
-        disclosure = IprDisclosureBase()    # dummy disclosure for inlineformset
+            form = ipr_form_mapping[_type]()
+        disclosure = IprDisclosureBase()  # dummy disclosure for inlineformset
         draft_formset = DraftFormset(instance=disclosure)
 
-    return render(request, "ipr/details_edit.html",  {
-        'form': form,
-        'draft_formset':draft_formset,
-        'type':type,
-    })
+    return render(
+        request,
+        "ipr/details_edit.html",
+        {
+            "form": form,
+            "draft_formset": draft_formset,
+            "type": _type,
+        },
+    )
+
 
 @role_required('Secretariat',)
 def notify(request, id, type):
diff --git a/ietf/static/js/ipr-edit.js b/ietf/static/js/ipr-edit.js
index 9af5b03591..9d07503791 100644
--- a/ietf/static/js/ipr-edit.js
+++ b/ietf/static/js/ipr-edit.js
@@ -69,4 +69,69 @@ $(document)
             form.find(".draft-row")
                 .each(updateRevisions);
         }, 10);
-    });
\ No newline at end of file
+
+        // Manage fields that depend on the Blanket IPR Disclosure choice
+        const blanketCheckbox = document.getElementById('id_is_blanket_disclosure') 
+        if (blanketCheckbox) {
+            const patentDetailInputs = [
+                // The ids are from the HolderIprDisclosureForm and its base form class,
+                // intentionally excluding patent_notes because it's never required
+                'id_patent_number',
+                'id_patent_inventor',
+                'id_patent_title',
+                'id_patent_date'
+            ].map((id) => document.getElementById(id))
+            const patentDetailRowDivs = patentDetailInputs.map(
+                (elt) => elt.closest('div.row')
+            )
+            const royaltyFreeLicensingRadio = document.querySelector(
+                '#id_licensing input[value="royalty-free"]'
+            )
+            let lastSelectedLicensingRadio
+            const otherLicensingRadios = document.querySelectorAll(
+                '#id_licensing input:not([value="royalty-free"])'
+            ) 
+            
+            const handleBlanketCheckboxChange = () => {
+                const isBlanket = blanketCheckbox.checked
+                // Update required fields
+                for (elt of patentDetailInputs) {
+                    // disable the input element
+                    elt.required = !isBlanket
+                }
+                for (elt of patentDetailRowDivs) {
+                    // update the styling on the row that indicates required field
+                    if (isBlanket) {
+                        elt.classList.remove('required')
+                    } else {
+                        elt.classList.add('required')
+                    }
+                }
+                // Update licensing selection
+                if (isBlanket) {
+                    lastSelectedLicensingRadio = document.querySelector(
+                        '#id_licensing input:checked'
+                    )
+                    royaltyFreeLicensingRadio.checked = true
+                    otherLicensingRadios
+                        .forEach(
+                            (elt) => elt.setAttribute('disabled', '')
+                        )
+                } else {
+                    royaltyFreeLicensingRadio.checked = false
+                    if (lastSelectedLicensingRadio) {
+                        lastSelectedLicensingRadio.checked = true
+                    }
+                    otherLicensingRadios
+                        .forEach(
+                            (elt) => elt.removeAttribute('disabled')
+                        )
+                }
+            }
+            handleBlanketCheckboxChange()
+            blanketCheckbox.addEventListener(
+                'change', 
+                (evt) => handleBlanketCheckboxChange()
+            )
+        }
+    });
diff --git a/ietf/templates/ipr/details_edit.html b/ietf/templates/ipr/details_edit.html
index 7caf28f1a7..1aadb5bb3d 100644
--- a/ietf/templates/ipr/details_edit.html
+++ b/ietf/templates/ipr/details_edit.html
@@ -32,7 +32,7 @@ <h1>The Patent Disclosure and Licensing Declaration Template for Third Party IPR
             regarding an IETF document or contribution when the person letting the
             IETF know about the patent has no relationship with the patent owners.
             Click
-            <a href="{% url 'ietf.ipr.views.new' type='specific' %}">here</a>
+            <a href="{% url 'ietf.ipr.views.new' 'specific' %}">here</a>
             if you want to disclose information about patents or patent
             applications where you do have a relationship to the patent owners or
             patent applicants.
@@ -121,12 +121,11 @@ <h2 class="mt-4">
         {% endif %}
         {% if type != "generic" %}
             <h2 class="mt-4">{% cycle section %}. IETF document or other contribution to which this IPR disclosure relates</h2>
-            <p class="form-text">
+            <p>
                 If an Internet-Draft or RFC includes multiple parts and it is not
                 reasonably apparent which part of such Internet-Draft or RFC is alleged
-                to be covered by the patent information disclosed in Section
-                V(A) or V(B), please identify the sections of
-                the Internet-Draft or RFC that are alleged to be so
+                to be covered by the patent information disclosed in Section V,
+                please identify the sections of the Internet-Draft or RFC that are alleged to be so
                 covered.
             </p>
             {{ draft_formset.management_form }}
@@ -154,6 +153,13 @@ <h2 class="mt-4">
             <small>i.e., patents or patent applications required to be disclosed by Section 5 of RFC8179</small>
         </h2>
         {% if form.patent_number %}
+            {% if form.is_blanket_disclosure %}
+                <p>
+                    This IPR disclosure must either identify a specific patent or patents in sections V(A) and V(B)
+                    below, or be made as a blanket IPR disclosure. 
+                </p>
+                {% bootstrap_field form.is_blanket_disclosure layout='horizontal' %}
+            {% endif %}
             <p>
                 A. For granted patents or published pending patent applications,
                 please provide the following information:

From 25fd4fce5dc6101435e287d5515ed12d638f6490 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 23 Sep 2024 09:41:18 -0500
Subject: [PATCH 079/656] fix: show correct time for cancelled sessions.
 (#7958)

---
 ietf/templates/group/meetings-row.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/templates/group/meetings-row.html b/ietf/templates/group/meetings-row.html
index 3bbe1d425b..25605ec0f1 100644
--- a/ietf/templates/group/meetings-row.html
+++ b/ietf/templates/group/meetings-row.html
@@ -18,7 +18,7 @@
                 <div class="badge rounded-pill text-bg-secondary">{{ s.current_status_name }}</div>
                 {% if s.current_status == "canceled" %}
                     {% with timeslot=s.official_timeslotassignment.timeslot %}
-                        <span class="text-decoration-line-through text-secondary session-time date me-3" data-start-utc="{{ timeslot.time|utc|date:'Y-m-d' }}" data-end-utc="{{ timeslot.end_time|utc|date:'Y-m-d' }}"></span>
+                        <span class="text-decoration-line-through text-secondary session-time date me-3" data-start-utc="{{ timeslot.time|utc|date:'Y-m-d H:i' }}" data-end-utc="{{ timeslot.end_time|utc|date:'Y-m-d H:i' }}"></span>
                     {% endwith %}
                 {% endif %}
             {% endif %}

From 06b9df10ee77b061fbde53d54acab77dab953f90 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 23 Sep 2024 10:09:50 -0500
Subject: [PATCH 080/656] feat: warn about materials for cancelled sessions
 (#7959)

* feat: warn about materials for cancelled sessions

* fix: handle viewing a DocHistory material object
---
 ietf/doc/tests.py                         | 11 +++++++++++
 ietf/doc/views_doc.py                     |  8 ++++++++
 ietf/templates/doc/document_material.html |  3 +++
 3 files changed, 22 insertions(+)

diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index 5e5b741fe5..f5f715162d 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -1683,6 +1683,17 @@ def test_document_material(self):
 
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
         self.assertEqual(r.status_code, 200)
+        self.assertNotContains(r, "The session for this document was cancelled.")
+
+        SchedulingEvent.objects.create(
+            session=session,
+            status_id='canceled',
+            by = Person.objects.get(user__username="marschairman"), 
+        )
+
+        r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, "The session for this document was cancelled.")
 
     def test_document_ballot(self):
         doc = IndividualDraftFactory()
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index d12dd04181..aad45920c6 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -870,6 +870,13 @@ def document_main(request, name, rev=None, document_html=False):
                     and doc.group.features.has_nonsession_materials
                     and doc.type_id in doc.group.features.material_types
             )
+
+        session_statusid = None
+        actual_doc = doc if isinstance(doc,Document) else doc.doc
+        if actual_doc.session_set.count() == 1:
+            if actual_doc.session_set.get().schedulingevent_set.exists():
+                session_statusid = actual_doc.session_set.get().schedulingevent_set.order_by("-time").first().status_id
+
         return render(request, "doc/document_material.html",
                                   dict(doc=doc,
                                        top=top,
@@ -882,6 +889,7 @@ def document_main(request, name, rev=None, document_html=False):
                                        can_upload = can_upload,
                                        other_types=other_types,
                                        presentations=presentations,
+                                       session_statusid=session_statusid,
                                        ))
 
 
diff --git a/ietf/templates/doc/document_material.html b/ietf/templates/doc/document_material.html
index 7537082df9..cf6dd1ab64 100644
--- a/ietf/templates/doc/document_material.html
+++ b/ietf/templates/doc/document_material.html
@@ -153,6 +153,9 @@
             </a>
         {% endif %}
     </p>
+    {% if session_statusid == "canceled" %}
+       <div class="alert alert-warning">The session for this document was cancelled.</div>
+    {% endif %}
     <div id="materials-content" class="card mt-5">
         <div class="card-header">{{ doc.name }}-{{ doc.rev }}</div>
         <div class="card-body{% if content_is_html %} text-break{% endif %}">

From 5581dc79e1c6f6f07b680b7be47ec2d3811d8f39 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 24 Sep 2024 12:46:00 -0300
Subject: [PATCH 081/656] feat: doc mgrs can edit/remind action holders (#7971)

* feat: doc mgrs can edit/remind action holders

* test: test docman_roles, not "chair"
---
 ietf/doc/tests.py                      | 12 ++++++++-
 ietf/doc/tests_draft.py                | 27 +++++++++++++++++---
 ietf/doc/views_doc.py                  | 34 +++++++++++++++++++++++---
 ietf/templates/doc/document_draft.html |  4 +--
 4 files changed, 67 insertions(+), 10 deletions(-)

diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index f5f715162d..e3534ba725 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -59,7 +59,7 @@
 from ietf.meeting.factories import ( MeetingFactory, SessionFactory, SessionPresentationFactory,
      ProceedingsMaterialFactory )
 
-from ietf.name.models import SessionStatusName, BallotPositionName, DocTypeName
+from ietf.name.models import SessionStatusName, BallotPositionName, DocTypeName, RoleName
 from ietf.person.models import Person
 from ietf.person.factories import PersonFactory, EmailFactory
 from ietf.utils.mail import outbox, empty_outbox
@@ -1450,6 +1450,14 @@ def test_document_draft_action_holders_buttons(self, mock_method):
         """Buttons for action holders should be shown when AD or secretary"""
         draft = WgDraftFactory()
         draft.action_holders.set([PersonFactory()])
+        other_group = GroupFactory(type_id=draft.group.type_id)
+
+        # create a test RoleName and put it in the docman_roles for the document group
+        RoleName.objects.create(slug="wrangler", name="Wrangler", used=True)
+        draft.group.features.docman_roles.append("wrangler")
+        draft.group.features.save()
+        wrangler = RoleFactory(group=draft.group, name_id="wrangler").person
+        wrangler_of_other_group = RoleFactory(group=other_group, name_id="wrangler").person
 
         url = urlreverse('ietf.doc.views_doc.document_main', kwargs=dict(name=draft.name))
         edit_ah_url = urlreverse('ietf.doc.views_doc.edit_action_holders', kwargs=dict(name=draft.name))
@@ -1482,6 +1490,8 @@ def _run_test(username=None, expect_buttons=False):
 
         _run_test(None, False)
         _run_test('plain', False)
+        _run_test(wrangler_of_other_group.user.username, False)
+        _run_test(wrangler.user.username, True)
         _run_test('ad', True)
         _run_test('secretary', True)
 
diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py
index cc22e22139..dcb71bdf69 100644
--- a/ietf/doc/tests_draft.py
+++ b/ietf/doc/tests_draft.py
@@ -26,7 +26,7 @@
     WriteupDocEvent, DocRelationshipName, IanaExpertDocEvent )
 from ietf.doc.utils import get_tags_for_stream_id, create_ballot_if_not_open
 from ietf.doc.views_draft import AdoptDraftForm
-from ietf.name.models import StreamName, DocTagName
+from ietf.name.models import StreamName, DocTagName, RoleName
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.group.models import Group, Role
 from ietf.person.factories import PersonFactory, EmailFactory
@@ -935,6 +935,7 @@ def setUp(self):
         super().setUp()
         doc = WgDraftFactory(group__acronym='mars',shepherd=PersonFactory(user__username='plain',name='Plain Man').email_set.first())
         self.docname = doc.name
+        self.doc_group = doc.group
 
     def test_doc_change_stream(self):
         url = urlreverse('ietf.doc.views_draft.change_stream', kwargs=dict(name=self.docname))
@@ -1319,8 +1320,10 @@ def do_doc_change_action_holders_test(self, username):
         RoleFactory(name_id='techadv', person=PersonFactory(), group=doc.group)
         RoleFactory(name_id='editor', person=PersonFactory(), group=doc.group)
         RoleFactory(name_id='secr', person=PersonFactory(), group=doc.group)
-        
+        some_other_chair = RoleFactory(name_id="chair").person
+
         url = urlreverse('ietf.doc.views_doc.edit_action_holders', kwargs=dict(name=doc.name))
+        login_testing_unauthorized(self, some_other_chair.user.username, url)  # other chair can't edit action holders
         login_testing_unauthorized(self, username, url)
         
         r = self.client.get(url)
@@ -1363,6 +1366,14 @@ def _test_changing_ah(action_holders, reason):
         _test_changing_ah(doc.authors(), 'authors can do it, too')
         _test_changing_ah([], 'clear it back out')
 
+    def test_doc_change_action_holders_as_doc_manager(self):
+        # create a test RoleName and put it in the docman_roles for the document group
+        RoleName.objects.create(slug="wrangler", name="Wrangler", used=True)
+        self.doc_group.features.docman_roles.append("wrangler")
+        self.doc_group.features.save()
+        wrangler = RoleFactory(group=self.doc_group, name_id="wrangler").person
+        self.do_doc_change_action_holders_test(wrangler.user.username)
+
     def test_doc_change_action_holders_as_secretary(self):
         self.do_doc_change_action_holders_test('secretary')
 
@@ -1372,9 +1383,11 @@ def test_doc_change_action_holders_as_ad(self):
     def do_doc_remind_action_holders_test(self, username):
         doc = Document.objects.get(name=self.docname)
         doc.action_holders.set(PersonFactory.create_batch(3))
-        
+        some_other_chair = RoleFactory(name_id="chair").person
+    
         url = urlreverse('ietf.doc.views_doc.remind_action_holders', kwargs=dict(name=doc.name))
         
+        login_testing_unauthorized(self, some_other_chair.user.username, url)  # other chair can't send reminder
         login_testing_unauthorized(self, username, url)
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
@@ -1401,6 +1414,14 @@ def do_doc_remind_action_holders_test(self, username):
         self.client.post(url)
         self.assertEqual(len(outbox), 1)  # still 1
 
+    def test_doc_remind_action_holders_as_doc_manager(self):
+        # create a test RoleName and put it in the docman_roles for the document group
+        RoleName.objects.create(slug="wrangler", name="Wrangler", used=True)
+        self.doc_group.features.docman_roles.append("wrangler")
+        self.doc_group.features.save()
+        wrangler = RoleFactory(group=self.doc_group, name_id="wrangler").person
+        self.do_doc_remind_action_holders_test(wrangler.user.username)
+
     def test_doc_remind_action_holders_as_ad(self):
         self.do_doc_remind_action_holders_test('ad')
 
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index aad45920c6..02d59f0df3 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -42,6 +42,7 @@
 from pathlib import Path
 
 from django.core.cache import caches
+from django.core.exceptions import PermissionDenied
 from django.db.models import Max
 from django.http import HttpResponse, Http404, HttpResponseBadRequest
 from django.shortcuts import render, get_object_or_404, redirect
@@ -403,6 +404,10 @@ def document_main(request, name, rev=None, document_html=False):
 
         can_edit_replaces = has_role(request.user, ("Area Director", "Secretariat", "IRTF Chair", "WG Chair", "RG Chair", "WG Secretary", "RG Secretary"))
 
+        can_edit_action_holders = can_edit or (
+            request.user.is_authenticated and group.has_role(request.user, group.features.docman_roles)
+        )
+
         is_author = request.user.is_authenticated and doc.documentauthor_set.filter(person__user=request.user).exists()
         can_view_possibly_replaces = can_edit_replaces or is_author
 
@@ -660,6 +665,7 @@ def document_main(request, name, rev=None, document_html=False):
                                        can_edit_iana_state=can_edit_iana_state,
                                        can_edit_consensus=can_edit_consensus,
                                        can_edit_replaces=can_edit_replaces,
+                                       can_edit_action_holders=can_edit_action_holders,
                                        can_view_possibly_replaces=can_view_possibly_replaces,
                                        can_request_review=can_request_review,
                                        can_submit_unsolicited_review_for_teams=can_submit_unsolicited_review_for_teams,
@@ -1871,11 +1877,21 @@ def add_fields(self, form, index):
         })
 
 
-@role_required('Area Director', 'Secretariat')
+@login_required
 def edit_action_holders(request, name):
     """Change the set of action holders for a doc"""
     doc = get_object_or_404(Document, name=name)
-    
+
+    can_edit = has_role(request.user, ("Area Director", "Secretariat")) or (
+        doc.group and doc.group.has_role(request.user, doc.group.features.docman_roles)
+    )
+    if not can_edit:
+        # Keep the list of roles in this message up-to-date with the can_edit logic
+        message = "Restricted to roles: Area Director, Secretariat"
+        if doc.group and doc.group.acronym != "none":
+            message += f", and document managers for the {doc.group.acronym} group"
+        raise PermissionDenied(message)
+
     if request.method == 'POST':
         form = ActionHoldersForm(request.POST)
         if form.is_valid():
@@ -1985,10 +2001,20 @@ class ReminderEmailForm(forms.Form):
         strip=True,
     )
 
-@role_required('Area Director', 'Secretariat')
+@login_required
 def remind_action_holders(request, name):
     doc = get_object_or_404(Document, name=name)
-    
+
+    can_edit = has_role(request.user, ("Area Director", "Secretariat")) or (
+        doc.group and doc.group.has_role(request.user, doc.group.features.docman_roles)
+    )
+    if not can_edit:
+        # Keep the list of roles in this message up-to-date with the can_edit logic
+        message = "Restricted to roles: Area Director, Secretariat"
+        if doc.group and doc.group.acronym != "none":
+            message += f", and document managers for the {doc.group.acronym} group"
+        raise PermissionDenied(message)
+
     if request.method == 'POST':
         form = ReminderEmailForm(request.POST)
         if form.is_valid():
diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html
index befbc759f1..d5ea074422 100644
--- a/ietf/templates/doc/document_draft.html
+++ b/ietf/templates/doc/document_draft.html
@@ -304,7 +304,7 @@
                             Action Holder{{ doc.documentactionholder_set.all|pluralize }}
                         </th>
                         <td class="edit">
-                            {% if can_edit %}
+                            {% if can_edit_action_holders %}
                                 <a class="btn btn-primary btn-sm"
                                    href="{% url 'ietf.doc.views_doc.edit_action_holders' name=doc.name %}">
                                     Edit
@@ -319,7 +319,7 @@
                                             {% person_link action_holder.person title=action_holder.role_for_doc %} {{ action_holder|action_holder_badge }}
                                         </div>
                                     {% endfor %}
-                                    {% if can_edit %}
+                                    {% if can_edit_action_holders %}
                                         <a class="btn btn-primary btn-sm mt-3"
                                            href="{% url "ietf.doc.views_doc.remind_action_holders" name=doc.name %}">
                                             <i class="bi bi-envelope">

From 5ea4cc13bb908cec98290069b74bdc11eb7a7f24 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Wed, 25 Sep 2024 03:53:08 +1200
Subject: [PATCH 082/656] feat: change theme dynamically when system
 preferences change (#7956)

---
 client/App.vue | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/client/App.vue b/client/App.vue
index 2a6c5e6e98..7750674296 100644
--- a/client/App.vue
+++ b/client/App.vue
@@ -30,17 +30,24 @@ const appContainer = ref(null)
 // Set user theme
 // --------------------------------------------------------------------
 
-const desiredTheme = window.localStorage?.getItem('theme')
-if (desiredTheme === 'dark') {
-  siteStore.theme = 'dark'
-} else if (desiredTheme === 'light') {
-  siteStore.theme = 'light'
-} else if (window.matchMedia("(prefers-color-scheme: dark)").matches) {
-  siteStore.theme = 'dark'
-} else {
-  siteStore.theme = 'light'
+function updateTheme() {
+  const desiredTheme = window.localStorage?.getItem('theme')
+  if (desiredTheme === 'dark') {
+    siteStore.theme = 'dark'
+  } else if (desiredTheme === 'light') {
+    siteStore.theme = 'light'
+  } else if (window.matchMedia("(prefers-color-scheme: dark)").matches) {
+    siteStore.theme = 'dark'
+  } else {
+    siteStore.theme = 'light'
+  }
 }
 
+updateTheme()
+
+// this change event fires for either light or dark changes
+window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', updateTheme)
+
 // --------------------------------------------------------------------
 // Handle browser resize
 // --------------------------------------------------------------------

From fbcfa19add42c4ea525a844763ae3fbb9d495239 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 24 Sep 2024 14:13:12 -0300
Subject: [PATCH 083/656] feat: remove "AD is watching" state (#7960)

* feat: remove "AD is watching" state

* chore: update names.json

* refactor: use idexists state, not dead

* fix: remove guidance to use watching state

* chore: remove references to 'watching' state

* feat: remove create_in_state from edit_info view

* test: update test

* style: Black + move class closer to use

* refactor: remove lint

* fix: restore missing admonition

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/doc/expire.py                            |   4 +-
 .../0024_remove_ad_is_watching_states.py      | 121 ++++++++++
 ietf/doc/templatetags/ballot_icon.py          |   2 +-
 ietf/doc/tests_ballot.py                      |   2 +-
 ietf/doc/tests_draft.py                       |  86 +++----
 ietf/doc/views_ballot.py                      |   4 +-
 ietf/doc/views_doc.py                         |   2 +-
 ietf/doc/views_draft.py                       | 222 +++++++++++-------
 ietf/doc/views_search.py                      |   2 +-
 ietf/group/views.py                           |   2 +-
 ietf/idindex/index.py                         |   2 +-
 ietf/name/fixtures/names.json                 |   2 +-
 ietf/templates/doc/draft/change_state.html    |   6 +-
 13 files changed, 311 insertions(+), 146 deletions(-)
 create mode 100644 ietf/doc/migrations/0024_remove_ad_is_watching_states.py

diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py
index 1650b4ddf5..681ca963a3 100644
--- a/ietf/doc/expire.py
+++ b/ietf/doc/expire.py
@@ -34,8 +34,8 @@ def expirable_drafts(queryset=None):
 
     # Populate this first time through (but after django has been set up)
     if nonexpirable_states is None:
-        # all IESG states except I-D Exists, AD Watching, and Dead block expiry
-        nonexpirable_states = list(State.objects.filter(used=True, type="draft-iesg").exclude(slug__in=("idexists","watching", "dead")))
+        # all IESG states except I-D Exists and Dead block expiry
+        nonexpirable_states = list(State.objects.filter(used=True, type="draft-iesg").exclude(slug__in=("idexists", "dead")))
         # sent to RFC Editor and RFC Published block expiry (the latter
         # shouldn't be possible for an active draft, though)
         nonexpirable_states += list(State.objects.filter(used=True, type__in=("draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"), slug__in=("rfc-edit", "pub")))
diff --git a/ietf/doc/migrations/0024_remove_ad_is_watching_states.py b/ietf/doc/migrations/0024_remove_ad_is_watching_states.py
new file mode 100644
index 0000000000..0c0fb0ad25
--- /dev/null
+++ b/ietf/doc/migrations/0024_remove_ad_is_watching_states.py
@@ -0,0 +1,121 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+
+from django.db import migrations
+
+
+def get_helper(DocHistory, RelatedDocument, RelatedDocHistory, DocumentAuthor, DocHistoryAuthor):
+    """Dependency injection wrapper"""
+
+    def save_document_in_history(doc):
+        """Save a snapshot of document and related objects in the database.
+        
+        Local copy of ietf.doc.utils.save_document_in_history() to avoid depending on the
+        code base in a migration.
+        """
+    
+        def get_model_fields_as_dict(obj):
+            return dict((field.name, getattr(obj, field.name))
+                        for field in obj._meta.fields
+                        if field is not obj._meta.pk)
+    
+        # copy fields
+        fields = get_model_fields_as_dict(doc)
+        fields["doc"] = doc
+        fields["name"] = doc.name
+    
+        dochist = DocHistory(**fields)
+        dochist.save()
+    
+        # copy many to many
+        for field in doc._meta.many_to_many:
+            if field.remote_field.through and field.remote_field.through._meta.auto_created:
+                hist_field = getattr(dochist, field.name)
+                hist_field.clear()
+                hist_field.set(getattr(doc, field.name).all())
+    
+        # copy remaining tricky many to many
+        def transfer_fields(obj, HistModel):
+            mfields = get_model_fields_as_dict(item)
+            # map doc -> dochist
+            for k, v in mfields.items():
+                if v == doc:
+                    mfields[k] = dochist
+            HistModel.objects.create(**mfields)
+    
+        for item in RelatedDocument.objects.filter(source=doc):
+            transfer_fields(item, RelatedDocHistory)
+    
+        for item in DocumentAuthor.objects.filter(document=doc):
+            transfer_fields(item, DocHistoryAuthor)
+    
+        return dochist
+    
+    return save_document_in_history
+
+
+def forward(apps, schema_editor):
+    """Mark watching draft-iesg state unused after removing it from Documents"""
+    StateDocEvent = apps.get_model("doc", "StateDocEvent")
+    Document = apps.get_model("doc", "Document")
+    State = apps.get_model("doc", "State")
+    StateType = apps.get_model("doc", "StateType")
+    Person = apps.get_model("person", "Person")
+    
+    save_document_in_history = get_helper(
+        DocHistory=apps.get_model("doc", "DocHistory"),
+        RelatedDocument=apps.get_model("doc", "RelatedDocument"),
+        RelatedDocHistory=apps.get_model("doc", "RelatedDocHistory"),
+        DocumentAuthor=apps.get_model("doc", "DocumentAuthor"),
+        DocHistoryAuthor=apps.get_model("doc", "DocHistoryAuthor"),
+    )
+
+    draft_iesg_state_type = StateType.objects.get(slug="draft-iesg")
+    idexists_state = State.objects.get(type=draft_iesg_state_type, slug="idexists")
+    watching_state = State.objects.get(type=draft_iesg_state_type, slug="watching")
+    system_person = Person.objects.get(name="(System)")
+
+    # Remove state from documents that currently have it
+    for doc in Document.objects.filter(states=watching_state):
+        assert doc.type_id == "draft"
+        doc.states.remove(watching_state)
+        doc.states.add(idexists_state)
+        e = StateDocEvent.objects.create(
+            type="changed_state",
+            by=system_person,
+            doc=doc,
+            rev=doc.rev,
+            desc=f"{draft_iesg_state_type.label} changed to <b>{idexists_state.name}</b> from {watching_state.name}",
+            state_type=draft_iesg_state_type,
+            state=idexists_state,
+        )
+        doc.time = e.time
+        doc.save()
+        save_document_in_history(doc)
+    assert not Document.objects.filter(states=watching_state).exists()
+
+    # Mark state as unused
+    watching_state.used = False
+    watching_state.save()
+
+
+def reverse(apps, schema_editor):
+    """Mark watching draft-iesg state as used
+    
+    Does not try to re-apply the state to Documents modified by the forward migration. This
+    could be done in theory, but would either require dangerous history rewriting or add a
+    lot of history junk.
+    """
+    State = apps.get_model("doc", "State")
+    StateType = apps.get_model("doc", "StateType")
+    State.objects.filter(
+        type=StateType.objects.get(slug="draft-iesg"), slug="watching"
+    ).update(used=True)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("doc", "0023_bofreqspamstate"),
+    ]
+
+    operations = [migrations.RunPython(forward, reverse)]
diff --git a/ietf/doc/templatetags/ballot_icon.py b/ietf/doc/templatetags/ballot_icon.py
index 5f59822c7b..a94c145007 100644
--- a/ietf/doc/templatetags/ballot_icon.py
+++ b/ietf/doc/templatetags/ballot_icon.py
@@ -184,7 +184,7 @@ def state_age_colored(doc):
         if not iesg_state:
             return ""
 
-        if iesg_state in ["dead", "watching", "pub", "idexists"]:
+        if iesg_state in ["dead", "pub", "idexists"]:
             return ""
         try:
             state_datetime = (
diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py
index 0c99ed281a..f95ba8812b 100644
--- a/ietf/doc/tests_ballot.py
+++ b/ietf/doc/tests_ballot.py
@@ -559,7 +559,7 @@ def test_issue_ballot_warn_if_early(self):
         q = PyQuery(r.content)
         self.assertFalse(q('[class=text-danger]:contains("not completed IETF Last Call")'))
 
-        for state_slug in ["lc", "watching", "ad-eval"]:
+        for state_slug in ["lc", "ad-eval"]:
             draft.set_state(State.objects.get(type="draft-iesg",slug=state_slug))
             r = self.client.get(url)
             self.assertEqual(r.status_code, 200)
diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py
index dcb71bdf69..09a7b38999 100644
--- a/ietf/doc/tests_draft.py
+++ b/ietf/doc/tests_draft.py
@@ -26,7 +26,7 @@
     WriteupDocEvent, DocRelationshipName, IanaExpertDocEvent )
 from ietf.doc.utils import get_tags_for_stream_id, create_ballot_if_not_open
 from ietf.doc.views_draft import AdoptDraftForm
-from ietf.name.models import StreamName, DocTagName, RoleName
+from ietf.name.models import DocTagName, RoleName
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.group.models import Group, Role
 from ietf.person.factories import PersonFactory, EmailFactory
@@ -471,69 +471,61 @@ def test_edit_telechat_date(self):
         self.assertIn("may not leave enough time", get_payload_text(outbox[-1]))
 
     def test_start_iesg_process_on_draft(self):
-
         draft = WgDraftFactory(
             name="draft-ietf-mars-test2",
-            group__acronym='mars',
+            group__acronym="mars",
             intended_std_level_id="ps",
-            authors=[Person.objects.get(user__username='ad')],
-            )
-        
-        url = urlreverse('ietf.doc.views_draft.edit_info', kwargs=dict(name=draft.name))
+            authors=[Person.objects.get(user__username="ad")],
+        )
+
+        url = urlreverse("ietf.doc.views_draft.edit_info", kwargs=dict(name=draft.name))
         login_testing_unauthorized(self, "secretary", url)
 
         # normal get
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('form select[name=intended_std_level]')), 1)
-        self.assertEqual("", q('form textarea[name=notify]')[0].value.strip())
+        self.assertEqual(len(q("form select[name=intended_std_level]")), 1)
+        self.assertEqual("", q("form textarea[name=notify]")[0].value.strip())
 
-        # add
-        events_before = draft.docevent_set.count()
+        events_before = list(draft.docevent_set.values_list("id", flat=True))
         mailbox_before = len(outbox)
 
         ad = Person.objects.get(name="Areað Irector")
 
-        r = self.client.post(url,
-                             dict(intended_std_level=str(draft.intended_std_level_id),
-                                  ad=ad.pk,
-                                  create_in_state=State.objects.get(used=True, type="draft-iesg", slug="watching").pk,
-                                  notify="test@example.com",
-                                  telechat_date="",
-                                  ))
+        r = self.client.post(
+            url,
+            dict(
+                intended_std_level=str(draft.intended_std_level_id),
+                ad=ad.pk,
+                notify="test@example.com",
+                telechat_date="",
+            ),
+        )
         self.assertEqual(r.status_code, 302)
 
         draft = Document.objects.get(name=draft.name)
-        self.assertEqual(draft.get_state_slug("draft-iesg"), "watching")
+        self.assertEqual(draft.get_state_slug("draft-iesg"), "pub-req")
+        self.assertEqual(draft.get_state_slug("draft-stream-ietf"), "sub-pub")
         self.assertEqual(draft.ad, ad)
-        self.assertTrue(not draft.latest_event(TelechatDocEvent, type="scheduled_for_telechat"))
-        self.assertEqual(draft.docevent_set.count(), events_before + 4)
-        self.assertCountEqual(draft.action_holders.all(), [draft.ad])
-        events = list(draft.docevent_set.order_by('time', 'id'))
-        self.assertEqual(events[-4].type, "started_iesg_process")
-        self.assertEqual(len(outbox), mailbox_before+1)
-        self.assertTrue('IESG processing' in outbox[-1]['Subject'])
-        self.assertTrue('draft-ietf-mars-test2@' in outbox[-1]['To']) 
-
-        # Redo, starting in publication requested to make sure WG state is also set
-        draft.set_state(State.objects.get(type_id='draft-iesg', slug='idexists'))
-        draft.set_state(State.objects.get(type='draft-stream-ietf',slug='writeupw'))
-        draft.stream = StreamName.objects.get(slug='ietf')
-        draft.action_holders.clear()
-        draft.save_with_history([DocEvent.objects.create(doc=draft, rev=draft.rev, type="changed_stream", by=Person.objects.get(user__username="secretary"), desc="Test")])
-        r = self.client.post(url,
-                             dict(intended_std_level=str(draft.intended_std_level_id),
-                                  ad=ad.pk,
-                                  create_in_state=State.objects.get(used=True, type="draft-iesg", slug="pub-req").pk,
-                                  notify="test@example.com",
-                                  telechat_date="",
-                                  ))
-        self.assertEqual(r.status_code, 302)
-        draft = Document.objects.get(name=draft.name)
-        self.assertEqual(draft.get_state_slug('draft-iesg'),'pub-req')
-        self.assertEqual(draft.get_state_slug('draft-stream-ietf'),'sub-pub')
+        self.assertTrue(
+            not draft.latest_event(TelechatDocEvent, type="scheduled_for_telechat")
+        )
+        # check that the expected events were created (don't insist on ordering)
+        self.assertCountEqual(
+            draft.docevent_set.exclude(id__in=events_before).values_list("type", flat=True),
+            [
+                "changed_action_holders",  # action holders set to AD
+                "changed_document",  # WG state set to sub-pub
+                "changed_document",  # AD set
+                "changed_document",  # state change notice email set
+                "started_iesg_process",  # IESG state is now pub-req
+            ],
+        )
         self.assertCountEqual(draft.action_holders.all(), [draft.ad])
+        self.assertEqual(len(outbox), mailbox_before + 1)
+        self.assertTrue("IESG processing" in outbox[-1]["Subject"])
+        self.assertTrue("draft-ietf-mars-test2@" in outbox[-1]["To"])
 
     def test_edit_consensus(self):
         draft = WgDraftFactory()
@@ -750,10 +742,6 @@ def test_expire_drafts(self):
 
         self.assertEqual(len(list(get_expired_drafts())), 1)
 
-        draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="watching"))
-
-        self.assertEqual(len(list(get_expired_drafts())), 1)
-
         draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="iesg-eva"))
 
         self.assertEqual(len(list(get_expired_drafts())), 0)
diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py
index 2b7faac92b..4ff40d2268 100644
--- a/ietf/doc/views_ballot.py
+++ b/ietf/doc/views_ballot.py
@@ -641,7 +641,7 @@ def ballot_writeupnotes(request, name):
                 existing.save()
 
             if "issue_ballot" in request.POST and not ballot_already_approved:
-                if prev_state.slug in ['watching', 'writeupw', 'goaheadw']:
+                if prev_state.slug in ['writeupw', 'goaheadw']:
                     new_state = State.objects.get(used=True, type="draft-iesg", slug='iesg-eva')
                     prev_tags = doc.tags.filter(slug__in=IESG_SUBSTATE_TAGS)
                     doc.set_state(new_state)
@@ -708,7 +708,7 @@ def ballot_writeupnotes(request, name):
                                    back_url=doc.get_absolute_url(),
                                    ballot_issued=bool(doc.latest_event(type="sent_ballot_announcement")),
                                    warn_lc = not doc.docevent_set.filter(lastcalldocevent__expires__date__lt=date_today(DEADLINE_TZINFO)).exists(),
-                                   warn_unexpected_state= prev_state if bool(prev_state.slug in ['watching', 'ad-eval', 'lc']) else None,
+                                   warn_unexpected_state= prev_state if bool(prev_state.slug in ['ad-eval', 'lc']) else None,
                                    ballot_writeup_form=form,
                                    need_intended_status=need_intended_status,
                                    ))
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 02d59f0df3..915dcebde5 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -587,7 +587,7 @@ def document_main(request, name, rev=None, document_html=False):
         if doc.get_state_slug() not in ["rfc", "expired"] and doc.stream_id in ("ietf",) and not snapshot:
             if iesg_state_slug == 'idexists' and can_edit:
                 actions.append(("Begin IESG Processing", urlreverse('ietf.doc.views_draft.edit_info', kwargs=dict(name=doc.name)) + "?new=1"))
-            elif can_edit_stream_info and (iesg_state_slug in ('idexists','watching')):
+            elif can_edit_stream_info and (iesg_state_slug == 'idexists'):
                 actions.append(("Submit to IESG for Publication", urlreverse('ietf.doc.views_draft.to_iesg', kwargs=dict(name=doc.name))))
 
         if request.user.is_authenticated and hasattr(request.user, "person"):
diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py
index 30175491da..d709aedd42 100644
--- a/ietf/doc/views_draft.py
+++ b/ietf/doc/views_draft.py
@@ -487,40 +487,6 @@ def change_intention(request, name):
                                    doc=doc,
                                    ))
 
-class EditInfoForm(forms.Form):
-    intended_std_level = forms.ModelChoiceField(IntendedStdLevelName.objects.filter(used=True), empty_label="(None)", required=True, label="Intended RFC status")
-    area = forms.ModelChoiceField(Group.objects.filter(type="area", state="active"), empty_label="(None - individual submission)", required=False, label="Assigned to area")
-    ad = forms.ModelChoiceField(Person.objects.filter(role__name="ad", role__group__state="active",role__group__type='area').order_by('name'), label="Responsible AD", empty_label="(None)", required=True)
-    create_in_state = forms.ModelChoiceField(State.objects.filter(used=True, type="draft-iesg", slug__in=("pub-req", "watching")), empty_label=None, required=False)
-    notify = forms.CharField(
-        widget=forms.Textarea,
-        max_length=1023,
-        label="Notice emails",
-        help_text="Separate email addresses with commas.",
-        required=False,
-    )
-    telechat_date = forms.TypedChoiceField(coerce=lambda x: datetime.datetime.strptime(x, '%Y-%m-%d').date(), empty_value=None, required=False, widget=forms.Select(attrs={'onchange':'make_bold()'}))
-    returning_item = forms.BooleanField(required=False)
-
-    def __init__(self, *args, **kwargs):
-        super(self.__class__, self).__init__(*args, **kwargs)
-
-        # if previous AD is now ex-AD, append that person to the list
-        ad_pk = self.initial.get('ad')
-        choices = self.fields['ad'].choices
-        if ad_pk and ad_pk not in [pk for pk, name in choices]:
-            self.fields['ad'].choices = list(choices) + [("", "-------"), (ad_pk, Person.objects.get(pk=ad_pk).plain_name())]
-        
-        # telechat choices
-        dates = [d.date for d in TelechatDate.objects.active().order_by('date')]
-        init = kwargs['initial']['telechat_date']
-        if init and init not in dates:
-            dates.insert(0, init)
-
-        self.fields['telechat_date'].choices = [("", "(not on agenda)")] + [(d, d.strftime("%Y-%m-%d")) for d in dates]
-
-        # returning item is rendered non-standard
-        self.standard_fields = [x for x in self.visible_fields() if x.name not in ('returning_item',)]
 
 def to_iesg(request,name):
     """ Submit an IETF stream document to the IESG for publication """ 
@@ -619,7 +585,71 @@ def doc_event(type, by, doc, desc):
                                    notify=notify,
                                   ))
 
-@role_required('Area Director','Secretariat')
+class EditInfoForm(forms.Form):
+    intended_std_level = forms.ModelChoiceField(
+        IntendedStdLevelName.objects.filter(used=True),
+        empty_label="(None)",
+        required=True,
+        label="Intended RFC status",
+    )
+    area = forms.ModelChoiceField(
+        Group.objects.filter(type="area", state="active"),
+        empty_label="(None - individual submission)",
+        required=False,
+        label="Assigned to area",
+    )
+    ad = forms.ModelChoiceField(
+        Person.objects.filter(
+            role__name="ad", role__group__state="active", role__group__type="area"
+        ).order_by("name"),
+        label="Responsible AD",
+        empty_label="(None)",
+        required=True,
+    )
+    notify = forms.CharField(
+        widget=forms.Textarea,
+        max_length=1023,
+        label="Notice emails",
+        help_text="Separate email addresses with commas.",
+        required=False,
+    )
+    telechat_date = forms.TypedChoiceField(
+        coerce=lambda x: datetime.datetime.strptime(x, "%Y-%m-%d").date(),
+        empty_value=None,
+        required=False,
+        widget=forms.Select(attrs={"onchange": "make_bold()"}),
+    )
+    returning_item = forms.BooleanField(required=False)
+
+    def __init__(self, *args, **kwargs):
+        super(self.__class__, self).__init__(*args, **kwargs)
+
+        # if previous AD is now ex-AD, append that person to the list
+        ad_pk = self.initial.get("ad")
+        choices = self.fields["ad"].choices
+        if ad_pk and ad_pk not in [pk for pk, name in choices]:
+            self.fields["ad"].choices = list(choices) + [
+                ("", "-------"),
+                (ad_pk, Person.objects.get(pk=ad_pk).plain_name()),
+            ]
+
+        # telechat choices
+        dates = [d.date for d in TelechatDate.objects.active().order_by("date")]
+        init = kwargs["initial"]["telechat_date"]
+        if init and init not in dates:
+            dates.insert(0, init)
+
+        self.fields["telechat_date"].choices = [("", "(not on agenda)")] + [
+            (d, d.strftime("%Y-%m-%d")) for d in dates
+        ]
+
+        # returning item is rendered non-standard
+        self.standard_fields = [
+            x for x in self.visible_fields() if x.name not in ("returning_item",)
+        ]
+
+
+@role_required("Area Director", "Secretariat")
 def edit_info(request, name):
     """Edit various Internet-Draft attributes, notifying parties as
     necessary and logging changes as document events."""
@@ -628,7 +658,8 @@ def edit_info(request, name):
         raise Http404
 
     new_document = False
-    if doc.get_state_slug("draft-iesg") == "idexists": # FIXME: should probably receive "new document" as argument to view instead of this
+    # FIXME: should probably receive "new document" as argument to view instead of this
+    if doc.get_state_slug("draft-iesg") == "idexists":
         new_document = True
         doc.notify = get_initial_notify(doc)
 
@@ -636,34 +667,45 @@ def edit_info(request, name):
     initial_telechat_date = e.telechat_date if e else None
     initial_returning_item = bool(e and e.returning_item)
 
-    if request.method == 'POST':
-        form = EditInfoForm(request.POST,
-                            initial=dict(ad=doc.ad_id,
-                                         telechat_date=initial_telechat_date))
+    if request.method == "POST":
+        form = EditInfoForm(
+            request.POST,
+            initial=dict(ad=doc.ad_id, telechat_date=initial_telechat_date),
+        )
         if form.is_valid():
             by = request.user.person
+            pubreq_state = State.objects.get(type="draft-iesg", slug="pub-req")
 
             r = form.cleaned_data
             events = []
 
             if new_document:
-                doc.set_state(r['create_in_state'])
+                doc.set_state(pubreq_state)
 
                 # Is setting the WG state here too much of a hidden side-effect?
-                if r['create_in_state'].slug=='pub-req':
-                    if doc.stream and doc.stream.slug=='ietf' and doc.group and doc.group.type_id == 'wg':
-                        submitted_state = State.objects.get(type='draft-stream-ietf',slug='sub-pub')
-                        doc.set_state(submitted_state)
-                        e = DocEvent()
-                        e.type = "changed_document"
-                        e.by = by
-                        e.doc = doc
-                        e.rev = doc.rev
-                        e.desc = "Working group state set to %s" % submitted_state.name
-                        e.save()
-                        events.append(e)
+                if (
+                    doc.stream
+                    and doc.stream.slug == "ietf"
+                    and doc.group
+                    and doc.group.type_id == "wg"
+                ):
+                    submitted_state = State.objects.get(
+                        type="draft-stream-ietf", slug="sub-pub"
+                    )
+                    doc.set_state(submitted_state)
+                    e = DocEvent()
+                    e.type = "changed_document"
+                    e.by = by
+                    e.doc = doc
+                    e.rev = doc.rev
+                    e.desc = "Working group state set to %s" % submitted_state.name
+                    e.save()
+                    events.append(e)
 
-                replaces = Document.objects.filter(targets_related__source=doc, targets_related__relationship="replaces")
+                replaces = Document.objects.filter(
+                    targets_related__source=doc,
+                    targets_related__relationship="replaces",
+                )
                 if replaces:
                     # this should perhaps be somewhere else, e.g. the
                     # place where the replace relationship is established?
@@ -672,7 +714,10 @@ def edit_info(request, name):
                     e.by = Person.objects.get(name="(System)")
                     e.doc = doc
                     e.rev = doc.rev
-                    e.desc = "Earlier history may be found in the Comment Log for <a href=\"%s\">%s</a>" % (replaces[0], replaces[0].get_absolute_url())
+                    e.desc = (
+                        'Earlier history may be found in the Comment Log for <a href="%s">%s</a>'
+                        % (replaces[0], replaces[0].get_absolute_url())
+                    )
                     e.save()
                     events.append(e)
 
@@ -681,7 +726,10 @@ def edit_info(request, name):
                 e.by = by
                 e.doc = doc
                 e.rev = doc.rev
-                e.desc = "Document is now in IESG state <b>%s</b>" % doc.get_state("draft-iesg").name
+                e.desc = (
+                    "Document is now in IESG state <b>%s</b>"
+                    % doc.get_state("draft-iesg").name
+                )
                 e.save()
                 events.append(e)
 
@@ -691,9 +739,9 @@ def desc(attr, new, old):
                 entry = "%(attr)s changed to <b>%(new)s</b> from <b>%(old)s</b>"
                 if new_document:
                     entry = "%(attr)s changed to <b>%(new)s</b>"
-                
+
                 return entry % dict(attr=attr, new=new, old=old)
-            
+
             def diff(attr, name):
                 v = getattr(doc, attr)
                 if r[attr] != v:
@@ -701,9 +749,9 @@ def diff(attr, name):
                     setattr(doc, attr, r[attr])
 
             # update the attributes, keeping track of what we're doing
-            diff('intended_std_level', "Intended Status")
-            diff('ad', "Responsible AD")
-            diff('notify', "State Change Notice email list")
+            diff("intended_std_level", "Intended Status")
+            diff("ad", "Responsible AD")
+            diff("notify", "State Change Notice email list")
 
             if doc.group.type_id in ("individ", "area"):
                 if not r["area"]:
@@ -717,12 +765,16 @@ def diff(attr, name):
                     doc.group = r["area"]
 
             for c in changes:
-                events.append(DocEvent.objects.create(doc=doc, rev=doc.rev, by=by, desc=c, type="changed_document"))
+                events.append(
+                    DocEvent.objects.create(
+                        doc=doc, rev=doc.rev, by=by, desc=c, type="changed_document"
+                    )
+                )
 
             # Todo - chase this
-            e = update_telechat(request, doc, by,
-                                r['telechat_date'], r['returning_item'])
-
+            e = update_telechat(
+                request, doc, by, r["telechat_date"], r["returning_item"]
+            )
             if e:
                 events.append(e)
 
@@ -730,40 +782,44 @@ def diff(attr, name):
 
             if new_document:
                 # If we created a new doc, update the action holders as though it
-                # started in idexists and moved to its create_in_state. Do this
+                # started in idexists and moved to pub-req. Do this
                 # after the doc has been updated so, e.g., doc.ad is set.
                 update_action_holders(
                     doc,
-                    State.objects.get(type='draft-iesg', slug='idexists'),
-                    r['create_in_state']
+                    State.objects.get(type="draft-iesg", slug="idexists"),
+                    pubreq_state,
                 )
 
             if changes:
                 email_iesg_processing_document(request, doc, changes)
-                
+
             return HttpResponseRedirect(doc.get_absolute_url())
     else:
-        init = dict(intended_std_level=doc.intended_std_level_id,
-                    area=doc.group_id,
-                    ad=doc.ad_id,
-                    notify=doc.notify,
-                    telechat_date=initial_telechat_date,
-                    returning_item=initial_returning_item,
-                    )
+        init = dict(
+            intended_std_level=doc.intended_std_level_id,
+            area=doc.group_id,
+            ad=doc.ad_id,
+            notify=doc.notify,
+            telechat_date=initial_telechat_date,
+            returning_item=initial_returning_item,
+        )
 
         form = EditInfoForm(initial=init)
 
     # optionally filter out some fields
-    if not new_document:
-        form.standard_fields = [x for x in form.standard_fields if x.name != "create_in_state"]
     if doc.group.type_id not in ("individ", "area"):
         form.standard_fields = [x for x in form.standard_fields if x.name != "area"]
 
-    return render(request, 'doc/draft/edit_info.html',
-                              dict(doc=doc,
-                                   form=form,
-                                   user=request.user,
-                                   ballot_issued=doc.latest_event(type="sent_ballot_announcement")))
+    return render(
+        request,
+        "doc/draft/edit_info.html",
+        dict(
+            doc=doc,
+            form=form,
+            user=request.user,
+            ballot_issued=doc.latest_event(type="sent_ballot_announcement"),
+        ),
+    )
 
 @role_required('Area Director','Secretariat')
 def request_resurrect(request, name):
diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py
index 528fb05a22..4fa3b2560c 100644
--- a/ietf/doc/views_search.py
+++ b/ietf/doc/views_search.py
@@ -757,7 +757,7 @@ def drafts_in_last_call(request):
     })
 
 def drafts_in_iesg_process(request):
-    states = State.objects.filter(type="draft-iesg").exclude(slug__in=('idexists', 'pub', 'dead', 'watching', 'rfcqueue'))
+    states = State.objects.filter(type="draft-iesg").exclude(slug__in=('idexists', 'pub', 'dead', 'rfcqueue'))
     title = "Documents in IESG process"
 
     grouped_docs = []
diff --git a/ietf/group/views.py b/ietf/group/views.py
index e3fd7e80d9..71986384e0 100644
--- a/ietf/group/views.py
+++ b/ietf/group/views.py
@@ -731,7 +731,7 @@ def dependencies(request, acronym, group_type=None):
             {
                 "id": x.became_rfc().name if x.became_rfc() else x.name,
                 "rfc": x.type_id == "rfc" or x.became_rfc() is not None,
-                "post-wg": x.get_state_slug("draft-iesg") not in ["idexists", "watching", "dead"],
+                "post-wg": x.get_state_slug("draft-iesg") not in ["idexists", "dead"],
                 "expired": x.get_state_slug("draft") == "expired",
                 "replaced": x.get_state_slug("draft") == "repl",
                 "group": x.group.acronym if x.group and x.group.acronym != "none" else "",
diff --git a/ietf/idindex/index.py b/ietf/idindex/index.py
index 11c4521485..4f021c0dc7 100644
--- a/ietf/idindex/index.py
+++ b/ietf/idindex/index.py
@@ -51,7 +51,7 @@ def add_line(f1, f2, f3, f4):
         res.append(f1 + "\t" + f2 + "\t" + f3 + "\t" + f4)
 
 
-    inactive_states = ["idexists", "pub", "watching", "dead"]
+    inactive_states = ["idexists", "pub", "dead"]
 
     excludes = list(State.objects.filter(type="draft", slug__in=["rfc","repl"]))
     includes = list(State.objects.filter(type="draft-iesg").exclude(slug__in=inactive_states))
diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json
index 401767657d..8f1262b4cf 100644
--- a/ietf/name/fixtures/names.json
+++ b/ietf/name/fixtures/names.json
@@ -312,7 +312,7 @@
       "order": 42,
       "slug": "watching",
       "type": "draft-iesg",
-      "used": true
+      "used": false
     },
     "model": "doc.state",
     "pk": 11
diff --git a/ietf/templates/doc/draft/change_state.html b/ietf/templates/doc/draft/change_state.html
index 3515c290e1..1cae8358a4 100644
--- a/ietf/templates/doc/draft/change_state.html
+++ b/ietf/templates/doc/draft/change_state.html
@@ -12,9 +12,9 @@ <h1>
     </h1>
     {% if state.slug == "dead" %}
         <p class="alert alert-warning my-3">
-           This document is in IESG state "Dead". It is unusual to change
-           this to anything other than "AD is watching", and this should
-           never be used as a replacement for Begin IESG Processing.
+            This document is in IESG state "Dead". It is unusual to change this to
+            anything other than "I-D Exists" and this should never be used as a
+            replacement for Begin IESG Processing.
         </p>
     {% endif %}
     <a class="btn btn-info my-3"

From a345673c0d4f9c5bade059ea2ba22142a1904a90 Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Wed, 2 Oct 2024 04:24:21 +1300
Subject: [PATCH 084/656] build: Install fonts from xml2rfc-fonts (#6300)

* build: Install fonts from xml2rfc-fonts

Fixes #6294

* fixup! build: Install fonts from xml2rfc-fonts
---
 docker/base.Dockerfile | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile
index 4a307c6c62..e2465f33c2 100644
--- a/docker/base.Dockerfile
+++ b/docker/base.Dockerfile
@@ -125,6 +125,15 @@ ENV LC_ALL en_US.UTF-8
 ADD https://raw.githubusercontent.com/ietf-tools/idnits-mirror/main/idnits /usr/local/bin/
 RUN chmod +rx /usr/local/bin/idnits
 
+# Install required fonts
+RUN mkdir -p /tmp/fonts && \
+    wget -q -O /tmp/fonts.tar.gz https://github.com/ietf-tools/xml2rfc-fonts/archive/refs/tags/3.22.0.tar.gz && \
+    tar zxf /tmp/fonts.tar.gz -C /tmp/fonts && \
+    mv /tmp/fonts/*/noto/* /usr/local/share/fonts/ && \
+    mv /tmp/fonts/*/roboto_mono/* /usr/local/share/fonts/ && \
+    rm -rf /tmp/fonts.tar.gz /tmp/fonts/ && \
+    fc-cache -f
+
 # Turn off rsyslog kernel logging (doesn't work in Docker)
 RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf
 

From b34bbe59ad93726872105a1ed18fec6114f89502 Mon Sep 17 00:00:00 2001
From: Sangho Na <sangho@nsh.nz>
Date: Thu, 3 Oct 2024 05:44:52 +1300
Subject: [PATCH 085/656] fix: hide rounded pill if session has no purpose
 specified (#7997)

---
 ietf/meeting/tests_views.py          | 32 ++++++++++++++++++++++++++++
 ietf/templates/meeting/requests.html |  2 +-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index f951eb682d..642edcb9b4 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -7173,6 +7173,20 @@ def test_meeting_requests(self):
             status_id='schedw',
             add_to_schedule=False,
         )
+        session_with_none_purpose = SessionFactory(
+            meeting=meeting,
+            group__parent=area,
+            purpose_id="none",
+            status_id="schedw",
+            add_to_schedule=False,
+        )
+        tutorial_session = SessionFactory(
+            meeting=meeting,
+            group__parent=area,
+            purpose_id="tutorial",
+            status_id="schedw",
+            add_to_schedule=False,
+        )
         def _sreq_edit_link(sess):
             return urlreverse(
                 'ietf.secr.sreq.views.edit',
@@ -7211,6 +7225,8 @@ def _sreq_edit_link(sess):
         self.assertContains(r, _sreq_edit_link(proposed_wg_session))  # link to the session request
         self.assertContains(r, rg_session.group.acronym)
         self.assertContains(r, _sreq_edit_link(rg_session))  # link to the session request
+        self.assertContains(r, session_with_none_purpose.group.acronym)
+        self.assertContains(r, tutorial_session.group.acronym)
         # check headings - note that the special types (has_meetings, etc) do not have a group parent
         # so they show up in 'other'
         q = PyQuery(r.content)
@@ -7218,6 +7234,22 @@ def _sreq_edit_link(sess):
         self.assertEqual(len(q('h2#other-groups')), 1)
         self.assertEqual(len(q('h2#irtf')), 1)  # rg group has irtf group as parent
 
+        # check rounded pills
+        self.assertNotContains(  # no rounded pill for sessions with regular purpose
+            r,
+            '<span class="badge rounded-pill text-bg-info">Regular</span>',
+            html=True,
+        )
+        self.assertNotContains(  # no rounded pill for session with no purpose specified
+            r,
+            '<span class="badge rounded-pill text-bg-info">None</span>',
+            html=True,
+        )
+        self.assertContains(  # rounded pill for session with non-regular purpose
+            r,
+            '<span class="badge rounded-pill text-bg-info">Tutorial</span>',
+            html=True,
+        )
 
     def test_request_minutes(self):
         meeting = MeetingFactory(type_id='ietf')
diff --git a/ietf/templates/meeting/requests.html b/ietf/templates/meeting/requests.html
index 18d8c5cf8c..3008ceb662 100644
--- a/ietf/templates/meeting/requests.html
+++ b/ietf/templates/meeting/requests.html
@@ -154,7 +154,7 @@ <h2 class="mt-5" id="{% firstof area.grouper.acronym "other-groups" %}">
                             <a href="{% url "ietf.secr.sreq.views.edit" num=meeting.number acronym=session.group.acronym %}">
                                 {{ session.group.acronym }}
                             </a>
-                            {% if session.purpose_id != "regular" %}
+                            {% if session.purpose_id != "regular" and session.purpose_id != "none" %}
                                 <br><span class="badge rounded-pill text-bg-info">{{session.purpose}}</span>
                             {% endif %}
                             {% if session.joint_with_groups.count %}joint with {{ session.joint_with_groups_acronyms|join:' ' }}{% endif %}

From 364dec3e33cc0df8804d1792584c4ac97336cba6 Mon Sep 17 00:00:00 2001
From: Sangho Na <sangho@nsh.nz>
Date: Thu, 3 Oct 2024 11:37:49 +1300
Subject: [PATCH 086/656] fix: fix wg anchor tags (case-sensitive) (#7995)

* fix: fix wg anchor tags in areas page

* fix: fix wg anchor tags in IESG agenda page

* test: ensure wg anchor tags to be all uppercase

* test: improve tests
---
 ietf/group/tests_info.py                |  6 ++++++
 ietf/iesg/tests.py                      | 14 ++++++++++++++
 ietf/templates/group/active_areas.html  |  2 +-
 ietf/templates/iesg/agenda_charter.html |  2 +-
 ietf/templates/iesg/agenda_doc.html     |  2 +-
 5 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py
index 561f542f42..95fe9aa2e5 100644
--- a/ietf/group/tests_info.py
+++ b/ietf/group/tests_info.py
@@ -87,6 +87,12 @@ def test_active_groups(self):
             r = self.client.get(url)
             self.assertEqual(r.status_code, 200)
             self.assertContains(r, g.acronym)
+            if t == "area":
+                q = PyQuery(r.content)
+                wg_url = urlreverse("ietf.group.views.active_groups", kwargs=dict(group_type="wg"))
+                href = f"{wg_url}#{g.acronym.upper()}"
+                self.assertEqual(q(f"h2#id-{g.acronym} a").attr("href"), href)
+                self.assertEqual(q(f'h2#id-{g.acronym} a[href="{href}"]').text(), f"({g.acronym.upper()})")
 
         url = urlreverse('ietf.group.views.active_groups', kwargs=dict())
         r = self.client.get(url)
diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py
index 8438cb44dd..42e838ebdf 100644
--- a/ietf/iesg/tests.py
+++ b/ietf/iesg/tests.py
@@ -404,6 +404,8 @@ def test_agenda(self):
 
         self.assertContains(r, action_items.text)
 
+        q = PyQuery(r.content)
+
         for k, d in self.telechat_docs.items():
             if d.type_id == "charter":
                 self.assertContains(r, d.group.name, msg_prefix="%s '%s' not in response" % (k, d.group.name))
@@ -412,6 +414,18 @@ def test_agenda(self):
                 self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name))
                 self.assertContains(r, d.title, msg_prefix="%s '%s' title not in response" % (k, d.title))
 
+            if d.type_id in ["charter", "draft"]:
+                if d.group.parent is None:
+                    continue
+                wg_url = urlreverse("ietf.group.views.active_groups", kwargs=dict(group_type="wg"))
+                href = f"{wg_url}#{d.group.parent.acronym.upper()}"
+                texts = [elem.text.strip() for elem in q(f'a[href="{href}"]')]
+                self.assertGreater(len(texts), 0)
+                if d.type_id == "charter":
+                    self.assertTrue(any(t == d.group.parent.acronym.upper() for t in texts))
+                elif d.type_id == "draft":
+                    self.assertTrue(any(t == f"({d.group.parent.acronym.upper()})" for t in texts))
+
         for i, mi in enumerate(self.mgmt_items, start=1):
             s = "6." + str(i)
             self.assertContains(r, s, msg_prefix="Section '%s' not in response" % s)
diff --git a/ietf/templates/group/active_areas.html b/ietf/templates/group/active_areas.html
index 63637889ea..8355d13bbc 100644
--- a/ietf/templates/group/active_areas.html
+++ b/ietf/templates/group/active_areas.html
@@ -65,7 +65,7 @@ <h1>Areas</h1>
     {% for area in areas %}
         <h2 class="mt-5" id="id-{{ area.acronym|slugify }}">
             {{ area.name }}
-            <a href="{% url 'ietf.group.views.active_groups' group_type='wg' %}#{{ area.acronym }}">({{ area.acronym|upper }})</a>
+            <a href="{% url 'ietf.group.views.active_groups' group_type='wg' %}#{{ area.acronym|upper }}">({{ area.acronym|upper }})</a>
         </h2>
         {% if area.description %}
             <p>
diff --git a/ietf/templates/iesg/agenda_charter.html b/ietf/templates/iesg/agenda_charter.html
index 20bd23e06f..b4124914db 100644
--- a/ietf/templates/iesg/agenda_charter.html
+++ b/ietf/templates/iesg/agenda_charter.html
@@ -24,7 +24,7 @@
         <div class="row">
             <div class="col-3 text-end fw-bold">Area</div>
             <div class="col">
-                <a href="{% url 'ietf.group.views.active_groups' group_type='wg' %}#{{ doc.group.parent.acronym }}">
+                <a href="{% url 'ietf.group.views.active_groups' group_type='wg' %}#{{ doc.group.parent.acronym|upper }}">
                     {{ doc.group.parent.acronym|upper }}</a>
                 ({% person_link doc.ad %})
             </div>
diff --git a/ietf/templates/iesg/agenda_doc.html b/ietf/templates/iesg/agenda_doc.html
index ae78d96d60..87051fda26 100644
--- a/ietf/templates/iesg/agenda_doc.html
+++ b/ietf/templates/iesg/agenda_doc.html
@@ -37,7 +37,7 @@
             <div class="col-3 text-end fw-bold">Token</div>
             <div class="col">
                 {% person_link doc.ad %}
-                <a href="{% url 'ietf.group.views.active_groups' group_type='wg' %}#{{ doc.group.parent.acronym }}">
+                <a href="{% url 'ietf.group.views.active_groups' group_type='wg' %}#{{ doc.group.parent.acronym|upper }}">
                     ({{ doc.group.parent.acronym|upper }})
                 </a>
             </div>

From 6aff818b72c9e2e94d3432288ec366ce7f16b86a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 9 Oct 2024 12:56:00 -0300
Subject: [PATCH 087/656] feat: remove deprecated /api/submit/ endpoint (#8017)

* feat: remove deprecated /api/submit endpoint

* chore: remove unused imports

* test: test api_submit_tombstone view
---
 ietf/api/urls.py                           |   2 +-
 ietf/submit/forms.py                       | 273 +--------------------
 ietf/submit/tests.py                       | 143 +----------
 ietf/submit/views.py                       | 107 +-------
 ietf/templates/api/index.html              |   2 +-
 ietf/templates/submit/api_submit_info.html |  57 +----
 6 files changed, 27 insertions(+), 557 deletions(-)

diff --git a/ietf/api/urls.py b/ietf/api/urls.py
index 396b3813d6..48525dfda2 100644
--- a/ietf/api/urls.py
+++ b/ietf/api/urls.py
@@ -59,7 +59,7 @@
     # Email alias listing
     url(r'^person/email/$', api_views.active_email_list),
     # Draft submission API
-    url(r'^submit/?$', submit_views.api_submit),
+    url(r'^submit/?$', submit_views.api_submit_tombstone),
     # Draft upload API
     url(r'^submission/?$', submit_views.api_submission),
     # Draft submission state API
diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py
index 4e5644b36e..2781d3365a 100644
--- a/ietf/submit/forms.py
+++ b/ietf/submit/forms.py
@@ -2,19 +2,16 @@
 # -*- coding: utf-8 -*-
 
 
-import io
 import os
 import re
 import datetime
 import email
 import sys
 import tempfile
-import xml2rfc
 from contextlib import ExitStack
 
 from email.utils import formataddr
 from typing import Tuple
-from unidecode import unidecode
 
 from django import forms
 from django.conf import settings
@@ -37,10 +34,8 @@
 from ietf.submit.utils import validate_submission_name, validate_submission_rev, validate_submission_document_date, remote_ip
 from ietf.submit.parsers.plain_parser import PlainParser
 from ietf.submit.parsers.xml_parser import XMLParser
-from ietf.utils import log
 from ietf.utils.draft import PlaintextDraft
 from ietf.utils.fields import ModelMultipleChoiceField
-from ietf.utils.text import normalize_text
 from ietf.utils.timezone import date_today
 from ietf.utils.xmldraft import InvalidXMLError, XMLDraft, XMLParseError
 
@@ -371,273 +366,6 @@ def deduce_group(name):
             return None
 
 
-class DeprecatedSubmissionBaseUploadForm(SubmissionBaseUploadForm):
-    def clean(self):
-        def format_messages(where, e, log):
-            out = log.write_out.getvalue().splitlines()
-            err = log.write_err.getvalue().splitlines()
-            m = str(e)
-            if m:
-                m = [ m ]
-            else:
-                import traceback
-                typ, val, tb = sys.exc_info()
-                m = traceback.format_exception(typ, val, tb)
-                m = [ l.replace('\n ', ':\n ') for l in m ]
-            msgs = [s for s in (["Error from xml2rfc (%s):" % (where,)] + m + out +  err) if s]
-            return msgs
-
-        if self.shutdown and not has_role(self.request.user, "Secretariat"):
-            raise forms.ValidationError(self.cutoff_warning)
-
-        for ext in self.formats:
-            f = self.cleaned_data.get(ext, None)
-            if not f:
-                continue
-            self.file_types.append('.%s' % ext)
-        if not ('.txt' in self.file_types or '.xml' in self.file_types):
-            if not self.errors:
-                raise forms.ValidationError('Unexpected submission file types; found %s, but %s is required' % (', '.join(self.file_types), ' or '.join(self.base_formats)))
-
-        #debug.show('self.cleaned_data["xml"]')
-        if self.cleaned_data.get('xml'):
-            #if not self.cleaned_data.get('txt'):
-            xml_file = self.cleaned_data.get('xml')
-            file_name = {}
-            xml2rfc.log.write_out = io.StringIO()   # open(os.devnull, "w")
-            xml2rfc.log.write_err = io.StringIO()   # open(os.devnull, "w")
-            tfn = None
-            with ExitStack() as stack:
-                @stack.callback
-                def cleanup():  # called when context exited, even in case of exception
-                    if tfn is not None:
-                        os.unlink(tfn)
-
-                # We need to write the xml file to disk in order to hand it
-                # over to the xml parser.  XXX FIXME: investigate updating
-                # xml2rfc to be able to work with file handles to in-memory
-                # files.
-                name, ext = os.path.splitext(os.path.basename(xml_file.name))
-                with tempfile.NamedTemporaryFile(prefix=name+'-',
-                                                 suffix='.xml',
-                                                 mode='wb+',
-                                                 delete=False) as tf:
-                    tfn = tf.name
-                    for chunk in xml_file.chunks():
-                        tf.write(chunk)
-
-                parser = xml2rfc.XmlRfcParser(str(tfn), quiet=True)
-                # --- Parse the xml ---
-                try:
-                    self.xmltree = parser.parse(remove_comments=False)
-                    # If we have v2, run it through v2v3. Keep track of the submitted version, though.
-                    self.xmlroot = self.xmltree.getroot()
-                    self.xml_version = self.xmlroot.get('version', '2')
-                    if self.xml_version == '2':
-                        v2v3 = xml2rfc.V2v3XmlWriter(self.xmltree)
-                        self.xmltree.tree = v2v3.convert2to3()
-                        self.xmlroot = self.xmltree.getroot()  # update to the new root
-
-                    draftname = self.xmlroot.attrib.get('docName')
-                    if draftname is None:
-                        self.add_error('xml', "No docName attribute found in the xml root element")
-                    name_error = validate_submission_name(draftname)
-                    if name_error:
-                        self.add_error('xml', name_error) # This is a critical and immediate failure - do not proceed with other validation.
-                    else:
-                        revmatch = re.search("-[0-9][0-9]$", draftname)
-                        if revmatch:
-                            self.revision = draftname[-2:]
-                            self.filename = draftname[:-3]
-                        else:
-                            self.revision = None
-                            self.filename = draftname
-                        self.title = self.xmlroot.findtext('front/title').strip()
-                        if type(self.title) is str:
-                            self.title = unidecode(self.title)
-                        self.title = normalize_text(self.title)
-                        self.abstract = (self.xmlroot.findtext('front/abstract') or '').strip()
-                        if type(self.abstract) is str:
-                            self.abstract = unidecode(self.abstract)
-                        author_info = self.xmlroot.findall('front/author')
-                        for author in author_info:
-                            info = {
-                                "name": author.attrib.get('fullname'),
-                                "email": author.findtext('address/email'),
-                                "affiliation": author.findtext('organization'),
-                            }
-                            elem = author.find('address/postal/country')
-                            if elem != None:
-                                ascii_country = elem.get('ascii', None)
-                                info['country'] = ascii_country if ascii_country else elem.text
-
-                            for item in info:
-                                if info[item]:
-                                    info[item] = info[item].strip()
-                            self.authors.append(info)
-
-                        # --- Prep the xml ---
-                        file_name['xml'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s%s' % (self.filename, self.revision, ext))
-                        try:
-                            prep = xml2rfc.PrepToolWriter(self.xmltree, quiet=True, liberal=True, keep_pis=[xml2rfc.V3_PI_TARGET])
-                            prep.options.accept_prepped = True
-                            self.xmltree.tree = prep.prep()
-                            if self.xmltree.tree == None:
-                                self.add_error('xml', "Error from xml2rfc (prep): %s" % prep.errors)
-                        except Exception as e:
-                                msgs = format_messages('prep', e, xml2rfc.log)
-                                self.add_error('xml', msgs)
-
-                        # --- Convert to txt ---
-                        if not ('txt' in self.cleaned_data and self.cleaned_data['txt']):
-                            file_name['txt'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (self.filename, self.revision))
-                            try:
-                                writer = xml2rfc.TextWriter(self.xmltree, quiet=True)
-                                writer.options.accept_prepped = True
-                                writer.write(file_name['txt'])
-                                log.log("In %s: xml2rfc %s generated %s from %s (version %s)" %
-                                        (   os.path.dirname(file_name['xml']),
-                                            xml2rfc.__version__,
-                                            os.path.basename(file_name['txt']),
-                                            os.path.basename(file_name['xml']),
-                                            self.xml_version))
-                            except Exception as e:
-                                msgs = format_messages('txt', e, xml2rfc.log)
-                                log.log('\n'.join(msgs))
-                                self.add_error('xml', msgs)
-
-                        # --- Convert to html ---
-                        try:
-                            file_name['html'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.html' % (self.filename, self.revision))
-                            writer = xml2rfc.HtmlWriter(self.xmltree, quiet=True)
-                            writer.write(file_name['html'])
-                            self.file_types.append('.html')
-                            log.log("In %s: xml2rfc %s generated %s from %s (version %s)" %
-                                (   os.path.dirname(file_name['xml']),
-                                    xml2rfc.__version__,
-                                    os.path.basename(file_name['html']),
-                                    os.path.basename(file_name['xml']),
-                                    self.xml_version))
-                        except Exception as e:
-                            msgs = format_messages('html', e, xml2rfc.log)
-                            self.add_error('xml', msgs)
-
-                except Exception as e:
-                    try:
-                        msgs = format_messages('txt', e, xml2rfc.log)
-                        log.log('\n'.join(msgs))
-                        self.add_error('xml', msgs)
-                    except Exception:
-                        self.add_error('xml', "An exception occurred when trying to process the XML file: %s" % e)
-
-        # The following errors are likely noise if we have previous field
-        # errors:
-        if self.errors:
-            raise forms.ValidationError('')
-
-        if self.cleaned_data.get('txt'):
-            # try to parse it
-            txt_file = self.cleaned_data['txt']
-            txt_file.seek(0)
-            bytes = txt_file.read()
-            txt_file.seek(0)
-            try:
-                text = bytes.decode(PlainParser.encoding)
-                self.parsed_draft = PlaintextDraft(text, txt_file.name)
-                if self.filename == None:
-                    self.filename = self.parsed_draft.filename
-                elif self.filename != self.parsed_draft.filename:
-                    self.add_error('txt', "Inconsistent name information: xml:%s, txt:%s" % (self.filename, self.parsed_draft.filename))
-                if self.revision == None:
-                    self.revision = self.parsed_draft.revision
-                elif self.revision != self.parsed_draft.revision:
-                    self.add_error('txt', "Inconsistent revision information: xml:%s, txt:%s" % (self.revision, self.parsed_draft.revision))
-                if self.title == None:
-                    self.title = self.parsed_draft.get_title()
-                elif self.title != self.parsed_draft.get_title():
-                    self.add_error('txt', "Inconsistent title information: xml:%s, txt:%s" % (self.title, self.parsed_draft.get_title()))
-            except (UnicodeDecodeError, LookupError) as e:
-                self.add_error('txt', 'Failed decoding the uploaded file: "%s"' % str(e))
-
-        rev_error = validate_submission_rev(self.filename, self.revision)
-        if rev_error:
-            raise forms.ValidationError(rev_error)
-
-        # The following errors are likely noise if we have previous field
-        # errors:
-        if self.errors:
-            raise forms.ValidationError('')
-
-        if not self.filename:
-            raise forms.ValidationError("Could not extract a valid Internet-Draft name from the upload.  "
-                "To fix this in a text upload, please make sure that the full Internet-Draft name including "
-                "revision number appears centered on its own line below the document title on the "
-                "first page.  In an xml upload, please make sure that the top-level <rfc/> "
-                "element has a docName attribute which provides the full Internet-Draft name including "
-                "revision number.")
-
-        if not self.revision:
-            raise forms.ValidationError("Could not extract a valid Internet-Draft revision from the upload.  "
-                "To fix this in a text upload, please make sure that the full Internet-Draft name including "
-                "revision number appears centered on its own line below the document title on the "
-                "first page.  In an xml upload, please make sure that the top-level <rfc/> "
-                "element has a docName attribute which provides the full Internet-Draft name including "
-                "revision number.")
-
-        if not self.title:
-            raise forms.ValidationError("Could not extract a valid title from the upload")
-
-        if self.cleaned_data.get('txt') or self.cleaned_data.get('xml'):
-            # check group
-            self.group = self.deduce_group(self.filename)
-
-            # check existing
-            existing = Submission.objects.filter(name=self.filename, rev=self.revision).exclude(state__in=("posted", "cancel", "waiting-for-draft"))
-            if existing:
-                raise forms.ValidationError(mark_safe('A submission with same name and revision is currently being processed. <a href="%s">Check the status here.</a>' % urlreverse("ietf.submit.views.submission_status", kwargs={ 'submission_id': existing[0].pk })))
-
-            # cut-off
-            if self.revision == '00' and self.in_first_cut_off:
-                raise forms.ValidationError(mark_safe(self.cutoff_warning))
-
-            # check thresholds
-            today = date_today()
-
-            self.check_submissions_thresholds(
-                "for the Internet-Draft %s" % self.filename,
-                dict(name=self.filename, rev=self.revision, submission_date=today),
-                settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME, settings.IDSUBMIT_MAX_DAILY_SAME_DRAFT_NAME_SIZE,
-            )
-            self.check_submissions_thresholds(
-                "for the same submitter",
-                dict(remote_ip=self.remote_ip, submission_date=today),
-                settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER, settings.IDSUBMIT_MAX_DAILY_SAME_SUBMITTER_SIZE,
-            )
-            if self.group:
-                self.check_submissions_thresholds(
-                    "for the group \"%s\"" % (self.group.acronym),
-                    dict(group=self.group, submission_date=today),
-                    settings.IDSUBMIT_MAX_DAILY_SAME_GROUP, settings.IDSUBMIT_MAX_DAILY_SAME_GROUP_SIZE,
-                )
-            self.check_submissions_thresholds(
-                "across all submitters",
-                dict(submission_date=today),
-                settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS, settings.IDSUBMIT_MAX_DAILY_SUBMISSIONS_SIZE,
-            )
-
-        return super().clean()
-
-
-class DeprecatedSubmissionAutoUploadForm(DeprecatedSubmissionBaseUploadForm):
-    """Full-service upload form, replaced by the asynchronous version"""
-    user = forms.EmailField(required=True)
-
-    def __init__(self, request, *args, **kwargs):
-        super(DeprecatedSubmissionAutoUploadForm, self).__init__(request, *args, **kwargs)
-        self.formats = ['xml', ]
-        self.base_formats = ['xml', ]
-
-
 class SubmissionManualUploadForm(SubmissionBaseUploadForm):
     txt = forms.FileField(label='.txt format', required=False)
     formats = SubmissionBaseUploadForm.formats + ('txt',)
@@ -676,6 +404,7 @@ def clean_txt(self):
                 )
         return txt_file
 
+
 class SubmissionAutoUploadForm(SubmissionBaseUploadForm):
     user = forms.EmailField(required=True)
     replaces = forms.CharField(required=False, max_length=1000, strip=True)
diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index b48168f8a6..ed28c7ef02 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -44,7 +44,7 @@
 from ietf.meeting.factories import MeetingFactory
 from ietf.name.models import DraftSubmissionStateName, FormalLanguageName
 from ietf.person.models import Person
-from ietf.person.factories import UserFactory, PersonFactory, EmailFactory
+from ietf.person.factories import UserFactory, PersonFactory
 from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactory
 from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm
 from ietf.submit.models import Submission, Preapproval, SubmissionExtResource
@@ -2345,6 +2345,12 @@ def setUp(self):
         super().setUp()
         MeetingFactory(type_id='ietf', date=date_today()+datetime.timedelta(days=60))
 
+    def test_api_submit_tombstone(self):
+        """Tombstone for obsolete API endpoint should return 410 Gone"""
+        url = urlreverse("ietf.submit.views.api_submit_tombstone")
+        self.assertEqual(self.client.get(url).status_code, 410)
+        self.assertEqual(self.client.post(url).status_code, 410)
+
     def test_upload_draft(self):
         """api_submission accepts a submission and queues it for processing"""
         url = urlreverse('ietf.submit.views.api_submission')
@@ -3191,141 +3197,6 @@ def test_cancel_stale_submissions(self):
             self.assertEqual(subm.state_id, "cancel")
             self.assertEqual(subm.submissionevent_set.count(), 2)
 
-
-class ApiSubmitTests(BaseSubmitTestCase):
-    def setUp(self):
-        super().setUp()
-        # break early in case of missing configuration
-        self.assertTrue(os.path.exists(settings.IDSUBMIT_IDNITS_BINARY))
-        MeetingFactory(type_id='ietf', date=date_today()+datetime.timedelta(days=60))
-
-    def do_post_submission(self, rev, author=None, name=None, group=None, email=None, title=None, year=None):
-        url = urlreverse('ietf.submit.views.api_submit')
-        if author is None:
-            author = PersonFactory()
-        if name is None:
-            slug = re.sub('[^a-z0-9-]+', '', author.ascii_parts()[3].lower())
-            name = 'draft-%s-foo' % slug
-        if email is None:
-            email = author.user.username
-        # submit
-        data = {}
-        data['xml'], author = submission_file(f'{name}-{rev}', f'{name}-{rev}.xml', group, "test_submission.xml", author=author, email=email, title=title, year=year)
-        data['user'] = email
-        r = self.client.post(url, data)
-        return r, author, name
-
-    def test_api_submit_info(self):
-        url = urlreverse('ietf.submit.views.api_submit')
-        r = self.client.get(url)
-        expected = "A simplified Internet-Draft submission interface, intended for automation"
-        self.assertContains(r, expected, status_code=200)
-
-    def test_api_submit_bad_method(self):
-        url = urlreverse('ietf.submit.views.api_submit')
-        r = self.client.put(url)
-        self.assertEqual(r.status_code, 405)
-
-    def test_api_submit_ok(self):
-        r, author, name = self.do_post_submission('00')
-        expected = "Upload of %s OK, confirmation requests sent to:\n  %s" % (name, author.formatted_email().replace('\n',''))
-        self.assertContains(r, expected, status_code=200)
-
-    def test_api_submit_secondary_email_active(self):
-        person = PersonFactory()
-        email = EmailFactory(person=person)
-        r, author, name = self.do_post_submission('00', author=person, email=email.address)
-        for expected in [
-                "Upload of %s OK, confirmation requests sent to:" % (name, ),
-                author.formatted_email().replace('\n',''),
-            ]:
-            self.assertContains(r, expected, status_code=200)
-
-    def test_api_submit_secondary_email_inactive(self):
-        person = PersonFactory()
-        prim = person.email()
-        prim.primary = True
-        prim.save()
-        email = EmailFactory(person=person, active=False)
-        r, author, name = self.do_post_submission('00', author=person, email=email.address)
-        expected = "No such user: %s" % email.address
-        self.assertContains(r, expected, status_code=400)
-
-    def test_api_submit_no_user(self):
-        email='nonexistant.user@example.org'
-        r, author, name = self.do_post_submission('00', email=email)
-        expected = "No such user: %s" % email
-        self.assertContains(r, expected, status_code=400)
-
-    def test_api_submit_no_person(self):
-        user = UserFactory()
-        email = user.username
-        r, author, name = self.do_post_submission('00', email=email)
-        expected = "No person with username %s" % email
-        self.assertContains(r, expected, status_code=400)
-
-    def test_api_submit_wrong_revision(self):
-        r, author, name = self.do_post_submission('01')
-        expected = "Invalid revision (revision 00 is expected)"
-        self.assertContains(r, expected, status_code=400)
-
-    def test_api_submit_update_existing_submissiondocevent_rev(self):
-        draft, _ = create_draft_submission_with_rev_mismatch(rev='01')
-        r, _, __ = self.do_post_submission(rev='01', name=draft.name)
-        expected = "Submission failed"
-        self.assertContains(r, expected, status_code=409)
-
-    def test_api_submit_update_later_submissiondocevent_rev(self):
-        draft, _ = create_draft_submission_with_rev_mismatch(rev='02')
-        r, _, __ = self.do_post_submission(rev='01', name=draft.name)
-        expected = "Submission failed"
-        self.assertContains(r, expected, status_code=409)
-
-    def test_api_submit_pending_submission(self):
-        r, author, name = self.do_post_submission('00')
-        expected = "Upload of"
-        self.assertContains(r, expected, status_code=200)
-        r, author, name = self.do_post_submission('00', author=author, name=name)
-        expected = "A submission with same name and revision is currently being processed"
-        self.assertContains(r, expected, status_code=400)
-
-    def test_api_submit_no_title(self):
-        r, author, name = self.do_post_submission('00', title=" ")
-        expected = "Could not extract a valid title from the upload"
-        self.assertContains(r, expected, status_code=400)
-
-    def test_api_submit_failed_idnits(self):
-        # `year` on the next line must be leap year or this test will fail every Feb 29
-        r, author, name = self.do_post_submission('00', year="2012")
-        expected = "Document date must be within 3 days of submission date"
-        self.assertContains(r, expected, status_code=400)
-
-    def test_api_submit_keeps_extresources(self):
-        """API submit should not disturb doc external resources
-        
-        Tests that the submission inherits the existing doc's docextresource_set.
-        Relies on separate testing that Submission external_resources will be
-        handled appropriately.
-        """
-        draft = WgDraftFactory()
-
-        # add an external resource
-        self.assertEqual(draft.docextresource_set.count(), 0)
-        extres = draft.docextresource_set.create(
-            name_id='faq',
-            display_name='this is a display name',
-            value='https://example.com/faq-for-test.html',
-        )
-        
-        r, _, __ = self.do_post_submission('01', name=draft.name)
-        self.assertEqual(r.status_code, 200)
-        # draft = Document.objects.get(pk=draft.pk)  # update the draft
-        sub = Submission.objects.get(name=draft.name)
-        self.assertEqual(
-            [str(r) for r in sub.external_resources.all()],
-            [str(extres)],
-        )
-
         
 class RefsTests(BaseSubmitTestCase):
 
diff --git a/ietf/submit/views.py b/ietf/submit/views.py
index 6f23ba49d4..3f745741e4 100644
--- a/ietf/submit/views.py
+++ b/ietf/submit/views.py
@@ -1,7 +1,5 @@
 # Copyright The IETF Trust 2011-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
-
-
 import re
 import datetime
 
@@ -29,17 +27,15 @@
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.person.models import Email
 from ietf.submit.forms import (SubmissionAutoUploadForm, AuthorForm, SubmitterForm, EditSubmissionForm,
-                               PreapprovalForm, ReplacesForm,
-                               DeprecatedSubmissionAutoUploadForm, SubmissionManualUploadForm)
+                               PreapprovalForm, ReplacesForm, SubmissionManualUploadForm)
 from ietf.submit.mail import send_full_url, send_manual_post_request
 from ietf.submit.models import (Submission, Preapproval, SubmissionExtResource,
     DraftSubmissionStateName )
 from ietf.submit.tasks import process_uploaded_submission_task, process_and_accept_uploaded_submission_task, poke
 from ietf.submit.utils import ( approvable_submissions_for_user, preapprovals_for_user,
     recently_approved_by_user, validate_submission, create_submission_event, docevent_from_submission,
-    post_submission, cancel_submission, rename_submission_files, remove_submission_files, get_draft_meta,
-    get_submission, fill_in_submission, apply_checkers, save_files, clear_existing_files,
-    check_submission_revision_consistency, accept_submission, accept_submission_requires_group_approval,
+    post_submission, cancel_submission, rename_submission_files, remove_submission_files,
+    get_submission, save_files, clear_existing_files, accept_submission, accept_submission_requires_group_approval,
     accept_submission_requires_prev_auth_approval, update_submission_external_resources)
 from ietf.stats.utils import clean_country_name
 from ietf.utils.accesstoken import generate_access_token
@@ -187,97 +183,14 @@ def api_submission_status(request, submission_id):
 
 
 @csrf_exempt
-def api_submit(request):
-    "Automated submission entrypoint"
-    submission = None
-    def err(code, text):
-        return HttpResponse(text, status=code, content_type='text/plain')
-
-    if request.method == 'GET':
-        return render(request, 'submit/api_submit_info.html')
-    elif request.method == 'POST':
-        exception = None
-        try:
-            form = DeprecatedSubmissionAutoUploadForm(request, data=request.POST, files=request.FILES)
-            if form.is_valid():
-                log('got valid submission form for %s' % form.filename)
-                username = form.cleaned_data['user']
-                user = User.objects.filter(username__iexact=username)
-                if user.count() == 0:
-                    # See if a secondary login was being used
-                    email = Email.objects.filter(address=username, active=True)
-                    # The error messages don't talk about 'email', as the field we're
-                    # looking at is still the 'username' field.
-                    if email.count() == 0:
-                        return err(400, "No such user: %s" % username)
-                    elif email.count() > 1:
-                        return err(500, "Multiple matching accounts for %s" % username)
-                    email = email.first()
-                    if not hasattr(email, 'person'):
-                        return err(400, "No person matches %s" % username)
-                    person = email.person
-                    if not hasattr(person, 'user'):
-                        return err(400, "No user matches: %s" % username)
-                    user = person.user
-                elif user.count() > 1:
-                    return err(500, "Multiple matching accounts for %s" % username)
-                else:
-                    user = user.first()
-                if not hasattr(user, 'person'):
-                    return err(400, "No person with username %s" % username)
-
-                saved_files = save_files(form)
-                authors, abstract, file_name, file_size = get_draft_meta(form, saved_files)
-                for a in authors:
-                    if not a['email']:
-                        raise ValidationError("Missing email address for author %s" % a)
-
-                submission = get_submission(form)
-                fill_in_submission(form, submission, authors, abstract, file_size)
-                apply_checkers(submission, file_name)
-
-                create_submission_event(request, submission, desc="Uploaded submission via api_submit")
-
-                errors = validate_submission(submission)
-                if errors:
-                    raise ValidationError(errors)
-
-                # must do this after validate_submission() or data needed for check may be invalid
-                if check_submission_revision_consistency(submission):
-                    return err( 409, "Submission failed due to a document revision inconsistency error "
-                                     "in the database. Please contact the secretariat for assistance.")
-
-                errors = [ c.message for c in submission.checks.all() if c.passed==False ]
-                if errors:
-                    raise ValidationError(errors)
-
-                if not username.lower() in [ a['email'].lower() for a in authors ]:
-                    raise ValidationError('Submitter %s is not one of the document authors' % user.username)
-
-                submission.submitter = user.person.formatted_email()
-                sent_to = accept_submission(submission, request)
+def api_submit_tombstone(request):
+    """Tombstone for removed automated submission entrypoint"""
+    return render(
+        request, 
+        'submit/api_submit_info.html',
+        status=410,  # Gone
+    )
 
-                return HttpResponse(
-                    "Upload of %s OK, confirmation requests sent to:\n  %s" % (submission.name, ',\n  '.join(sent_to)),
-                    content_type="text/plain")
-            else:
-                raise ValidationError(form.errors)
-        except IOError as e:
-            exception = e
-            return err(500, "IO Error: %s" % str(e))
-        except ValidationError as e:
-            exception = e
-            return err(400, "Validation Error: %s" % str(e))
-        except Exception as e:
-            exception = e
-            raise
-            return err(500, "Exception: %s" % str(e))
-        finally:
-            if exception and submission:
-                remove_submission_files(submission)
-                submission.delete()
-    else:
-        return err(405, "Method not allowed")
 
 def tool_instructions(request):
     return render(request, 'submit/tool_instructions.html', {'selected': 'instructions'})
diff --git a/ietf/templates/api/index.html b/ietf/templates/api/index.html
index 8373a387f9..e21a50101a 100644
--- a/ietf/templates/api/index.html
+++ b/ietf/templates/api/index.html
@@ -9,7 +9,7 @@ <h2 id="framework">Framework API</h2>
         <p>
             This section describes the autogenerated read-only API towards the database tables.  See also
             the
-            <a href="{% url 'ietf.submit.views.api_submit' %}">Internet-Draft submission API description</a>
+            <a href="{% url 'ietf.submit.views.api_submission' %}">Internet-Draft submission API description</a>
             and the
             <a href="#iesg-position-api">IESG ballot position API description</a>
         </p>
diff --git a/ietf/templates/submit/api_submit_info.html b/ietf/templates/submit/api_submit_info.html
index cd0d52410b..75fc1abfc2 100644
--- a/ietf/templates/submit/api_submit_info.html
+++ b/ietf/templates/submit/api_submit_info.html
@@ -1,56 +1,13 @@
 {% extends "base.html" %}
-{# Copyright The IETF Trust 2015-2022, All Rights Reserved #}
+{# Copyright The IETF Trust 2015-2024, All Rights Reserved #}
 {% load origin ietf_filters %}
-{% block title %}I-D submission API instructions{% endblock %}
+{% block title %}Obsolete I-D submission API notice{% endblock %}
 {% block content %}
     {% origin %}
-    <h1 class="mb-3">Internet-Draft submission API instructions</h1>
+    <h1 class="mb-3">Obsolete Internet-Draft submission API notice</h1>
     <p>
-        Note: API endpoint described here is known to have a slow response time or to fail
-        due to timeout for some Internet-Draft submissions, particularly those with large file sizes.
-        It is recommended to use the <a href="{% url 'ietf.submit.views.api_submission' %}">new API endpoint</a>
-        instead for increased reliability.
+        The API endpoint previously available here is obsolete and is no longer supported.
+        Please use the <a href="{% url 'ietf.submit.views.api_submission' %}">new API endpoint</a>
+        instead.
     </p>
-    <p>
-        A simplified Internet-Draft submission interface, intended for automation,
-        is available at <code>{% absurl 'ietf.submit.views.api_submit' %}</code>.
-    </p>
-    <p>
-        The interface accepts only XML uploads that can be processed on the server, and
-        requires the user to have a datatracker account.  A successful submit still requires
-        the same email confirmation round-trip as submissions done through the regular
-        <a href="{% url 'ietf.submit.views.upload_submission' %}">submission tool</a>.
-    </p>
-    <p>
-        This interface does not provide all the options which the regular submission tool does.
-        Some limitations:
-    </p>
-    <ul>
-        <li>Only XML-only uploads are supported, not text or combined.</li>
-        <li>Document replacement information cannot be supplied.</li>
-        <li>
-            The server expects <code>multipart/form-data</code>, supported by <code>curl</code> but <b>not</b> by <code>wget</code>.
-        </li>
-    </ul>
-    <p>
-        It takes two parameters:
-    </p>
-    <ul>
-        <li>
-            <code>user</code> which is the user login
-        </li>
-        <li>
-            <code>xml</code>, which is the submitted file
-        </li>
-    </ul>
-    <p>
-        It returns an appropriate http result code, and a brief explanatory text message.
-    </p>
-    <p>
-        Here is an example:
-    </p>
-    <pre class="border p-3">
-$ curl -S -F "user=user.name@example.com" -F "xml=@~/draft-user-example.xml" {% absurl 'ietf.submit.views.api_submit' %}
-Upload of draft-user-example OK, confirmation requests sent to:
-User Name &lt;user.name@example.com&gt;</pre>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

From a6d15c98ba213fc37078957ae343c7f21b58cb97 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 9 Oct 2024 13:04:54 -0300
Subject: [PATCH 088/656] test: fill in milestone descriptions (#8002)

* test: fill in milestone descriptions

If "Next" or "Last" appear in the descriptions, the
selectors used by the tests get confused.

* test: avoid fragile selectors
---
 ietf/iesg/tests.py | 58 +++++++++++++++++++++++++++++++++++-----------
 1 file changed, 44 insertions(+), 14 deletions(-)

diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py
index 42e838ebdf..86910bc0ce 100644
--- a/ietf/iesg/tests.py
+++ b/ietf/iesg/tests.py
@@ -87,9 +87,18 @@ def test_milestones_needing_review_ordering(self):
             group=dated_group,
             person=Person.objects.get(user__username='ad'),
         )
-        dated_milestones = DatedGroupMilestoneFactory.create_batch(
-            2, group=dated_group, state_id="review"
-        )
+        dated_milestones = [
+            DatedGroupMilestoneFactory(
+                group=dated_group,
+                state_id="review",
+                desc="This is the description of one dated group milestone",
+            ),
+            DatedGroupMilestoneFactory(
+                group=dated_group,
+                state_id="review",
+                desc="This is the description of another dated group milestone",
+            ),
+        ]
         dated_milestones[0].due -= datetime.timedelta(days=1)  # make this one earlier
         dated_milestones[0].save()
 
@@ -99,9 +108,18 @@ def test_milestones_needing_review_ordering(self):
             group=dateless_group,
             person=Person.objects.get(user__username='ad'),
         )
-        dateless_milestones = DatelessGroupMilestoneFactory.create_batch(
-            2, group=dateless_group, state_id="review"
-        )
+        dateless_milestones = [
+            DatelessGroupMilestoneFactory(
+                group=dateless_group,
+                state_id="review",
+                desc="This is the description of one dateless group milestone",
+            ),
+            DatelessGroupMilestoneFactory(
+                group=dateless_group,
+                state_id="review",
+                desc="This is the description of another dateless group milestone",
+            ),
+        ]
 
         url = urlreverse("ietf.iesg.views.milestones_needing_review")
         self.client.login(username="ad", password="ad+password")
@@ -111,17 +129,29 @@ def test_milestones_needing_review_ordering(self):
         
         # check order-by-date
         dated_tbody = pq(f'td:contains("{dated_milestones[0].desc}")').closest("tbody")
-        next_td = dated_tbody.find('td:contains("Next")')
-        self.assertEqual(next_td.siblings()[0].text.strip(), dated_milestones[0].desc)
-        last_td = dated_tbody.find('td:contains("Last")')
-        self.assertEqual(last_td.siblings()[0].text.strip(), dated_milestones[1].desc)
+        rows = list(dated_tbody.items("tr"))  # keep as pyquery objects
+        self.assertTrue(rows[0].find('td:first:contains("Last")'))  # Last milestone shown first
+        self.assertFalse(rows[0].find('td:first:contains("Next")'))
+        self.assertTrue(rows[0].find(f'td:contains("{dated_milestones[1].desc}")'))
+        self.assertFalse(rows[0].find(f'td:contains("{dated_milestones[0].desc}")'))
+
+        self.assertFalse(rows[1].find('td:first:contains("Last")'))  # Last milestone shown first
+        self.assertTrue(rows[1].find('td:first:contains("Next")'))
+        self.assertFalse(rows[1].find(f'td:contains("{dated_milestones[1].desc}")'))
+        self.assertTrue(rows[1].find(f'td:contains("{dated_milestones[0].desc}")'))
 
         # check order-by-order
         dateless_tbody = pq(f'td:contains("{dateless_milestones[0].desc}")').closest("tbody")
-        next_td = dateless_tbody.find('td:contains("Next")')
-        self.assertEqual(next_td.siblings()[0].text.strip(), dateless_milestones[0].desc)
-        last_td = dateless_tbody.find('td:contains("Last")')
-        self.assertEqual(last_td.siblings()[0].text.strip(), dateless_milestones[1].desc)
+        rows = list(dateless_tbody.items("tr"))  # keep as pyquery objects
+        self.assertTrue(rows[0].find('td:first:contains("Last")'))  # Last milestone shown first
+        self.assertFalse(rows[0].find('td:first:contains("Next")'))
+        self.assertTrue(rows[0].find(f'td:contains("{dateless_milestones[1].desc}")'))
+        self.assertFalse(rows[0].find(f'td:contains("{dateless_milestones[0].desc}")'))
+
+        self.assertFalse(rows[1].find('td:first:contains("Last")'))  # Last milestone shown first
+        self.assertTrue(rows[1].find('td:first:contains("Next")'))
+        self.assertFalse(rows[1].find(f'td:contains("{dateless_milestones[1].desc}")'))
+        self.assertTrue(rows[1].find(f'td:contains("{dateless_milestones[0].desc}")'))
 
 
     def test_review_decisions(self):

From f7e0a67095df4f89739a38ff5d288be739d8c238 Mon Sep 17 00:00:00 2001
From: Sangho Na <sangho@nsh.nz>
Date: Thu, 10 Oct 2024 08:49:05 +1300
Subject: [PATCH 089/656] fix: turn off automatic escaping in session request
 templates (#8007)

* fix: turn off automatic escaping in session request templates

* test: ensure no unwanted escaping in session request emails

* test: use PersonFactory to create test users

* test: minor refactoring
---
 ietf/secr/sreq/tests.py                       | 56 ++++++++++++++++++-
 .../sreq/session_cancel_notification.txt      |  5 +-
 .../sreq/session_request_notification.txt     |  4 +-
 3 files changed, 59 insertions(+), 6 deletions(-)

diff --git a/ietf/secr/sreq/tests.py b/ietf/secr/sreq/tests.py
index 7fb13f1796..847b993e1c 100644
--- a/ietf/secr/sreq/tests.py
+++ b/ietf/secr/sreq/tests.py
@@ -13,9 +13,10 @@
 from ietf.meeting.models import Session, ResourceAssociation, SchedulingEvent, Constraint
 from ietf.meeting.factories import MeetingFactory, SessionFactory
 from ietf.name.models import ConstraintName, TimerangeName
+from ietf.person.factories import PersonFactory
 from ietf.person.models import Person
 from ietf.secr.sreq.forms import SessionForm
-from ietf.utils.mail import outbox, empty_outbox, get_payload_text
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text, send_mail
 from ietf.utils.timezone import date_today
 
 
@@ -78,6 +79,32 @@ def test_cancel(self):
         self.assertRedirects(r,reverse('ietf.secr.sreq.views.main'))
         self.assertEqual(SchedulingEvent.objects.filter(session=session).order_by('-id')[0].status_id, 'deleted')
 
+    def test_cancel_notification_msg(self):
+        to = "<iesg-secretary@ietf.org>"
+        subject = "Dummy subject"
+        template = "sreq/session_cancel_notification.txt"
+        meeting = MeetingFactory(type_id="ietf", date=date_today())
+        requester = PersonFactory(name="James O'Rourke", user__username="jimorourke")
+        context = {"meeting": meeting, "requester": requester}
+        cc = "cc.a@example.com, cc.b@example.com"
+        bcc = "bcc@example.com"
+
+        msg = send_mail(
+            None,
+            to,
+            None,
+            subject,
+            template,
+            context,
+            cc=cc,
+            bcc=bcc,
+        )
+        self.assertEqual(requester.name, "James O'Rourke")  # note ' (single quote) in the name
+        self.assertIn(
+            f"A request to cancel a meeting session has just been submitted by {requester.name}.",
+            get_payload_text(msg),
+        )
+
     def test_edit(self):
         meeting = MeetingFactory(type_id='ietf', date=date_today())
         mars = RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars').group
@@ -701,6 +728,33 @@ def test_request_notification(self):
         self.assertNotIn('1 Hour, 1 Hour, 1 Hour', notification_payload)
         self.assertNotIn('The third session requires your approval', notification_payload)
 
+    def test_request_notification_msg(self):
+        to = "<iesg-secretary@ietf.org>"
+        subject = "Dummy subject"
+        template = "sreq/session_request_notification.txt"
+        header = "A new"
+        meeting = MeetingFactory(type_id="ietf", date=date_today())
+        requester = PersonFactory(name="James O'Rourke", user__username="jimorourke")
+        context = {"header": header, "meeting": meeting, "requester": requester}
+        cc = "cc.a@example.com, cc.b@example.com"
+        bcc = "bcc@example.com"
+
+        msg = send_mail(
+            None,
+            to,
+            None,
+            subject,
+            template,
+            context,
+            cc=cc,
+            bcc=bcc,
+        )
+        self.assertEqual(requester.name, "James O'Rourke")  # note ' (single quote) in the name
+        self.assertIn(
+            f"{header} meeting session request has just been submitted by {requester.name}.",
+            get_payload_text(msg),
+        )
+
     def test_request_notification_third_session(self):
         meeting = MeetingFactory(type_id='ietf', date=date_today())
         ad = Person.objects.get(user__username='ad')
diff --git a/ietf/secr/templates/sreq/session_cancel_notification.txt b/ietf/secr/templates/sreq/session_cancel_notification.txt
index 3e6dd43f69..8aee6c89db 100644
--- a/ietf/secr/templates/sreq/session_cancel_notification.txt
+++ b/ietf/secr/templates/sreq/session_cancel_notification.txt
@@ -1,4 +1,3 @@
-{% load ams_filters %}
-
-A request to cancel a meeting session has just been submitted by {{ requester }}.
+{% autoescape off %}{% load ams_filters %}
 
+A request to cancel a meeting session has just been submitted by {{ requester }}.{% endautoescape %}
diff --git a/ietf/secr/templates/sreq/session_request_notification.txt b/ietf/secr/templates/sreq/session_request_notification.txt
index a41f202447..75f2cbbae4 100644
--- a/ietf/secr/templates/sreq/session_request_notification.txt
+++ b/ietf/secr/templates/sreq/session_request_notification.txt
@@ -1,5 +1,5 @@
-{% load ams_filters %}
+{% autoescape off %}{% load ams_filters %}
 
 {% filter wordwrap:78 %}{{ header }} meeting session request has just been submitted by {{ requester }}.{% endfilter %}
 
-{% include "includes/session_info.txt" %}
+{% include "includes/session_info.txt" %}{% endautoescape %}

From 148fa2f9f62d04fd11d2bac74c5ec1727a13a91b Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Oct 2024 14:23:25 -0300
Subject: [PATCH 090/656] fix: decode header fields for alert msg (#8034)

* style: Black

* chore: type hints

* refactor: % to f-string

* refactor: helper to decode header values

* fix: decode header fields for pop-up msg

* test: add tests

* fix: use truthiness check

We want to suppress empty strings, too...

* refactor: use f-string for time formatting

* test: clarify side_effect intention
---
 ietf/utils/mail.py  |  68 ++++++++++++++++-----
 ietf/utils/tests.py | 143 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 195 insertions(+), 16 deletions(-)

diff --git a/ietf/utils/mail.py b/ietf/utils/mail.py
index 4585fdb846..5417161451 100644
--- a/ietf/utils/mail.py
+++ b/ietf/utils/mail.py
@@ -19,11 +19,13 @@
 from email.header import Header, decode_header
 from email import message_from_bytes, message_from_string
 from email import charset as Charset
+from typing import Optional
 
 from django.conf import settings
 from django.contrib import messages
 from django.core.exceptions import ImproperlyConfigured, ValidationError
 from django.core.validators import validate_email
+from django.http import HttpRequest
 from django.template.loader import render_to_string
 from django.template import Context,RequestContext
 from django.utils import timezone
@@ -64,6 +66,18 @@ def add_headers(msg):
         msg['From'] = settings.DEFAULT_FROM_EMAIL
     return msg
 
+
+def decode_header_value(value: str) -> str:
+    """Decode a header value
+    
+    Easier-to-use wrapper around email.message.decode_header()
+    """
+    return "".join(
+        part.decode(charset if charset else "utf-8") if isinstance(part, bytes) else part
+        for part, charset in decode_header(value)
+    )
+
+
 class SMTPSomeRefusedRecipients(smtplib.SMTPException):
 
     def __init__(self, message, original_msg, refusals):
@@ -251,8 +265,7 @@ def parseaddr(addr):
 
     """
 
-    addr = ''.join( [ ( s.decode(m) if m else s.decode()) if isinstance(s, bytes) else s for (s,m) in decode_header(addr) ] )
-    name, addr = simple_parseaddr(addr)
+    name, addr = simple_parseaddr(decode_header_value(addr))
     return name, addr
 
 def excludeaddrs(addrlist, exlist):
@@ -330,18 +343,45 @@ def condition_message(to, frm, subject, msg, cc, extra):
         msg['Message-ID'] = make_msgid()
 
 
-def show_that_mail_was_sent(request,leadline,msg,bcc):
-        if request and request.user:
-            from ietf.ietfauth.utils import has_role
-            if has_role(request.user,['Area Director','Secretariat','IANA','RFC Editor','ISE','IAD','IRTF Chair','WG Chair','RG Chair','WG Secretary','RG Secretary']):
-                info =  "%s at %s %s\n" % (leadline,timezone.now().strftime("%Y-%m-%d %H:%M:%S"),settings.TIME_ZONE)
-                info += "Subject: %s\n" % force_str(msg.get('Subject','[no subject]'))
-                info += "To: %s\n" % msg.get('To','[no to]')
-                if msg.get('Cc'):
-                    info += "Cc: %s\n" % msg.get('Cc')
-                if bcc:
-                    info += "Bcc: %s\n" % bcc
-                messages.info(request,info,extra_tags='preformatted',fail_silently=True)
+def show_that_mail_was_sent(request: HttpRequest, leadline: str, msg: Message, bcc: Optional[str]):
+    if request and request.user:
+        from ietf.ietfauth.utils import has_role
+
+        if has_role(
+            request.user,
+            [
+                "Area Director",
+                "Secretariat",
+                "IANA",
+                "RFC Editor",
+                "ISE",
+                "IAD",
+                "IRTF Chair",
+                "WG Chair",
+                "RG Chair",
+                "WG Secretary",
+                "RG Secretary",
+            ],
+        ):
+            subject = decode_header_value(msg.get("Subject", "[no subject]"))
+            _to = decode_header_value(msg.get("To", "[no to]"))
+            info_lines = [
+                f"{leadline} at {timezone.now():%Y-%m-%d %H:%M:%S %Z}",
+                f"Subject: {subject}",
+                f"To: {_to}",
+            ]
+            cc = msg.get("Cc", None)
+            if cc:
+                info_lines.append(f"Cc: {decode_header_value(cc)}")
+            if bcc:
+                info_lines.append(f"Bcc: {decode_header_value(bcc)}")
+            messages.info(
+                request,
+                "\n".join(info_lines),
+                extra_tags="preformatted",
+                fail_silently=True,
+            )
+
 
 def save_as_message(request, msg, bcc):
     by = ((request and request.user and not request.user.is_anonymous and request.user.person)
diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py
index decdd778d9..0a1986a608 100644
--- a/ietf/utils/tests.py
+++ b/ietf/utils/tests.py
@@ -11,10 +11,11 @@
 import shutil
 import types
 
-from mock import patch
+from mock import call, patch
 from pyquery import PyQuery
 from typing import Dict, List       # pyflakes:ignore
 
+from email.message import Message
 from email.mime.image import MIMEImage
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
@@ -32,6 +33,7 @@
 from django.template.defaulttags import URLNode
 from django.template.loader import get_template, render_to_string
 from django.templatetags.static import StaticNode
+from django.test import RequestFactory
 from django.urls import reverse as urlreverse
 
 import debug                            # pyflakes:ignore
@@ -42,7 +44,15 @@
 from ietf.utils.draft import PlaintextDraft, getmeta
 from ietf.utils.fields import SearchableField
 from ietf.utils.log import unreachable, assertion
-from ietf.utils.mail import send_mail_preformatted, send_mail_text, send_mail_mime, outbox, get_payload_text
+from ietf.utils.mail import (
+    send_mail_preformatted,
+    send_mail_text,
+    send_mail_mime,
+    outbox,
+    get_payload_text,
+    decode_header_value,
+    show_that_mail_was_sent,
+)
 from ietf.utils.test_runner import get_template_paths, set_coverage_checking
 from ietf.utils.test_utils import TestCase, unicontent
 from ietf.utils.text import parse_unicode
@@ -109,6 +119,135 @@ def test_send_mail_preformatted(self):
         recv = outbox[-1]
         self.assertEqual(recv['Fuzz'], 'bucket, monger')
 
+
+class MailUtilsTests(TestCase):
+    def test_decode_header_value(self):
+        self.assertEqual(
+            decode_header_value("cake"),
+            "cake",
+            "decodes simple string value",
+        )
+        self.assertEqual(
+            decode_header_value("=?utf-8?b?8J+Ogg==?="),
+            "\U0001f382",
+            "decodes single utf-8-encoded part",
+        )
+        self.assertEqual(
+            decode_header_value("=?utf-8?b?8J+Ogg==?= = =?macintosh?b?jYxrjg==?="),
+            "\U0001f382 = çåké",
+            "decodes a value with non-utf-8 encodings",
+        )
+
+    # Patch in a side_effect so we can distinguish values that came from decode_header_value.
+    @patch("ietf.utils.mail.decode_header_value", side_effect=lambda s: f"decoded-{s}")
+    @patch("ietf.utils.mail.messages")
+    def test_show_that_mail_was_sent(self, mock_messages, mock_decode_header_value):
+        request = RequestFactory().get("/some/path")
+        request.user = object()  # just needs to exist
+        msg = Message()
+        msg["To"] = "to-value"
+        msg["Subject"] = "subject-value"
+        msg["Cc"] = "cc-value"
+        with patch("ietf.ietfauth.utils.has_role", return_value=True):
+            show_that_mail_was_sent(request, "mail was sent", msg, "bcc-value")
+        self.assertCountEqual(
+            mock_decode_header_value.call_args_list,
+            [call("to-value"), call("subject-value"), call("cc-value"), call("bcc-value")],
+        )
+        self.assertEqual(mock_messages.info.call_args[0][0], request)
+        self.assertIn("mail was sent", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-subject-value", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-to-value", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-cc-value", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-bcc-value", mock_messages.info.call_args[0][1])
+        mock_messages.reset_mock()
+        mock_decode_header_value.reset_mock()
+
+        # no bcc
+        with patch("ietf.ietfauth.utils.has_role", return_value=True):
+            show_that_mail_was_sent(request, "mail was sent", msg, None)
+        self.assertCountEqual(
+            mock_decode_header_value.call_args_list,
+            [call("to-value"), call("subject-value"), call("cc-value")],
+        )
+        self.assertEqual(mock_messages.info.call_args[0][0], request)
+        self.assertIn("mail was sent", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-subject-value", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-to-value", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-cc-value", mock_messages.info.call_args[0][1])
+        # Note: here and below - when using assertNotIn(), leaving off the "decoded-" prefix
+        # proves that neither the original value nor the decoded value appear.
+        self.assertNotIn("bcc-value", mock_messages.info.call_args[0][1])
+        mock_messages.reset_mock()
+        mock_decode_header_value.reset_mock()
+
+        # no cc
+        del msg["Cc"]
+        with patch("ietf.ietfauth.utils.has_role", return_value=True):
+            show_that_mail_was_sent(request, "mail was sent", msg, None)
+        self.assertCountEqual(
+            mock_decode_header_value.call_args_list,
+            [call("to-value"), call("subject-value")],
+        )
+        self.assertEqual(mock_messages.info.call_args[0][0], request)
+        self.assertIn("mail was sent", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-subject-value", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-to-value", mock_messages.info.call_args[0][1])
+        self.assertNotIn("cc-value", mock_messages.info.call_args[0][1])
+        self.assertNotIn("bcc-value", mock_messages.info.call_args[0][1])
+        mock_messages.reset_mock()
+        mock_decode_header_value.reset_mock()
+
+        # no to
+        del msg["To"]
+        with patch("ietf.ietfauth.utils.has_role", return_value=True):
+            show_that_mail_was_sent(request, "mail was sent", msg, None)
+        self.assertCountEqual(
+            mock_decode_header_value.call_args_list,
+            [call("[no to]"), call("subject-value")],
+        )
+        self.assertEqual(mock_messages.info.call_args[0][0], request)
+        self.assertIn("mail was sent", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-subject-value", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-[no to]", mock_messages.info.call_args[0][1])
+        self.assertNotIn("to-value", mock_messages.info.call_args[0][1])
+        self.assertNotIn("cc-value", mock_messages.info.call_args[0][1])
+        self.assertNotIn("bcc-value", mock_messages.info.call_args[0][1])
+        mock_messages.reset_mock()
+        mock_decode_header_value.reset_mock()
+
+        # no subject
+        del msg["Subject"]
+        with patch("ietf.ietfauth.utils.has_role", return_value=True):
+            show_that_mail_was_sent(request, "mail was sent", msg, None)
+        self.assertCountEqual(
+            mock_decode_header_value.call_args_list,
+            [call("[no to]"), call("[no subject]")],
+        )
+        self.assertEqual(mock_messages.info.call_args[0][0], request)
+        self.assertIn("mail was sent", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-[no subject]", mock_messages.info.call_args[0][1])
+        self.assertNotIn("subject-value", mock_messages.info.call_args[0][1])
+        self.assertIn("decoded-[no to]", mock_messages.info.call_args[0][1])
+        self.assertNotIn("to-value", mock_messages.info.call_args[0][1])
+        self.assertNotIn("cc-value", mock_messages.info.call_args[0][1])
+        self.assertNotIn("bcc-value", mock_messages.info.call_args[0][1])
+        mock_messages.reset_mock()
+        mock_decode_header_value.reset_mock()
+        
+        # user does not have role
+        with patch("ietf.ietfauth.utils.has_role", return_value=False):
+            show_that_mail_was_sent(request, "mail was sent", msg, None)
+        self.assertFalse(mock_messages.called)
+        
+        # no user
+        request.user = None
+        with patch("ietf.ietfauth.utils.has_role", return_value=True) as mock_has_role:
+            show_that_mail_was_sent(request, "mail was sent", msg, None)
+        self.assertFalse(mock_messages.called)
+        self.assertFalse(mock_has_role.called)
+
+
 class TestSMTPServer(TestCase):
 
     def test_address_rejected(self):

From 83f5fc8d1adbad3da99bb95dbbc15110be69e166 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 11 Oct 2024 12:24:35 -0500
Subject: [PATCH 091/656] fix: add exception to last<->first name swap (#8043)

---
 ietf/person/name.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/person/name.py b/ietf/person/name.py
index dc57f58f4b..0dbeaa9b99 100644
--- a/ietf/person/name.py
+++ b/ietf/person/name.py
@@ -59,7 +59,7 @@ def name_parts(name):
         last = parts[0]
     if len(parts) >= 2:
         # Handle reverse-order names with uppercase surname correctly
-        if len(first)>1 and re.search("^[A-Z-]+$", first):
+        if len(first)>1 and re.search("^[A-Z-]+$", first) and first != "JP":
             first, last = last, first.capitalize()
     # Handle exception for RFC Editor
     if (prefix, first, middle, last, suffix) == ('', 'Editor', '', 'Rfc', ''):

From 2d2e879bd15e2cfb71dbaf9c05a29edf01883343 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 14 Oct 2024 12:22:19 -0300
Subject: [PATCH 092/656] fix: versioned href for slides URL (#8040)

---
 ietf/meeting/views.py        |  2 +-
 ietf/utils/meetecho.py       | 14 ++++++--------
 ietf/utils/tests_meetecho.py |  8 ++++----
 3 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 211cdec9a5..6e6c4dfe23 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -1840,7 +1840,7 @@ def agenda_extract_slide(item):
         "id": item.id,
         "title": item.title,
         "rev": item.rev,
-        "url": item.get_versionless_href(),
+        "url": item.get_href(),
         "ext": item.file_extension(),
     }
 
diff --git a/ietf/utils/meetecho.py b/ietf/utils/meetecho.py
index 2f5f146766..0dbf75736a 100644
--- a/ietf/utils/meetecho.py
+++ b/ietf/utils/meetecho.py
@@ -481,12 +481,10 @@ def delete_conference(self, conf: Conference):
 class SlidesManager(Manager):
     """Interface between Datatracker models and Meetecho API
     
-    Note: The URL we send comes from get_versionless_href(). This should match what we use as the
-    URL in api_get_session_materials(). Additionally, it _must_ give the right result for a Document
-    instance that has not yet been persisted to the database. This is because upload_session_slides()
-    (as of 2024-03-07) SessionPresentations before saving its updated Documents. This means, for
-    example, using get_absolute_url() will cause bugs. (We should refactor upload_session_slides() to
-    avoid this requirement.) 
+    Note: the URL sent for a slide deck comes from DocumentInfo.get_href() and includes the revision
+    of the slides being sent. Be sure that 1) the URL matches what api_get_session_materials() returns
+    for the slides; and 2) the URL is valid if it is fetched immediately - possibly even before the call
+    to SlidesManager.add() or send_update() returns.
     """
 
     def __init__(self, api_config):
@@ -521,7 +519,7 @@ def add(self, session: "Session", slides: "Document", order: int):
             deck={
                 "id": slides.pk,
                 "title": slides.title,
-                "url": slides.get_versionless_href(),  # see above note re: get_versionless_href()
+                "url": slides.get_href(),
                 "rev": slides.rev,
                 "order": order,
             }
@@ -575,7 +573,7 @@ def send_update(self, session: "Session"):
                 {
                     "id": deck.document.pk,
                     "title": deck.document.title,
-                    "url": deck.document.get_versionless_href(),  # see note above re: get_versionless_href()
+                    "url": deck.document.get_href(),
                     "rev": deck.document.rev,
                     "order": deck.order,
                 }
diff --git a/ietf/utils/tests_meetecho.py b/ietf/utils/tests_meetecho.py
index 1aef5894e2..a10ac68c27 100644
--- a/ietf/utils/tests_meetecho.py
+++ b/ietf/utils/tests_meetecho.py
@@ -558,7 +558,7 @@ def test_add(self, mock_add, mock_wg_token):
                 deck={
                     "id": slides_doc.pk,
                     "title": slides_doc.title,
-                    "url": slides_doc.get_versionless_href(),
+                    "url": slides_doc.get_href(session.meeting),
                     "rev": slides_doc.rev,
                     "order": 13,
                 },
@@ -597,7 +597,7 @@ def test_delete(self, mock_delete, mock_update, mock_wg_token):
                     {
                         "id": slides_doc.pk,
                         "title": slides_doc.title,
-                        "url": slides_doc.get_versionless_href(),
+                        "url": slides_doc.get_href(session.meeting),
                         "rev": slides_doc.rev,
                         "order": 1,
                     },
@@ -635,7 +635,7 @@ def test_revise(self, mock_add, mock_delete, mock_wg_token):
                 deck={
                     "id": slides_doc.pk,
                     "title": slides_doc.title,
-                    "url": slides_doc.get_versionless_href(),
+                    "url": slides_doc.get_href(slides.session.meeting),
                     "rev": slides_doc.rev,
                     "order": 23,
                 },
@@ -660,7 +660,7 @@ def test_send_update(self, mock_send_update, mock_wg_token):
                     {
                         "id": slides.document_id,
                         "title": slides.document.title,
-                        "url": slides.document.get_versionless_href(),
+                        "url": slides.document.get_href(slides.session.meeting),
                         "rev": slides.document.rev,
                         "order": 0,
                     }

From 166340842ed3e4927cd8dd174aecdce8a4df50aa Mon Sep 17 00:00:00 2001
From: Martin Thomson <mt@lowentropy.net>
Date: Tue, 15 Oct 2024 07:24:49 +1100
Subject: [PATCH 093/656] fix: Set overflow-y to clip for pre (#8047)

Browsers treat an overflow: auto directive as a request to make an
element scrollable, even if the element does not need it.
This results in scrolling being confined to that element sometimes,
such as when a document is scrolled using a mouse wheel
if the document scrolls the pointer into that element.
The result is that scrolling can be "trapped" on an element that
doesn't really need to scroll.

This disables vertical scrolling for those pre elements that might
trap scrolling in this way.
---
 ietf/static/css/document_html_txt.scss | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/static/css/document_html_txt.scss b/ietf/static/css/document_html_txt.scss
index b0fec7c4d6..a5991056c9 100644
--- a/ietf/static/css/document_html_txt.scss
+++ b/ietf/static/css/document_html_txt.scss
@@ -344,7 +344,7 @@ div:is(.artwork, .sourcecode) pre {
   flex: 0 0 content;
   margin: 0;
   max-width: 72ch;
-  overflow: auto;
+  overflow: auto clip;
 }
 div:is(.artwork, .sourcecode) .pilcrow {
   flex: 0 0 1ch;

From 68ceea9c060fa0f13bef2b9efe225b41bc6ff519 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Wed, 16 Oct 2024 08:38:02 +1300
Subject: [PATCH 094/656] chore: Bypass Firefox for Playwright test due to
 flakiness (#8050)

---
 playwright/tests/status/status.spec.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/playwright/tests/status/status.spec.js b/playwright/tests/status/status.spec.js
index 7b3b90bfa4..daef7a88f1 100644
--- a/playwright/tests/status/status.spec.js
+++ b/playwright/tests/status/status.spec.js
@@ -20,6 +20,10 @@ test.describe('site status', () => {
     by: 'Exile is a cool Amiga game'
   }
 
+  test.beforeEach(({ browserName }) => {
+    test.skip(browserName === 'firefox', 'bypassing flaky tests on Firefox')
+  })
+
   test('Renders server status as Notification', async ({ page }) => {
     await page.route('/status/latest.json', route => {
       route.fulfill({

From 34363040cd47d2f7ee407b56f84a44bf256c2af2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 15 Oct 2024 15:58:14 -0400
Subject: [PATCH 095/656] build(deps): bump appleboy/ssh-action from 1.0.3 to
 1.1.0 (#8019)

Bumps [appleboy/ssh-action](https://github.com/appleboy/ssh-action) from 1.0.3 to 1.1.0.
- [Release notes](https://github.com/appleboy/ssh-action/releases)
- [Changelog](https://github.com/appleboy/ssh-action/blob/master/.goreleaser.yaml)
- [Commits](https://github.com/appleboy/ssh-action/compare/029f5b4aeeeb58fdfe1410a5d17f967dacf36262...25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1)

---
updated-dependencies:
- dependency-name: appleboy/ssh-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/tests-az.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml
index bfd1fdd584..3f828430a9 100644
--- a/.github/workflows/tests-az.yml
+++ b/.github/workflows/tests-az.yml
@@ -38,7 +38,7 @@ jobs:
         ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts
 
     - name: Remote SSH into VM
-      uses: appleboy/ssh-action@029f5b4aeeeb58fdfe1410a5d17f967dacf36262
+      uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:

From c37fec2f1c665cae18d0e96418d6617e0722521b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 15 Oct 2024 15:58:52 -0400
Subject: [PATCH 096/656] build(deps): bump codecov/codecov-action from 4.5.0
 to 4.6.0 (#8020)

Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4.5.0 to 4.6.0.
- [Release notes](https://github.com/codecov/codecov-action/releases)
- [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/codecov/codecov-action/compare/v4.5.0...v4.6.0)

---
updated-dependencies:
- dependency-name: codecov/codecov-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 616ffdcc63..64db03db09 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -59,7 +59,7 @@ jobs:
         path: geckodriver.log
 
     - name: Upload Coverage Results to Codecov
-      uses: codecov/codecov-action@v4.5.0
+      uses: codecov/codecov-action@v4.6.0
       with:
         files: coverage.xml
         

From 39ba425089a2ec9b24b5c8ac8b07a57ded62f911 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 15 Oct 2024 15:59:22 -0400
Subject: [PATCH 097/656] build(deps): bump the npm group across 1 directory
 with 2 updates (#8021)

Bumps the npm group with 2 updates in the /dev/coverage-action directory: [@actions/core](https://github.com/actions/toolkit/tree/HEAD/packages/core) and [luxon](https://github.com/moment/luxon).


Updates `@actions/core` from 1.10.1 to 1.11.1
- [Changelog](https://github.com/actions/toolkit/blob/main/packages/core/RELEASES.md)
- [Commits](https://github.com/actions/toolkit/commits/HEAD/packages/core)

Updates `luxon` from 3.4.4 to 3.5.0
- [Changelog](https://github.com/moment/luxon/blob/master/CHANGELOG.md)
- [Commits](https://github.com/moment/luxon/compare/3.4.4...3.5.0)

---
updated-dependencies:
- dependency-name: "@actions/core"
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm
- dependency-name: luxon
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/coverage-action/package-lock.json | 75 ++++++++++++++++-----------
 dev/coverage-action/package.json      |  4 +-
 2 files changed, 46 insertions(+), 33 deletions(-)

diff --git a/dev/coverage-action/package-lock.json b/dev/coverage-action/package-lock.json
index 8c9b97e026..f23a9b66c7 100644
--- a/dev/coverage-action/package-lock.json
+++ b/dev/coverage-action/package-lock.json
@@ -9,19 +9,27 @@
       "version": "1.0.0",
       "license": "BSD-3-Clause",
       "dependencies": {
-        "@actions/core": "1.10.1",
+        "@actions/core": "1.11.1",
         "@actions/github": "6.0.0",
         "lodash": "4.17.21",
-        "luxon": "3.4.4"
+        "luxon": "3.5.0"
       }
     },
     "node_modules/@actions/core": {
-      "version": "1.10.1",
-      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.1.tgz",
-      "integrity": "sha512-3lBR9EDAY+iYIpTnTIXmWcNbX3T2kCkAEQGIQx4NVQ0575nk2k3GRZDTPQG+vVtS2izSLmINlxXf0uLtnrTP+g==",
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.11.1.tgz",
+      "integrity": "sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==",
       "dependencies": {
-        "@actions/http-client": "^2.0.1",
-        "uuid": "^8.3.2"
+        "@actions/exec": "^1.1.1",
+        "@actions/http-client": "^2.0.1"
+      }
+    },
+    "node_modules/@actions/exec": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz",
+      "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==",
+      "dependencies": {
+        "@actions/io": "^1.0.1"
       }
     },
     "node_modules/@actions/github": {
@@ -44,6 +52,11 @@
         "undici": "^5.25.4"
       }
     },
+    "node_modules/@actions/io": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz",
+      "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q=="
+    },
     "node_modules/@fastify/busboy": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.0.0.tgz",
@@ -196,9 +209,9 @@
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
     },
     "node_modules/luxon": {
-      "version": "3.4.4",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz",
-      "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==",
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
+      "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==",
       "engines": {
         "node": ">=12"
       }
@@ -235,14 +248,6 @@
       "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz",
       "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w=="
     },
-    "node_modules/uuid": {
-      "version": "8.3.2",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
-      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
-      "bin": {
-        "uuid": "dist/bin/uuid"
-      }
-    },
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
@@ -251,12 +256,20 @@
   },
   "dependencies": {
     "@actions/core": {
-      "version": "1.10.1",
-      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.1.tgz",
-      "integrity": "sha512-3lBR9EDAY+iYIpTnTIXmWcNbX3T2kCkAEQGIQx4NVQ0575nk2k3GRZDTPQG+vVtS2izSLmINlxXf0uLtnrTP+g==",
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.11.1.tgz",
+      "integrity": "sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==",
       "requires": {
-        "@actions/http-client": "^2.0.1",
-        "uuid": "^8.3.2"
+        "@actions/exec": "^1.1.1",
+        "@actions/http-client": "^2.0.1"
+      }
+    },
+    "@actions/exec": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz",
+      "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==",
+      "requires": {
+        "@actions/io": "^1.0.1"
       }
     },
     "@actions/github": {
@@ -279,6 +292,11 @@
         "undici": "^5.25.4"
       }
     },
+    "@actions/io": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz",
+      "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q=="
+    },
     "@fastify/busboy": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.0.0.tgz",
@@ -395,9 +413,9 @@
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
     },
     "luxon": {
-      "version": "3.4.4",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz",
-      "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA=="
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
+      "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ=="
     },
     "once": {
       "version": "1.4.0",
@@ -425,11 +443,6 @@
       "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz",
       "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w=="
     },
-    "uuid": {
-      "version": "8.3.2",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
-      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
-    },
     "wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
diff --git a/dev/coverage-action/package.json b/dev/coverage-action/package.json
index 0a8794d2a2..1d03fe3dab 100644
--- a/dev/coverage-action/package.json
+++ b/dev/coverage-action/package.json
@@ -6,9 +6,9 @@
   "author": "IETF Trust",
   "license": "BSD-3-Clause",
   "dependencies": {
-    "@actions/core": "1.10.1",
+    "@actions/core": "1.11.1",
     "@actions/github": "6.0.0",
     "lodash": "4.17.21",
-    "luxon": "3.4.4"
+    "luxon": "3.5.0"
   }
 }

From b1b14024e3a3291c334b867f8a5e5cd8842f0fec Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 15 Oct 2024 16:00:38 -0400
Subject: [PATCH 098/656] chore(deps): bump slackapi/slack-github-action from
 1.26.0 to 1.27.0 (#7885)

Bumps [slackapi/slack-github-action](https://github.com/slackapi/slack-github-action) from 1.26.0 to 1.27.0.
- [Release notes](https://github.com/slackapi/slack-github-action/releases)
- [Commits](https://github.com/slackapi/slack-github-action/compare/v1.26.0...v1.27.0)

---
updated-dependencies:
- dependency-name: slackapi/slack-github-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8f88c7c22e..3887661cac 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -335,7 +335,7 @@ jobs:
     steps:      
     - name: Notify on Slack (Success)
       if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
-      uses: slackapi/slack-github-action@v1.26.0
+      uses: slackapi/slack-github-action@v1.27.0
       with:
         channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
         payload: |
@@ -358,7 +358,7 @@ jobs:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}
     - name: Notify on Slack (Failure)
       if: ${{ contains(join(needs.*.result, ','), 'failure') }}
-      uses: slackapi/slack-github-action@v1.26.0
+      uses: slackapi/slack-github-action@v1.27.0
       with:
         channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
         payload: |

From 51f96eb952160997292af32998467e9f90c40782 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 15 Oct 2024 16:01:15 -0400
Subject: [PATCH 099/656] chore(deps): bump tar in /dev/deploy-to-container in
 the npm group (#7793)

Bumps the npm group in /dev/deploy-to-container with 1 update: [tar](https://github.com/isaacs/node-tar).


Updates `tar` from 7.4.0 to 7.4.3
- [Release notes](https://github.com/isaacs/node-tar/releases)
- [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md)
- [Commits](https://github.com/isaacs/node-tar/compare/v7.4.0...v7.4.3)

---
updated-dependencies:
- dependency-name: tar
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/deploy-to-container/package-lock.json | 14 +++++++-------
 dev/deploy-to-container/package.json      |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json
index 7b86986d56..f636a45f05 100644
--- a/dev/deploy-to-container/package-lock.json
+++ b/dev/deploy-to-container/package-lock.json
@@ -11,7 +11,7 @@
         "nanoid": "5.0.7",
         "nanoid-dictionary": "5.0.0-beta.1",
         "slugify": "1.6.6",
-        "tar": "^7.4.0",
+        "tar": "^7.4.3",
         "yargs": "^17.7.2"
       },
       "engines": {
@@ -788,9 +788,9 @@
       }
     },
     "node_modules/tar": {
-      "version": "7.4.0",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.0.tgz",
-      "integrity": "sha512-XQs0S8fuAkQWuqhDeCdMlJXDX80D7EOVLDPVFkna9yQfzS+PHKgfxcei0jf6/+QAWcjqrnC8uM3fSAnrQl+XYg==",
+      "version": "7.4.3",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz",
+      "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==",
       "dependencies": {
         "@isaacs/fs-minipass": "^4.0.0",
         "chownr": "^3.0.0",
@@ -1503,9 +1503,9 @@
       }
     },
     "tar": {
-      "version": "7.4.0",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.0.tgz",
-      "integrity": "sha512-XQs0S8fuAkQWuqhDeCdMlJXDX80D7EOVLDPVFkna9yQfzS+PHKgfxcei0jf6/+QAWcjqrnC8uM3fSAnrQl+XYg==",
+      "version": "7.4.3",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz",
+      "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==",
       "requires": {
         "@isaacs/fs-minipass": "^4.0.0",
         "chownr": "^3.0.0",
diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json
index 2f582dd2a7..be77fa5cce 100644
--- a/dev/deploy-to-container/package.json
+++ b/dev/deploy-to-container/package.json
@@ -7,7 +7,7 @@
     "nanoid": "5.0.7",
     "nanoid-dictionary": "5.0.0-beta.1",
     "slugify": "1.6.6",
-    "tar": "^7.4.0",
+    "tar": "^7.4.3",
     "yargs": "^17.7.2"
   },
   "engines": {

From db4ff66fbd3950c280ac9334a8718d6de4f10a5b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 15 Oct 2024 16:02:08 -0400
Subject: [PATCH 100/656] chore(deps): bump the npm group across 1 directory
 with 2 updates (#7789)

Bumps the npm group with 2 updates in the /dev/diff directory: [luxon](https://github.com/moment/luxon) and [tar](https://github.com/isaacs/node-tar).


Updates `luxon` from 3.4.4 to 3.5.0
- [Changelog](https://github.com/moment/luxon/blob/master/CHANGELOG.md)
- [Commits](https://github.com/moment/luxon/compare/3.4.4...3.5.0)

Updates `tar` from 7.4.0 to 7.4.3
- [Release notes](https://github.com/isaacs/node-tar/releases)
- [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md)
- [Commits](https://github.com/isaacs/node-tar/compare/v7.4.0...v7.4.3)

---
updated-dependencies:
- dependency-name: luxon
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm
- dependency-name: tar
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/diff/package-lock.json | 28 ++++++++++++++--------------
 dev/diff/package.json      |  4 ++--
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/dev/diff/package-lock.json b/dev/diff/package-lock.json
index 3500ccec48..87aaf3f16f 100644
--- a/dev/diff/package-lock.json
+++ b/dev/diff/package-lock.json
@@ -15,9 +15,9 @@
         "keypress": "^0.2.1",
         "listr2": "^6.6.1",
         "lodash-es": "^4.17.21",
-        "luxon": "^3.4.4",
+        "luxon": "^3.5.0",
         "pretty-bytes": "^6.1.1",
-        "tar": "^7.4.0",
+        "tar": "^7.4.3",
         "yargs": "^17.7.2"
       },
       "engines": {
@@ -1060,9 +1060,9 @@
       }
     },
     "node_modules/luxon": {
-      "version": "3.4.4",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz",
-      "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==",
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
+      "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==",
       "engines": {
         "node": ">=12"
       }
@@ -1493,9 +1493,9 @@
       }
     },
     "node_modules/tar": {
-      "version": "7.4.0",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.0.tgz",
-      "integrity": "sha512-XQs0S8fuAkQWuqhDeCdMlJXDX80D7EOVLDPVFkna9yQfzS+PHKgfxcei0jf6/+QAWcjqrnC8uM3fSAnrQl+XYg==",
+      "version": "7.4.3",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz",
+      "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==",
       "dependencies": {
         "@isaacs/fs-minipass": "^4.0.0",
         "chownr": "^3.0.0",
@@ -2410,9 +2410,9 @@
       "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ=="
     },
     "luxon": {
-      "version": "3.4.4",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz",
-      "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA=="
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
+      "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ=="
     },
     "mimic-fn": {
       "version": "2.1.0",
@@ -2691,9 +2691,9 @@
       }
     },
     "tar": {
-      "version": "7.4.0",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.0.tgz",
-      "integrity": "sha512-XQs0S8fuAkQWuqhDeCdMlJXDX80D7EOVLDPVFkna9yQfzS+PHKgfxcei0jf6/+QAWcjqrnC8uM3fSAnrQl+XYg==",
+      "version": "7.4.3",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz",
+      "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==",
       "requires": {
         "@isaacs/fs-minipass": "^4.0.0",
         "chownr": "^3.0.0",
diff --git a/dev/diff/package.json b/dev/diff/package.json
index 1b5540e346..4e0e1be8f4 100644
--- a/dev/diff/package.json
+++ b/dev/diff/package.json
@@ -11,9 +11,9 @@
     "keypress": "^0.2.1",
     "listr2": "^6.6.1",
     "lodash-es": "^4.17.21",
-    "luxon": "^3.4.4",
+    "luxon": "^3.5.0",
     "pretty-bytes": "^6.1.1",
-    "tar": "^7.4.0",
+    "tar": "^7.4.3",
     "yargs": "^17.7.2"
   },
   "engines": {

From a338df16f23322b43fc57c5379d0807e61e12984 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 16 Oct 2024 13:36:54 -0300
Subject: [PATCH 101/656] fix: 404 instead of 300 for ambiguous email_or_person
 (#8004)

* fix: 404 on CommunityList name collision

* fix: 404 on ambiuous person for photo() view

* test: update tests

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/community/tests.py |  4 +--
 ietf/community/views.py | 54 ++++++++++++-----------------------------
 ietf/person/tests.py    |  4 +--
 ietf/person/views.py    |  2 +-
 4 files changed, 19 insertions(+), 45 deletions(-)

diff --git a/ietf/community/tests.py b/ietf/community/tests.py
index 181e9e0fa6..5329267d8e 100644
--- a/ietf/community/tests.py
+++ b/ietf/community/tests.py
@@ -95,9 +95,7 @@ def test_view_list_duplicates(self):
 
         url = urlreverse(ietf.community.views.view_list, kwargs={ "email_or_name": person.plain_name()})
         r = self.client.get(url)
-        self.assertEqual(r.status_code, 300)
-        self.assertIn("bazquux@example.com", r.content.decode())
-        self.assertIn("foobar@example.com", r.content.decode())
+        self.assertEqual(r.status_code, 404)
 
     def complex_person(self, *args, **kwargs):
         person = PersonFactory(*args, **kwargs)
diff --git a/ietf/community/views.py b/ietf/community/views.py
index 78b8144d60..923ec556f3 100644
--- a/ietf/community/views.py
+++ b/ietf/community/views.py
@@ -28,11 +28,14 @@
 from ietf.utils.http import is_ajax
 from ietf.utils.response import permission_denied
 
-class MultiplePersonError(Exception):
-    """More than one Person record matches the given email or name"""
-    pass
 
 def lookup_community_list(request, email_or_name=None, acronym=None):
+    """Finds a CommunityList for a person or group
+    
+    Instantiates an unsaved CommunityList if one is not found.
+    
+    If the person or group cannot be found and uniquely identified, raises an Http404 exception
+    """
     assert email_or_name or acronym
 
     if acronym:
@@ -44,19 +47,14 @@ def lookup_community_list(request, email_or_name=None, acronym=None):
             if hasattr(request.user, 'person') and request.user.person in persons:
                 person = request.user.person
             else:
-                raise MultiplePersonError("\r\n".join([p.user.username for p in persons]))
+                raise Http404(f"Unable to identify the CommunityList for {email_or_name}")
         else:
             person = persons[0]
         clist = CommunityList.objects.filter(person=person).first() or CommunityList(person=person)
-
     return clist
 
 def view_list(request, email_or_name=None):
-    try:
-        clist = lookup_community_list(request, email_or_name)
-    except MultiplePersonError as err:
-        return HttpResponse(str(err), status=300)
-
+    clist = lookup_community_list(request, email_or_name)  # may raise Http404
     docs = docs_tracked_by_community_list(clist)
     docs, meta = prepare_document_table(request, docs, request.GET)
 
@@ -76,10 +74,7 @@ def view_list(request, email_or_name=None):
 def manage_list(request, email_or_name=None, acronym=None):
     # we need to be a bit careful because clist may not exist in the
     # database so we can't call related stuff on it yet
-    try:
-        clist = lookup_community_list(request, email_or_name, acronym)
-    except MultiplePersonError as err:
-        return HttpResponse(str(err), status=300)
+    clist = lookup_community_list(request, email_or_name, acronym)  # may raise Http404
 
     if not can_manage_community_list(request.user, clist):
         permission_denied(request, "You do not have permission to access this view")
@@ -166,10 +161,7 @@ def track_document(request, name, email_or_name=None, acronym=None):
     doc = get_object_or_404(Document, name=name)
 
     if request.method == "POST":
-        try:
-            clist = lookup_community_list(request, email_or_name, acronym)
-        except MultiplePersonError as err:
-            return HttpResponse(str(err), status=300)
+        clist = lookup_community_list(request, email_or_name, acronym)  # may raise Http404
         if not can_manage_community_list(request.user, clist):
             permission_denied(request, "You do not have permission to access this view")
 
@@ -191,10 +183,7 @@ def track_document(request, name, email_or_name=None, acronym=None):
 @login_required
 def untrack_document(request, name, email_or_name=None, acronym=None):
     doc = get_object_or_404(Document, name=name)
-    try:
-        clist = lookup_community_list(request, email_or_name, acronym)
-    except MultiplePersonError as err:
-        return HttpResponse(str(err), status=300)
+    clist = lookup_community_list(request, email_or_name, acronym)  # may raise Http404
     if not can_manage_community_list(request.user, clist):
         permission_denied(request, "You do not have permission to access this view")
 
@@ -214,11 +203,7 @@ def untrack_document(request, name, email_or_name=None, acronym=None):
 
 @ignore_view_kwargs("group_type")
 def export_to_csv(request, email_or_name=None, acronym=None):
-    try:
-        clist = lookup_community_list(request, email_or_name, acronym)
-    except MultiplePersonError as err:
-        return HttpResponse(str(err), status=300)
-
+    clist = lookup_community_list(request, email_or_name, acronym)  # may raise Http404
     response = HttpResponse(content_type='text/csv')
 
     if clist.group:
@@ -259,11 +244,7 @@ def export_to_csv(request, email_or_name=None, acronym=None):
 
 @ignore_view_kwargs("group_type")
 def feed(request, email_or_name=None, acronym=None):
-    try:
-        clist = lookup_community_list(request, email_or_name, acronym)
-    except MultiplePersonError as err:
-        return HttpResponse(str(err), status=300)
-
+    clist = lookup_community_list(request, email_or_name, acronym)  # may raise Http404
     significant = request.GET.get('significant', '') == '1'
 
     documents = docs_tracked_by_community_list(clist).values_list('pk', flat=True)
@@ -299,12 +280,9 @@ def feed(request, email_or_name=None, acronym=None):
 @login_required
 @ignore_view_kwargs("group_type")
 def subscription(request, email_or_name=None, acronym=None):
-    try:
-        clist = lookup_community_list(request, email_or_name, acronym)
-        if clist.pk is None:
-            raise Http404
-    except MultiplePersonError as err:
-        return HttpResponse(str(err), status=300)
+    clist = lookup_community_list(request, email_or_name, acronym)  # may raise Http404
+    if clist.pk is None:
+        raise Http404
 
     person = request.user.person
 
diff --git a/ietf/person/tests.py b/ietf/person/tests.py
index 9da201b707..61d9b0ed70 100644
--- a/ietf/person/tests.py
+++ b/ietf/person/tests.py
@@ -173,9 +173,7 @@ def test_person_photo_duplicates(self):
 
         url = urlreverse("ietf.person.views.photo", kwargs={ "email_or_name": person.plain_name()})
         r = self.client.get(url)
-        self.assertEqual(r.status_code, 300)
-        self.assertIn("bazquux@example.com", r.content.decode())
-        self.assertIn("foobar@example.com", r.content.decode())
+        self.assertEqual(r.status_code, 404)
 
     def test_name_methods(self):
         person = PersonFactory(name="Dr. Jens F. Möller", )
diff --git a/ietf/person/views.py b/ietf/person/views.py
index 6d9daf4a81..bb1fa79f84 100644
--- a/ietf/person/views.py
+++ b/ietf/person/views.py
@@ -76,7 +76,7 @@ def profile(request, email_or_name):
 def photo(request, email_or_name):
     persons = lookup_persons(email_or_name)
     if len(persons) > 1:
-        return HttpResponse(r"\r\n".join([p.user.username for p in persons]), status=300)
+        raise Http404("No photo found")
     person = persons[0]
     if not person.photo:
         raise Http404("No photo found")

From de2e66ede3396e40baf6e20caf3f85736c9d50aa Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 17 Oct 2024 18:23:13 -0300
Subject: [PATCH 102/656] feat: report xml2rfc errors for submissions (#8000)

* feat: capture xml2rfc output on exception

* fix: chain exception properly

* feat: log xml2rfc stderr/stdout

May want to pass these back to the submitter, but
let's watch them to see what sorts of sanitizing we
should do first.

* Revert "feat: log xml2rfc stderr/stdout"

This reverts commit 959c54d30733a4a3df7ed0250fef347ed6f03a08.

* feat: log xml2rfc stderr/stdout

This time without reformatting all the imports

* feat: path sanitization for submission errors

* fix: parentheses in regex

* test: test SubmissionError sanitization

* style: consolidate imports

* fix: apply sanitization to all args

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/submit/tests.py |  38 +++++++++++--
 ietf/submit/utils.py | 128 +++++++++++++++++++++++++++++++++++++------
 2 files changed, 144 insertions(+), 22 deletions(-)

diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index ed28c7ef02..21ed6672df 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -27,11 +27,6 @@
 from django.utils.encoding import force_str
 import debug                            # pyflakes:ignore
 
-from ietf.submit.utils import (expirable_submissions, expire_submission, find_submission_filenames,
-                               post_submission, validate_submission_name, validate_submission_rev,
-                               process_and_accept_uploaded_submission, SubmissionError, process_submission_text,
-                               process_submission_xml, process_uploaded_submission, 
-                               process_and_validate_submission)
 from ietf.doc.factories import (DocumentFactory, WgDraftFactory, IndividualDraftFactory,
                                 ReviewFactory, WgRfcFactory)
 from ietf.doc.models import ( Document, DocEvent, State,
@@ -49,7 +44,12 @@
 from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm
 from ietf.submit.models import Submission, Preapproval, SubmissionExtResource
 from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task
-from ietf.submit.utils import apply_yang_checker_to_draft, run_all_yang_model_checks
+from ietf.submit.utils import (expirable_submissions, expire_submission, find_submission_filenames,
+                               post_submission, validate_submission_name, validate_submission_rev,
+                               process_and_accept_uploaded_submission, SubmissionError, process_submission_text,
+                               process_submission_xml, process_uploaded_submission, 
+                               process_and_validate_submission, apply_yang_checker_to_draft, 
+                               run_all_yang_model_checks)
 from ietf.utils import tool_version
 from ietf.utils.accesstoken import generate_access_token
 from ietf.utils.mail import outbox, get_payload_text
@@ -3384,3 +3384,29 @@ def test_apply_yang_checker_to_draft(self):
         apply_yang_checker_to_draft(checker, draft)
         self.assertEqual(checker.check_file_txt.call_args, mock.call(draft.get_file_name()))
 
+
+@override_settings(IDSUBMIT_REPOSITORY_PATH="/some/path/", IDSUBMIT_STAGING_PATH="/some/other/path")
+class SubmissionErrorTests(TestCase):
+    def test_sanitize_message(self):
+        sanitized = SubmissionError.sanitize_message(
+            "This refers to /some/path/with-a-file\n"
+            "and also /some/other/path/with-a-different-file isn't that neat?\n"
+            "and has /some/path//////with-slashes"
+        )
+        self.assertEqual(
+            sanitized,
+            "This refers to **/with-a-file\n"
+            "and also **/with-a-different-file isn't that neat?\n"
+            "and has **/with-slashes"
+        )
+    
+    @mock.patch.object(SubmissionError, "sanitize_message")
+    def test_submissionerror(self, mock_sanitize_message):
+        SubmissionError()
+        self.assertFalse(mock_sanitize_message.called)
+        SubmissionError("hi", "there")
+        self.assertTrue(mock_sanitize_message.called)
+        self.assertCountEqual(
+            mock_sanitize_message.call_args_list,
+            [mock.call("hi"), mock.call("there")],
+        )
diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index c814b84657..e6cbcb12f7 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -17,6 +17,7 @@
 from shutil import move
 from typing import Optional, Union  # pyflakes:ignore
 from unidecode import unidecode
+from xml2rfc import RfcWriterError
 from xym import xym
 
 from django.conf import settings
@@ -918,8 +919,51 @@ def accept_submission_requires_group_approval(submission):
 
 
 class SubmissionError(Exception):
-    """Exception for errors during submission processing"""
-    pass
+    """Exception for errors during submission processing
+    
+    Sanitizes paths appearing in exception messages.
+    """
+    def __init__(self, *args):
+        if len(args) > 0:
+            args = (self.sanitize_message(arg) for arg in args)
+        super().__init__(*args)
+
+    @staticmethod
+    def sanitize_message(msg):
+        # Paths likely to appear in submission-related errors
+        paths = [
+            p for p in (
+                getattr(settings, "ALL_ID_DOWNLOAD_DIR", None),
+                getattr(settings, "BIBXML_BASE_PATH", None),
+                getattr(settings, "DERIVED_DIR", None),
+                getattr(settings, "FTP_DIR", None),
+                getattr(settings, "IDSUBMIT_REPOSITORY_PATH", None),
+                getattr(settings, "IDSUBMIT_STAGING_PATH", None),
+                getattr(settings, "INTERNET_ALL_DRAFTS_ARCHIVE_DIR", None),
+                getattr(settings, "INTERNET_DRAFT_PATH", None),
+                getattr(settings, "INTERNET_DRAFT_ARCHIVE_DIR", None),
+                getattr(settings, "INTERNET_DRAFT_PDF_PATH", None),
+                getattr(settings, "RFC_PATH", None),
+                getattr(settings, "SUBMIT_YANG_CATALOG_MODEL_DIR", None),
+                getattr(settings, "SUBMIT_YANG_DRAFT_MODEL_DIR", None),
+                getattr(settings, "SUBMIT_YANG_IANA_MODEL_DIR", None),
+                getattr(settings, "SUBMIT_YANG_RFC_MODEL_DIR", None),
+                "/tmp/",
+            ) if p is not None
+        ]
+        return re.sub(fr"({'|'.join(paths)})/*", "**/", msg)
+
+
+class XmlRfcError(SubmissionError):
+    """SubmissionError caused by xml2rfc
+    
+    Includes the output from xml2rfc, if any, in xml2rfc_stdout / xml2rfc_stderr
+    """
+    def __init__(self, *args, xml2rfc_stdout: str, xml2rfc_stderr: str):
+        super().__init__(*args)
+        self.xml2rfc_stderr = xml2rfc_stderr
+        self.xml2rfc_stdout = xml2rfc_stdout
+
 
 class InconsistentRevisionError(SubmissionError):
     """SubmissionError caused by an inconsistent revision"""
@@ -937,27 +981,55 @@ def render_missing_formats(submission):
     If a txt file already exists, leaves it in place. Overwrites an existing html file
     if there is one.
     """
-    xml2rfc.log.write_out = io.StringIO()   # open(os.devnull, "w")
-    xml2rfc.log.write_err = io.StringIO()   # open(os.devnull, "w")
+    # Capture stdio/stdout from xml2rfc
+    xml2rfc_stdout = io.StringIO()
+    xml2rfc_stderr = io.StringIO()
+    xml2rfc.log.write_out = xml2rfc_stdout
+    xml2rfc.log.write_err = xml2rfc_stderr
     xml_path = staging_path(submission.name, submission.rev, '.xml')
     parser = xml2rfc.XmlRfcParser(str(xml_path), quiet=True)
-    # --- Parse the xml ---
-    xmltree = parser.parse(remove_comments=False)
+    try:
+        # --- Parse the xml ---
+        xmltree = parser.parse(remove_comments=False)
+    except Exception as err:
+        raise XmlRfcError(
+            "Error parsing XML",
+            xml2rfc_stdout=xml2rfc_stdout.getvalue(),
+            xml2rfc_stderr=xml2rfc_stderr.getvalue(),
+        ) from err
     # If we have v2, run it through v2v3. Keep track of the submitted version, though.
     xmlroot = xmltree.getroot()
     xml_version = xmlroot.get('version', '2')
     if xml_version == '2':
         v2v3 = xml2rfc.V2v3XmlWriter(xmltree)
-        xmltree.tree = v2v3.convert2to3()
+        try:
+            xmltree.tree = v2v3.convert2to3()
+        except Exception as err:
+            raise XmlRfcError(
+                "Error converting v2 XML to v3",
+                xml2rfc_stdout=xml2rfc_stdout.getvalue(),
+                xml2rfc_stderr=xml2rfc_stderr.getvalue(),
+            ) from err
 
     # --- Prep the xml ---
     today = date_today()
     prep = xml2rfc.PrepToolWriter(xmltree, quiet=True, liberal=True, keep_pis=[xml2rfc.V3_PI_TARGET])
     prep.options.accept_prepped = True
     prep.options.date = today
-    xmltree.tree = prep.prep()
-    if xmltree.tree == None:
-        raise SubmissionError(f'Error from xml2rfc (prep): {prep.errors}')
+    try:
+        xmltree.tree = prep.prep()
+    except RfcWriterError:
+        raise XmlRfcError(
+            f"Error during xml2rfc prep: {prep.errors}",
+            xml2rfc_stdout=xml2rfc_stdout.getvalue(),
+            xml2rfc_stderr=xml2rfc_stderr.getvalue(),
+        )
+    except Exception as err:
+        raise XmlRfcError(
+            "Unexpected error during xml2rfc prep",
+            xml2rfc_stdout=xml2rfc_stdout.getvalue(),
+            xml2rfc_stderr=xml2rfc_stderr.getvalue(),
+        ) from err
 
     # --- Convert to txt ---
     txt_path = staging_path(submission.name, submission.rev, '.txt')
@@ -965,7 +1037,14 @@ def render_missing_formats(submission):
         writer = xml2rfc.TextWriter(xmltree, quiet=True)
         writer.options.accept_prepped = True
         writer.options.date = today
-        writer.write(txt_path)
+        try:
+            writer.write(txt_path)
+        except Exception as err:
+            raise XmlRfcError(
+                "Error generating text format from XML",
+            xml2rfc_stdout=xml2rfc_stdout.getvalue(),
+            xml2rfc_stderr=xml2rfc_stderr.getvalue(),
+            ) from err
         log.log(
             'In %s: xml2rfc %s generated %s from %s (version %s)' % (
                 str(xml_path.parent),
@@ -980,7 +1059,14 @@ def render_missing_formats(submission):
     html_path = staging_path(submission.name, submission.rev, '.html')
     writer = xml2rfc.HtmlWriter(xmltree, quiet=True)
     writer.options.date = today
-    writer.write(str(html_path))
+    try:
+        writer.write(str(html_path))
+    except Exception as err:
+        raise XmlRfcError(
+            "Error generating HTML format from XML",
+            xml2rfc_stdout=xml2rfc_stdout.getvalue(),
+            xml2rfc_stderr=xml2rfc_stderr.getvalue(),
+        ) from err
     log.log(
         'In %s: xml2rfc %s generated %s from %s (version %s)' % (
             str(xml_path.parent),
@@ -1263,7 +1349,7 @@ def process_submission_text(filename, revision):
 def process_and_validate_submission(submission):
     """Process and validate a submission
 
-    Raises SubmissionError if an error is encountered.
+    Raises SubmissionError or a subclass if an error is encountered.
     """
     if len(set(submission.file_types.split(",")).intersection({".xml", ".txt"})) == 0:
         raise SubmissionError("Require XML and/or text format to process an Internet-Draft submission.")
@@ -1273,7 +1359,16 @@ def process_and_validate_submission(submission):
         # Parse XML first, if we have it
         if ".xml" in submission.file_types:
             xml_metadata = process_submission_xml(submission.name, submission.rev)
-            render_missing_formats(submission)  # makes HTML and text, unless text was uploaded
+            try:
+                render_missing_formats(submission)  # makes HTML and text, unless text was uploaded
+            except XmlRfcError as err:
+                # log stdio/stderr
+                log.log(
+                    f"xml2rfc failure when rendering missing formats for {submission.name}-{submission.rev}:\n"
+                    f">> stdout:\n{err.xml2rfc_stdout}\n"
+                    f">> stderr:\n{err.xml2rfc_stderr}"
+                )
+                raise
         # Parse text, whether uploaded or generated from XML
         text_metadata = process_submission_text(submission.name, submission.rev)
 
@@ -1332,11 +1427,11 @@ def process_and_validate_submission(submission):
             raise SubmissionError('Checks failed: ' + ' / '.join(errors))
     except SubmissionError:
         raise  # pass SubmissionErrors up the stack
-    except Exception:
+    except Exception as err:
         # convert other exceptions into SubmissionErrors
         log.log(f'Unexpected exception while processing submission {submission.pk}.')
         log.log(traceback.format_exc())
-        raise SubmissionError('A system error occurred while processing the submission.')
+        raise SubmissionError('A system error occurred while processing the submission.') from err
 
 
 def submitter_is_author(submission):
@@ -1428,6 +1523,7 @@ def process_uploaded_submission(submission):
         create_submission_event(None, submission, desc="Uploaded submission (diverted to manual process)")
         send_manual_post_request(None, submission, errors=dict(consistency=str(consistency_error)))
     except SubmissionError as err:
+        # something generic went wrong
         submission.refresh_from_db()  # guard against incomplete changes in submission validation / processing
         cancel_submission(submission)  # changes Submission.state
         create_submission_event(None, submission, f"Submission rejected: {err}")

From 9c56ba9a0a53b6b92aa1cc0ac97764e976bf1463 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 17 Oct 2024 18:23:34 -0300
Subject: [PATCH 103/656] fix: drop, not bounce, uninteresting ipr emails
 (#8057)

* fix: drop, not bounce, uninteresting ipr emails

* chore: log to address

* chore: unused import
---
 ietf/ipr/mail.py                              | 35 +++++++----
 ietf/ipr/management/commands/process_email.py |  4 +-
 ietf/ipr/tests.py                             | 59 +++++++++++++------
 ietf/ipr/utils.py                             | 13 ++--
 4 files changed, 72 insertions(+), 39 deletions(-)

diff --git a/ietf/ipr/mail.py b/ietf/ipr/mail.py
index 842426d820..167b11956c 100644
--- a/ietf/ipr/mail.py
+++ b/ietf/ipr/mail.py
@@ -171,31 +171,44 @@ def message_from_message(message,by=None):
     )
     return msg
 
+
+class UndeliverableIprResponseError(Exception):
+    """Response email could not be delivered and should be treated as an error"""
+
+
 def process_response_email(msg):
-    """Saves an incoming message.  msg=string.  Message "To" field is expected to
-    be in the format ietf-ipr+[identifier]@ietf.org.  Expect to find a message with
-    a matching value in the reply_to field, associated to an IPR disclosure through
-    IprEvent.  Create a Message object for the incoming message and associate it to
-    the original message via new IprEvent"""
+    """Save an incoming IPR response email message
+    
+    Message "To" field is expected to be in the format ietf-ipr+[identifier]@ietf.org. If
+    the address or identifier is missing, the message will be silently dropped.
+
+    Expect to find a message with a matching value in the reply_to field, associated to an
+    IPR disclosure through IprEvent. If it cannot be matched, raises UndeliverableIprResponseError 
+    
+    Creates a Message object for the incoming message and associates it to
+    the original message via new IprEvent
+    """
     message = message_from_bytes(force_bytes(msg))
     to = message.get('To', '')
 
     # exit if this isn't a response we're interested in (with plus addressing)
-    local,domain = get_base_ipr_request_address().split('@')
+    local, domain = get_base_ipr_request_address().split('@')
     if not re.match(r'^{}\+[a-zA-Z0-9_\-]{}@{}'.format(local,'{16}',domain),to):
-        return None
-    
+        _from = message.get("From", "<unknown>")
+        log(f"Ignoring IPR email without a message identifier from {_from} to {to}")
+        return
+
     try:
         to_message = Message.objects.get(reply_to=to)
     except Message.DoesNotExist:
         log('Error finding matching message ({})'.format(to))
-        return None
+        raise UndeliverableIprResponseError(f"Unable to find message matching {to}")
 
     try:
         disclosure = to_message.msgevents.first().disclosure
     except:
         log('Error processing message ({})'.format(to))
-        return None
+        raise UndeliverableIprResponseError("Error processing message for {to}")
 
     ietf_message = message_from_message(message)
     IprEvent.objects.create(
@@ -207,4 +220,4 @@ def process_response_email(msg):
     )
     
     log("Received IPR email from %s" % ietf_message.frm)
-    return ietf_message
+
diff --git a/ietf/ipr/management/commands/process_email.py b/ietf/ipr/management/commands/process_email.py
index 0b15fb0651..616cade5c4 100644
--- a/ietf/ipr/management/commands/process_email.py
+++ b/ietf/ipr/management/commands/process_email.py
@@ -9,7 +9,7 @@
 from django.core.management import CommandError
 
 from ietf.utils.management.base import EmailOnFailureCommand
-from ietf.ipr.mail import process_response_email
+from ietf.ipr.mail import process_response_email, UndeliverableIprResponseError
 
 import debug                            # pyflakes:ignore
 
@@ -31,7 +31,7 @@ def handle(self, *args, **options):
             self.msg_bytes = sys.stdin.buffer.read()
         try:
             process_response_email(self.msg_bytes)
-        except ValueError as e:
+        except (ValueError, UndeliverableIprResponseError) as e:
             raise CommandError(e)
 
     failure_subject = 'Error during ipr email processing'
diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py
index 3c70567fd8..d72018f10b 100644
--- a/ietf/ipr/tests.py
+++ b/ietf/ipr/tests.py
@@ -4,6 +4,7 @@
 
 import datetime
 import mock
+import re
 
 from pyquery import PyQuery
 from urllib.parse import quote, urlparse
@@ -35,9 +36,9 @@
 )
 from ietf.ipr.forms import DraftForm, HolderIprDisclosureForm
 from ietf.ipr.mail import (process_response_email, get_reply_to, get_update_submitter_emails,
-    get_pseudo_submitter, get_holders, get_update_cc_addrs)
-from ietf.ipr.models import (IprDisclosureBase,GenericIprDisclosure,HolderIprDisclosure,
-    ThirdPartyIprDisclosure)
+                           get_pseudo_submitter, get_holders, get_update_cc_addrs, UndeliverableIprResponseError)
+from ietf.ipr.models import (IprDisclosureBase, GenericIprDisclosure, HolderIprDisclosure,
+                             ThirdPartyIprDisclosure, IprEvent)
 from ietf.ipr.templatetags.ipr_filters import no_revisions_message
 from ietf.ipr.utils import get_genitive, get_ipr_summary, ingest_response_email
 from ietf.mailtrigger.utils import gather_address_lists
@@ -712,7 +713,7 @@ def test_notify_generic(self):
         )
         self.assertIn(f'{settings.IDTRACKER_BASE_URL}{urlreverse("ietf.ipr.views.showlist")}', get_payload_text(outbox[1]).replace('\n',' '))
 
-    def send_ipr_email_helper(self):
+    def send_ipr_email_helper(self) -> tuple[str, IprEvent, HolderIprDisclosure]:
         ipr = HolderIprDisclosureFactory()
         url = urlreverse('ietf.ipr.views.email',kwargs={ "id": ipr.id })
         self.client.login(username="secretary", password="secretary+password")
@@ -730,10 +731,11 @@ def send_ipr_email_helper(self):
         q = Message.objects.filter(reply_to=data['reply_to'])
         self.assertEqual(q.count(),1)
         event = q[0].msgevents.first()
+        assert event is not None
         self.assertTrue(event.response_past_due())
         self.assertEqual(len(outbox), 1)
         self.assertTrue('joe@test.com' in outbox[0]['To'])
-        return data['reply_to'], event
+        return data['reply_to'], event, ipr
 
     uninteresting_ipr_message_strings = [
         ("To: {to}\nCc: {cc}\nFrom: joe@test.com\nDate: {date}\nSubject: test\n"),
@@ -747,34 +749,46 @@ def send_ipr_email_helper(self):
 
     def test_process_response_email(self):
         # first send a mail
-        reply_to, event = self.send_ipr_email_helper()
+        reply_to, event, _ = self.send_ipr_email_helper()
 
         # test process response uninteresting messages
         addrs = gather_address_lists('ipr_disclosure_submitted').as_strings()
         for message_string in self.uninteresting_ipr_message_strings:
-            result = process_response_email(
+            process_response_email(
                 message_string.format(
                     to=addrs.to,
                     cc=addrs.cc,
                     date=timezone.now().ctime()
                 )
             )
-            self.assertIsNone(result)
-        
+
         # test process response
         message_string = """To: {}
 From: joe@test.com
 Date: {}
 Subject: test
 """.format(reply_to, timezone.now().ctime())
-        result = process_response_email(message_string)
-
-        self.assertIsInstance(result, Message)
+        process_response_email(message_string)
         self.assertFalse(event.response_past_due())
 
+        # test with an unmatchable message identifier
+        bad_reply_to = re.sub(
+            r"\+.{16}@",
+            '+0123456789abcdef@',
+            reply_to,
+        )
+        self.assertNotEqual(reply_to, bad_reply_to)
+        message_string = f"""To: {bad_reply_to}
+        From: joe@test.com
+        Date: {timezone.now().ctime()}
+        Subject: test
+        """
+        with self.assertRaises(UndeliverableIprResponseError):
+            process_response_email(message_string)
+
     def test_process_response_email_with_invalid_encoding(self):
         """Interesting emails with invalid encoding should be handled"""
-        reply_to, _ = self.send_ipr_email_helper()
+        reply_to, _, disclosure = self.send_ipr_email_helper()
         # test process response
         message_string = """To: {}
 From: joe@test.com
@@ -782,8 +796,8 @@ def test_process_response_email_with_invalid_encoding(self):
 Subject: test
 """.format(reply_to, timezone.now().ctime())
         message_bytes = message_string.encode('utf8') + b'\nInvalid stuff: \xfe\xff\n'
-        result = process_response_email(message_bytes)
-        self.assertIsInstance(result, Message)
+        process_response_email(message_bytes)
+        result = IprEvent.objects.filter(disclosure=disclosure).first().message  # newest
         # \ufffd is a rhombus character with an inverse ?, used to replace invalid characters
         self.assertEqual(result.body, 'Invalid stuff: \ufffd\ufffd\n\n',  # not sure where the extra \n is from
                          'Invalid characters should be replaced with \ufffd characters')
@@ -798,8 +812,7 @@ def test_process_response_email_uninteresting_with_invalid_encoding(self):
                                 cc=addrs.cc,
                                 date=timezone.now().ctime(),
             ).encode('utf8') + b'\nInvalid stuff: \xfe\xff\n'
-            result = process_response_email(message_bytes)
-            self.assertIsNone(result)
+            process_response_email(message_bytes)
 
     @override_settings(ADMINS=(("Some Admin", "admin@example.com"),))
     @mock.patch("ietf.ipr.utils.process_response_email")
@@ -816,8 +829,8 @@ def test_ingest_response_email(self, mock_process_response_email):
         self.assertEqual(mock_process_response_email.call_args, mock.call(message))
         mock_process_response_email.reset_mock()
         
-        mock_process_response_email.side_effect = None
-        mock_process_response_email.return_value = None  # rejected message
+        mock_process_response_email.side_effect = UndeliverableIprResponseError
+        mock_process_response_email.return_value = None
         with self.assertRaises(EmailIngestionError) as context:
             ingest_response_email(message)
         self.assertIsNone(context.exception.as_emailmessage())  # should not send an email on a clean rejection
@@ -825,6 +838,14 @@ def test_ingest_response_email(self, mock_process_response_email):
         self.assertEqual(mock_process_response_email.call_args, mock.call(message))
         mock_process_response_email.reset_mock()
 
+        mock_process_response_email.side_effect = None
+        mock_process_response_email.return_value = None  # ignored message
+        ingest_response_email(message)  # should not raise an exception
+        self.assertIsNone(context.exception.as_emailmessage())  # should not send an email on ignored message
+        self.assertTrue(mock_process_response_email.called)
+        self.assertEqual(mock_process_response_email.call_args, mock.call(message))
+        mock_process_response_email.reset_mock()
+
         # successful operation
         mock_process_response_email.return_value = MessageFactory()
         ingest_response_email(message)
diff --git a/ietf/ipr/utils.py b/ietf/ipr/utils.py
index 42d485ccad..8f0b9cf3f2 100644
--- a/ietf/ipr/utils.py
+++ b/ietf/ipr/utils.py
@@ -3,7 +3,7 @@
 
 from textwrap import dedent
 
-from ietf.ipr.mail import process_response_email
+from ietf.ipr.mail import process_response_email, UndeliverableIprResponseError
 from ietf.ipr.models import IprDocRel
 
 import debug                            # pyflakes:ignore
@@ -92,7 +92,11 @@ def generate_draft_recursive_txt():
 def ingest_response_email(message: bytes):
     from ietf.api.views import EmailIngestionError  # avoid circular import
     try:
-        result = process_response_email(message)
+        process_response_email(message)
+    except UndeliverableIprResponseError:
+        # Message was rejected due to some problem the sender can fix, so bounce but don't send
+        # an email to the admins
+        raise EmailIngestionError("IPR response rejected", email_body=None)
     except Exception as err:
         # Message was rejected due to an unhandled exception. This is likely something
         # the admins need to address, so send them a copy of the email.
@@ -106,8 +110,3 @@ def ingest_response_email(message: bytes):
             email_original_message=message,
             email_attach_traceback=True,
         ) from err
-
-    if result is None:
-        # Message was rejected due to some problem the sender can fix, so bounce but don't send
-        # an email to the admins
-        raise EmailIngestionError("IPR response rejected", email_body=None)

From 9303aaf5f7d1722f727ded1372f04ae11043dcce Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 17 Oct 2024 21:07:51 -0400
Subject: [PATCH 104/656] ci: Update build.yml

---
 .github/workflows/build.yml | 125 ++++++++++++++++++++++++++++++------
 1 file changed, 105 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3887661cac..1ab42ae80a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -234,31 +234,116 @@ jobs:
         docker-compose.yml
         EOL
 
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-    
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+    - name: Launch build VM
+      id: azlaunch
+      run: |
+        echo "Authenticating to Azure..."
+        az login --service-principal -u ${{ secrets.AZ_BUILD_APP_ID }} -p ${{ secrets.AZ_BUILD_PWD }} --tenant ${{ secrets.AZ_BUILD_TENANT_ID }}
+        echo "Creating VM..."
+        vminfo=$(az vm create \
+          --resource-group ghaDatatracker \
+          --name tmpGhaBuildVM \
+          --image Ubuntu2404 \
+          --admin-username azureuser \
+          --generate-ssh-keys \
+          --priority Spot \
+          --size Standard_D8as_v6 \
+          --max-price -1 \
+          --os-disk-size-gb 30 \
+          --eviction-policy Delete \
+          --nic-delete-option Delete \
+          --output tsv \
+          --query "publicIpAddress")
+        echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT"
+        echo "VM Public IP: $vminfo"
+        cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key
+        ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts
 
-    - name: Login to GitHub Container Registry
-      uses: docker/login-action@v3
+    - name: Copy build files
+      uses: appleboy/scp-action@917f8b81dfc1ccd331fef9e2d61bdc6c8be94634
       with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
+        host: ${{ steps.azlaunch.outputs.ipaddr }}
+        port: 22
+        username: azureuser
+        key_path: ${{ github.workspace }}/prvkey.key
+        source: "${{ github.workspace }},!${{ github.workspace }}/.git,!${{ github.workspace }}/prvkey.key"
+        target: /workspace
 
-    - name: Build Release Docker Image
-      uses: docker/build-push-action@v6
+    - name: Remote SSH into Build VM
+      uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1
       env:
-        DOCKER_BUILD_SUMMARY: false
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_ACTOR: ${{ github.actor }}
+        PKG_VERSION: ${{ env.PKG_VERSION }}
       with:
-        context: .
-        file: dev/build/Dockerfile
-        platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }}
-        push: true
-        tags: ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }}
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
+        host: ${{ steps.azlaunch.outputs.ipaddr }}
+        port: 22
+        username: azureuser
+        command_timeout: 60m
+        key_path: ${{ github.workspace }}/prvkey.key
+        envs: GITHUB_TOKEN
+        script_stop: true
+        script: |
+          export DEBIAN_FRONTEND=noninteractive
+          lsb_release -a
+          sudo apt-get update
+          sudo apt-get upgrade -y
+          
+          echo "Installing Docker..."
+          curl -fsSL https://get.docker.com -o get-docker.sh
+          sudo sh get-docker.sh
+
+          echo "Login to ghcr.io..."
+          echo $GITHUB_TOKEN | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
+          
+          echo "Building Images..."
+          cd /workspace
+          ls
+
+    - name: Destroy Build VM + resources
+      if: always()
+      shell: pwsh
+      run: |
+        echo "Destroying VM..."        
+        az vm delete -g ghaDatatracker -n tmpGhaBuildVM --yes --force-deletion true
+
+        $resourceOrderRemovalOrder = [ordered]@{
+            "Microsoft.Compute/virtualMachines" = 0
+            "Microsoft.Compute/disks" = 1
+            "Microsoft.Network/networkInterfaces" = 2
+            "Microsoft.Network/publicIpAddresses" = 3
+            "Microsoft.Network/networkSecurityGroups" = 4
+            "Microsoft.Network/virtualNetworks" = 5
+        }
+        echo "Fetching remaining resources..."
+        $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json
+
+        $orderedResources = $resources 
+            | Sort-Object @{
+                Expression = {$resourceOrderRemovalOrder[$_.type]}
+                Descending = $False
+            }
+
+        echo "Deleting remaining resources..."
+        $orderedResources | ForEach-Object {
+            az resource delete --resource-group ghaDatatracker --ids $_.id --verbose
+        }
+
+        echo "Logout from Azure..."
+        az logout
+
+    # - name: Build Release Docker Image
+    #   uses: docker/build-push-action@v6
+    #   env:
+    #     DOCKER_BUILD_SUMMARY: false
+    #   with:
+    #     context: .
+    #     file: dev/build/Dockerfile
+    #     platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }}
+    #     push: true
+    #     tags: ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }}
+    #     cache-from: type=gha
+    #     cache-to: type=gha,mode=max
         
     - name: Update CHANGELOG
       id: changelog

From 5c2369362d7474d9e62de10b2a9bedbf44e691ed Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 17 Oct 2024 21:12:46 -0400
Subject: [PATCH 105/656] ci: Update build.yml

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 1ab42ae80a..07aca3794c 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -243,7 +243,7 @@ jobs:
         vminfo=$(az vm create \
           --resource-group ghaDatatracker \
           --name tmpGhaBuildVM \
-          --image Ubuntu2404 \
+          --image Ubuntu2204 \
           --admin-username azureuser \
           --generate-ssh-keys \
           --priority Spot \

From bbfbb25844d412a227d2b801714eb86c61a77ba7 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 17 Oct 2024 21:46:58 -0400
Subject: [PATCH 106/656] ci: Update build.yml

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 07aca3794c..158ce19107 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -247,7 +247,7 @@ jobs:
           --admin-username azureuser \
           --generate-ssh-keys \
           --priority Spot \
-          --size Standard_D8as_v6 \
+          --size Standard_D8as_v5 \
           --max-price -1 \
           --os-disk-size-gb 30 \
           --eviction-policy Delete \

From 453ddc87ecfb3f9f668adb085d1f9558780c9b50 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 17 Oct 2024 23:15:27 -0400
Subject: [PATCH 107/656] ci: Update build.yml

---
 .github/workflows/build.yml | 188 +++++++++++++++++++-----------------
 1 file changed, 101 insertions(+), 87 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 158ce19107..55c9b0a1b2 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -156,83 +156,6 @@ jobs:
       with:
         fetch-depth: 1
         fetch-tags: false
-        
-    - name: Setup Node.js
-      uses: actions/setup-node@v4
-      with:
-        node-version: '16'
-    
-    - name: Setup Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.x'
-        
-    - name: Download a Coverage Results
-      if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
-      uses: actions/download-artifact@v4.1.8
-      with:
-        name: coverage
-
-    - name: Make Release Build
-      env:
-        DEBIAN_FRONTEND: noninteractive
-        BROWSERSLIST_IGNORE_OLD_DATA: 1
-      run: |
-        echo "PKG_VERSION: $PKG_VERSION"
-        echo "GITHUB_SHA: $GITHUB_SHA"
-        echo "GITHUB_REF_NAME: $GITHUB_REF_NAME"
-        echo "Running frontend build script..."
-        echo "Compiling native node packages..."
-        yarn rebuild
-        echo "Packaging static assets..."
-        yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/
-        yarn legacy:build
-        echo "Setting version $PKG_VERSION..."
-        sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py
-        sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py
-        sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py
-
-    - name: Set Production Flags
-      if: ${{ env.SHOULD_DEPLOY == 'true' }}
-      run: |
-        echo "Setting production flags in settings.py..."
-        sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py
-        
-    - name: Make Release Tarball
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      run: |
-        echo "Build release tarball..."
-        mkdir -p /home/runner/work/release
-        tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt .
-
-    - name: Collect + Push Statics
-      env:
-        DEBIAN_FRONTEND: noninteractive
-        AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }}
-        AWS_DEFAULT_REGION: auto
-        AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }}
-      run: |
-        echo "Collecting statics..."
-        docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:latest sh dev/build/collectstatics.sh
-        echo "Pushing statics..."
-        cd static
-        aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
-
-    - name: Augment dockerignore for docker image build
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      run: |
-        cat >> .dockerignore <<EOL
-        .devcontainer
-        .github
-        .vscode
-        k8s
-        playwright
-        svn-history
-        docker-compose.yml
-        EOL
 
     - name: Launch build VM
       id: azlaunch
@@ -259,22 +182,33 @@ jobs:
         cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key
         ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts
 
-    - name: Copy build files
-      uses: appleboy/scp-action@917f8b81dfc1ccd331fef9e2d61bdc6c8be94634
-      with:
-        host: ${{ steps.azlaunch.outputs.ipaddr }}
-        port: 22
-        username: azureuser
-        key_path: ${{ github.workspace }}/prvkey.key
-        source: "${{ github.workspace }},!${{ github.workspace }}/.git,!${{ github.workspace }}/prvkey.key"
-        target: /workspace
+    # - name: Copy build files
+    #   uses: appleboy/scp-action@917f8b81dfc1ccd331fef9e2d61bdc6c8be94634
+    #   with:
+    #     host: ${{ steps.azlaunch.outputs.ipaddr }}
+    #     port: 22
+    #     username: azureuser
+    #     key_path: ${{ github.workspace }}/prvkey.key
+    #     source: "${{ github.workspace }},!${{ github.workspace }}/.git,!${{ github.workspace }}/prvkey.key"
+    #     target: /workspace
 
     - name: Remote SSH into Build VM
       uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         GITHUB_ACTOR: ${{ github.actor }}
+        GITHUB_SHA: ${{ github.sha }}
+        GITHUB_REF_NAME: ${{ github.ref_name }}
+        GITHUB_RUN_ID: ${{ github.run_id }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }}
+        AWS_DEFAULT_REGION: auto
+        AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }}
         PKG_VERSION: ${{ env.PKG_VERSION }}
+        SHOULD_DEPLOY: ${{ env.SHOULD_DEPLOY }}
+        SKIP_TESTS: ${{ github.event.inputs.skiptests }}
+        DEBIAN_FRONTEND: noninteractive
+        BROWSERSLIST_IGNORE_OLD_DATA: 1
       with:
         host: ${{ steps.azlaunch.outputs.ipaddr }}
         port: 22
@@ -295,10 +229,90 @@ jobs:
 
           echo "Login to ghcr.io..."
           echo $GITHUB_TOKEN | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
+
+          echo "Installing GH CLI..."
+          (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \
+          	&& sudo mkdir -p -m 755 /etc/apt/keyrings \
+          	&& wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
+          	&& sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
+          	&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+          	&& sudo apt update \
+          	&& sudo apt install gh -y
+
+          echo "Installing AWS CLI..."
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+          unzip awscliv2.zip
+          sudo ./aws/install
+
+          echo "Install Node.js..."
+          curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh
+          sudo bash nodesource_setup.sh
+          sudo apt-get install -y nodejs
+          sudo corepack enable
+
+          echo "Install Python 3.x..."
+          sudo apt-get install python3 python3-dev -y
+          python3 --version
+
+          echo "Clone project..."
+          sudo mkdir -p /workspace
+          sudo chown azureuser /workspace
+          cd /workspace
+          gh repo clone ietf-tools/datatracker -- --depth=1 --no-tags
+          cd datatracker
+
+          if [ "$SHOULD_DEPLOY" = false ] || [ "$GITHUB_REF_NAME" = release ] ; then
+            echo "Downloading coverage..."
+            gh run download $GITHUB_RUN_ID -n coverage
+          fi
+
+          echo "Building project..."
+          echo "PKG_VERSION: $PKG_VERSION"
+          echo "GITHUB_SHA: $GITHUB_SHA"
+          echo "GITHUB_REF_NAME: $GITHUB_REF_NAME"
+          echo "Running frontend build script..."
+          echo "Compiling native node packages..."
+          yarn rebuild
+          echo "Packaging static assets..."
+          yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/
+          yarn legacy:build
+          echo "Setting version $PKG_VERSION..."
+          sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py
+          sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py
+          sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py
+
+          if [ "$SHOULD_DEPLOY" = true ] ; then
+            echo "Setting production flags in settings.py..."
+            sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py
+          fi
+
+          echo "Build release tarball..."
+          mkdir -p /workspace/release
+          tar -czf /workspace/release.tar.gz -X dev/build/exclude-patterns.txt .
+
+          echo "Collecting statics..."
+          docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:latest sh dev/build/collectstatics.sh
+          
+          echo "Pushing statics..."
+          cd static
+          aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
+          cd ..
+
+          echo "Augment dockerignore for docker image build..."
+          cat >> .dockerignore <<EOL
+          .devcontainer
+          .github
+          .vscode
+          k8s
+          playwright
+          svn-history
+          docker-compose.yml
+          EOL
           
           echo "Building Images..."
-          cd /workspace
           ls
+          echo "Error!" 1>&2
+          exit 64
 
     - name: Destroy Build VM + resources
       if: always()

From a074cd4b85f6584e63fa7491cfd0d464bb83279e Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 17 Oct 2024 23:25:14 -0400
Subject: [PATCH 108/656] ci: Update build.yml

---
 .github/workflows/build.yml | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 55c9b0a1b2..cc7b622036 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -222,15 +222,21 @@ jobs:
           lsb_release -a
           sudo apt-get update
           sudo apt-get upgrade -y
-          
+
+          echo "=========================================================================="
           echo "Installing Docker..."
+          echo "=========================================================================="
           curl -fsSL https://get.docker.com -o get-docker.sh
           sudo sh get-docker.sh
 
+          echo "=========================================================================="
           echo "Login to ghcr.io..."
-          echo $GITHUB_TOKEN | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
+          echo "=========================================================================="
+          echo $GITHUB_TOKEN | sudo docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
 
+          echo "=========================================================================="
           echo "Installing GH CLI..."
+          echo "=========================================================================="
           (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \
           	&& sudo mkdir -p -m 755 /etc/apt/keyrings \
           	&& wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
@@ -239,22 +245,30 @@ jobs:
           	&& sudo apt update \
           	&& sudo apt install gh -y
 
+          echo "=========================================================================="
           echo "Installing AWS CLI..."
+          echo "=========================================================================="
           curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
           unzip awscliv2.zip
           sudo ./aws/install
 
+          echo "=========================================================================="
           echo "Install Node.js..."
+          echo "=========================================================================="
           curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh
           sudo bash nodesource_setup.sh
           sudo apt-get install -y nodejs
           sudo corepack enable
 
+          echo "=========================================================================="
           echo "Install Python 3.x..."
+          echo "=========================================================================="
           sudo apt-get install python3 python3-dev -y
           python3 --version
 
+          echo "=========================================================================="
           echo "Clone project..."
+          echo "=========================================================================="
           sudo mkdir -p /workspace
           sudo chown azureuser /workspace
           cd /workspace
@@ -262,11 +276,15 @@ jobs:
           cd datatracker
 
           if [ "$SHOULD_DEPLOY" = false ] || [ "$GITHUB_REF_NAME" = release ] ; then
+            echo "=========================================================================="
             echo "Downloading coverage..."
+            echo "=========================================================================="
             gh run download $GITHUB_RUN_ID -n coverage
           fi
 
+          echo "=========================================================================="
           echo "Building project..."
+          echo "=========================================================================="
           echo "PKG_VERSION: $PKG_VERSION"
           echo "GITHUB_SHA: $GITHUB_SHA"
           echo "GITHUB_REF_NAME: $GITHUB_REF_NAME"
@@ -282,23 +300,30 @@ jobs:
           sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py
 
           if [ "$SHOULD_DEPLOY" = true ] ; then
+            echo "=========================================================================="
             echo "Setting production flags in settings.py..."
+            echo "=========================================================================="
             sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py
           fi
 
+          echo "=========================================================================="
           echo "Build release tarball..."
+          echo "=========================================================================="
           mkdir -p /workspace/release
           tar -czf /workspace/release.tar.gz -X dev/build/exclude-patterns.txt .
 
+          echo "=========================================================================="
           echo "Collecting statics..."
+          echo "=========================================================================="
           docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:latest sh dev/build/collectstatics.sh
-          
           echo "Pushing statics..."
           cd static
           aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
           cd ..
 
+          echo "=========================================================================="
           echo "Augment dockerignore for docker image build..."
+          echo "=========================================================================="
           cat >> .dockerignore <<EOL
           .devcontainer
           .github
@@ -308,8 +333,10 @@ jobs:
           svn-history
           docker-compose.yml
           EOL
-          
+
+          echo "=========================================================================="
           echo "Building Images..."
+          echo "=========================================================================="
           ls
           echo "Error!" 1>&2
           exit 64

From a104360ace7ce04944cb603ddf4eff5adf47663d Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 17 Oct 2024 23:31:50 -0400
Subject: [PATCH 109/656] ci: Update build.yml

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index cc7b622036..3c7cf523c1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -215,7 +215,7 @@ jobs:
         username: azureuser
         command_timeout: 60m
         key_path: ${{ github.workspace }}/prvkey.key
-        envs: GITHUB_TOKEN
+        envs: GITHUB_TOKEN,GITHUB_ACTOR,GITHUB_SHA,GITHUB_REF_NAME,GITHUB_RUN_ID,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_DEFAULT_REGION,AWS_ENDPOINT_URL,PKG_VERSION,SHOULD_DEPLOY,SKIP_TESTS,DEBIAN_FRONTEND,BROWSERSLIST_IGNORE_OLD_DATA
         script_stop: true
         script: |
           export DEBIAN_FRONTEND=noninteractive

From 60553adee4303ffb95f1a4a238d7d72e4805fe68 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 17 Oct 2024 23:37:43 -0400
Subject: [PATCH 110/656] ci: Update build.yml

---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3c7cf523c1..877e7dc95a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -222,6 +222,7 @@ jobs:
           lsb_release -a
           sudo apt-get update
           sudo apt-get upgrade -y
+          sudo apt-get install wget unzip curl -y
 
           echo "=========================================================================="
           echo "Installing Docker..."
@@ -237,8 +238,7 @@ jobs:
           echo "=========================================================================="
           echo "Installing GH CLI..."
           echo "=========================================================================="
-          (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \
-          	&& sudo mkdir -p -m 755 /etc/apt/keyrings \
+          sudo mkdir -p -m 755 /etc/apt/keyrings \
           	&& wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
           	&& sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
           	&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \

From b324d75583efab775d68141567ca2ddf7bcb75c8 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 17 Oct 2024 23:46:56 -0400
Subject: [PATCH 111/656] ci: Update build.yml

---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 877e7dc95a..4eb03f4604 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -275,7 +275,7 @@ jobs:
           gh repo clone ietf-tools/datatracker -- --depth=1 --no-tags
           cd datatracker
 
-          if [ "$SHOULD_DEPLOY" = false ] || [ "$GITHUB_REF_NAME" = release ] ; then
+          if [ "$SHOULD_DEPLOY" = "false" ] || [ "$GITHUB_REF_NAME" = "release" ] ; then
             echo "=========================================================================="
             echo "Downloading coverage..."
             echo "=========================================================================="
@@ -299,7 +299,7 @@ jobs:
           sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py
           sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py
 
-          if [ "$SHOULD_DEPLOY" = true ] ; then
+          if [ "$SHOULD_DEPLOY" = "true" ] ; then
             echo "=========================================================================="
             echo "Setting production flags in settings.py..."
             echo "=========================================================================="

From 400bbca958442926917d1df49dd5fb8b0b2f27f9 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 17 Oct 2024 23:56:15 -0400
Subject: [PATCH 112/656] ci: Update build.yml

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4eb03f4604..0d4e7932d4 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -275,7 +275,7 @@ jobs:
           gh repo clone ietf-tools/datatracker -- --depth=1 --no-tags
           cd datatracker
 
-          if [ "$SHOULD_DEPLOY" = "false" ] || [ "$GITHUB_REF_NAME" = "release" ] ; then
+          if [ "$SKIP_TESTS" = "false" ] || [ "$GITHUB_REF_NAME" = "release" ] ; then
             echo "=========================================================================="
             echo "Downloading coverage..."
             echo "=========================================================================="

From ebc7cde351afcfb1869af92749c046354523c39a Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 18 Oct 2024 00:04:37 -0400
Subject: [PATCH 113/656] ci: Update build.yml

---
 .github/workflows/build.yml | 64 ++++++++++++++++++-------------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0d4e7932d4..e69b4e31c2 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -258,7 +258,7 @@ jobs:
           curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh
           sudo bash nodesource_setup.sh
           sudo apt-get install -y nodejs
-          sudo corepack enable
+          corepack enable
 
           echo "=========================================================================="
           echo "Install Python 3.x..."
@@ -341,37 +341,37 @@ jobs:
           echo "Error!" 1>&2
           exit 64
 
-    - name: Destroy Build VM + resources
-      if: always()
-      shell: pwsh
-      run: |
-        echo "Destroying VM..."        
-        az vm delete -g ghaDatatracker -n tmpGhaBuildVM --yes --force-deletion true
-
-        $resourceOrderRemovalOrder = [ordered]@{
-            "Microsoft.Compute/virtualMachines" = 0
-            "Microsoft.Compute/disks" = 1
-            "Microsoft.Network/networkInterfaces" = 2
-            "Microsoft.Network/publicIpAddresses" = 3
-            "Microsoft.Network/networkSecurityGroups" = 4
-            "Microsoft.Network/virtualNetworks" = 5
-        }
-        echo "Fetching remaining resources..."
-        $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json
-
-        $orderedResources = $resources 
-            | Sort-Object @{
-                Expression = {$resourceOrderRemovalOrder[$_.type]}
-                Descending = $False
-            }
-
-        echo "Deleting remaining resources..."
-        $orderedResources | ForEach-Object {
-            az resource delete --resource-group ghaDatatracker --ids $_.id --verbose
-        }
-
-        echo "Logout from Azure..."
-        az logout
+    # - name: Destroy Build VM + resources
+    #   if: always()
+    #   shell: pwsh
+    #   run: |
+    #     echo "Destroying VM..."        
+    #     az vm delete -g ghaDatatracker -n tmpGhaBuildVM --yes --force-deletion true
+
+    #     $resourceOrderRemovalOrder = [ordered]@{
+    #         "Microsoft.Compute/virtualMachines" = 0
+    #         "Microsoft.Compute/disks" = 1
+    #         "Microsoft.Network/networkInterfaces" = 2
+    #         "Microsoft.Network/publicIpAddresses" = 3
+    #         "Microsoft.Network/networkSecurityGroups" = 4
+    #         "Microsoft.Network/virtualNetworks" = 5
+    #     }
+    #     echo "Fetching remaining resources..."
+    #     $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json
+
+    #     $orderedResources = $resources 
+    #         | Sort-Object @{
+    #             Expression = {$resourceOrderRemovalOrder[$_.type]}
+    #             Descending = $False
+    #         }
+
+    #     echo "Deleting remaining resources..."
+    #     $orderedResources | ForEach-Object {
+    #         az resource delete --resource-group ghaDatatracker --ids $_.id --verbose
+    #     }
+
+    #     echo "Logout from Azure..."
+    #     az logout
 
     # - name: Build Release Docker Image
     #   uses: docker/build-push-action@v6

From 324e09af5ae375d0d80908818ad2686854168b84 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 18 Oct 2024 00:12:49 -0400
Subject: [PATCH 114/656] ci: Update build.yml

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e69b4e31c2..92f4e93b5b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -258,7 +258,7 @@ jobs:
           curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh
           sudo bash nodesource_setup.sh
           sudo apt-get install -y nodejs
-          corepack enable
+          sudo corepack enable
 
           echo "=========================================================================="
           echo "Install Python 3.x..."

From aa36fca2805f144f45a4ebd7e4d2813a239ca2e1 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 18 Oct 2024 00:27:00 -0400
Subject: [PATCH 115/656] ci: Update build.yml

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 92f4e93b5b..c3112c460e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -255,7 +255,7 @@ jobs:
           echo "=========================================================================="
           echo "Install Node.js..."
           echo "=========================================================================="
-          curl -fsSL https://deb.nodesource.com/setup_22.x -o nodesource_setup.sh
+          curl -fsSL https://deb.nodesource.com/setup_18.x -o nodesource_setup.sh
           sudo bash nodesource_setup.sh
           sudo apt-get install -y nodejs
           sudo corepack enable

From dce23818b6d11e8ddddee32c83c84cb38b45d762 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 18 Oct 2024 00:33:56 -0400
Subject: [PATCH 116/656] ci: Update build.yml

---
 .github/workflows/build.yml | 64 ++++++++++++++++++-------------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c3112c460e..f54ab43aa2 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -315,7 +315,7 @@ jobs:
           echo "=========================================================================="
           echo "Collecting statics..."
           echo "=========================================================================="
-          docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:latest sh dev/build/collectstatics.sh
+          sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:latest sh dev/build/collectstatics.sh
           echo "Pushing statics..."
           cd static
           aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
@@ -341,37 +341,37 @@ jobs:
           echo "Error!" 1>&2
           exit 64
 
-    # - name: Destroy Build VM + resources
-    #   if: always()
-    #   shell: pwsh
-    #   run: |
-    #     echo "Destroying VM..."        
-    #     az vm delete -g ghaDatatracker -n tmpGhaBuildVM --yes --force-deletion true
-
-    #     $resourceOrderRemovalOrder = [ordered]@{
-    #         "Microsoft.Compute/virtualMachines" = 0
-    #         "Microsoft.Compute/disks" = 1
-    #         "Microsoft.Network/networkInterfaces" = 2
-    #         "Microsoft.Network/publicIpAddresses" = 3
-    #         "Microsoft.Network/networkSecurityGroups" = 4
-    #         "Microsoft.Network/virtualNetworks" = 5
-    #     }
-    #     echo "Fetching remaining resources..."
-    #     $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json
-
-    #     $orderedResources = $resources 
-    #         | Sort-Object @{
-    #             Expression = {$resourceOrderRemovalOrder[$_.type]}
-    #             Descending = $False
-    #         }
-
-    #     echo "Deleting remaining resources..."
-    #     $orderedResources | ForEach-Object {
-    #         az resource delete --resource-group ghaDatatracker --ids $_.id --verbose
-    #     }
-
-    #     echo "Logout from Azure..."
-    #     az logout
+    - name: Destroy Build VM + resources
+      if: always()
+      shell: pwsh
+      run: |
+        echo "Destroying VM..."        
+        az vm delete -g ghaDatatracker -n tmpGhaBuildVM --yes --force-deletion true
+
+        $resourceOrderRemovalOrder = [ordered]@{
+            "Microsoft.Compute/virtualMachines" = 0
+            "Microsoft.Compute/disks" = 1
+            "Microsoft.Network/networkInterfaces" = 2
+            "Microsoft.Network/publicIpAddresses" = 3
+            "Microsoft.Network/networkSecurityGroups" = 4
+            "Microsoft.Network/virtualNetworks" = 5
+        }
+        echo "Fetching remaining resources..."
+        $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json
+
+        $orderedResources = $resources 
+            | Sort-Object @{
+                Expression = {$resourceOrderRemovalOrder[$_.type]}
+                Descending = $False
+            }
+
+        echo "Deleting remaining resources..."
+        $orderedResources | ForEach-Object {
+            az resource delete --resource-group ghaDatatracker --ids $_.id --verbose
+        }
+
+        echo "Logout from Azure..."
+        az logout
 
     # - name: Build Release Docker Image
     #   uses: docker/build-push-action@v6

From d9d191fba56a77297cb20c5cbd634406fff79b12 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 18 Oct 2024 00:57:31 -0400
Subject: [PATCH 117/656] ci: Update build.yml

---
 .github/workflows/build.yml | 38 ++++++++++++-------------------------
 1 file changed, 12 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f54ab43aa2..48113ffb30 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -182,16 +182,6 @@ jobs:
         cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key
         ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts
 
-    # - name: Copy build files
-    #   uses: appleboy/scp-action@917f8b81dfc1ccd331fef9e2d61bdc6c8be94634
-    #   with:
-    #     host: ${{ steps.azlaunch.outputs.ipaddr }}
-    #     port: 22
-    #     username: azureuser
-    #     key_path: ${{ github.workspace }}/prvkey.key
-    #     source: "${{ github.workspace }},!${{ github.workspace }}/.git,!${{ github.workspace }}/prvkey.key"
-    #     target: /workspace
-
     - name: Remote SSH into Build VM
       uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1
       env:
@@ -337,9 +327,12 @@ jobs:
           echo "=========================================================================="
           echo "Building Images..."
           echo "=========================================================================="
-          ls
-          echo "Error!" 1>&2
-          exit 64
+          sudo docker buildx build --file dev/build/Dockerfile --platform linux/amd64,linux/arm64 --tag ghcr.io/ietf-tools/datatracker:$PKG_VERSION --push .
+
+    - name: Fetch release tarball
+      run: |
+        mkdir -p /home/runner/work/release
+        scp -i ${{ github.workspace }}/prvkey.key azureuser@${{ steps.azlaunch.outputs.ipaddr }}:/workspace/release.tar.gz /home/runner/work/release/release.tar.gz
 
     - name: Destroy Build VM + resources
       if: always()
@@ -372,19 +365,6 @@ jobs:
 
         echo "Logout from Azure..."
         az logout
-
-    # - name: Build Release Docker Image
-    #   uses: docker/build-push-action@v6
-    #   env:
-    #     DOCKER_BUILD_SUMMARY: false
-    #   with:
-    #     context: .
-    #     file: dev/build/Dockerfile
-    #     platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }}
-    #     push: true
-    #     tags: ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }}
-    #     cache-from: type=gha
-    #     cache-to: type=gha,mode=max
         
     - name: Update CHANGELOG
       id: changelog
@@ -396,6 +376,12 @@ jobs:
         toTag: ${{ env.TO_TAG }}
         writeToFile: false
 
+    - name: Download Coverage Results
+      if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
+      uses: actions/download-artifact@v4.1.8
+      with:
+        name: coverage
+
     - name: Prepare Coverage Action
       if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
       working-directory: ./dev/coverage-action

From 8b417470fd0697f95bb4e809b87b6f84a355c020 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 18 Oct 2024 01:07:21 -0400
Subject: [PATCH 118/656] ci: Update build.yml

---
 .github/workflows/build.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 48113ffb30..3a7779494a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -219,6 +219,10 @@ jobs:
           echo "=========================================================================="
           curl -fsSL https://get.docker.com -o get-docker.sh
           sudo sh get-docker.sh
+          sudo docker buildx create \
+            --name container-builder \
+            --driver docker-container \
+            --bootstrap --use
 
           echo "=========================================================================="
           echo "Login to ghcr.io..."

From d26bd5f4204c48743ca363adfd0d9036cc910d6a Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 18 Oct 2024 01:37:58 -0400
Subject: [PATCH 119/656] ci: Update build.yml

---
 .github/workflows/build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3a7779494a..104b29c3fe 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -336,6 +336,7 @@ jobs:
     - name: Fetch release tarball
       run: |
         mkdir -p /home/runner/work/release
+        chmod 0600 ${{ github.workspace }}/prvkey.key
         scp -i ${{ github.workspace }}/prvkey.key azureuser@${{ steps.azlaunch.outputs.ipaddr }}:/workspace/release.tar.gz /home/runner/work/release/release.tar.gz
 
     - name: Destroy Build VM + resources

From 9873439cdcc12e50a283e3db763b575bedd53adb Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 18 Oct 2024 04:06:34 -0400
Subject: [PATCH 120/656] ci: Update build.yml

---
 .github/workflows/build.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 104b29c3fe..756600cc1e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -170,9 +170,10 @@ jobs:
           --admin-username azureuser \
           --generate-ssh-keys \
           --priority Spot \
-          --size Standard_D8as_v5 \
+          --size Standard_D8ads_v5 \
           --max-price -1 \
-          --os-disk-size-gb 30 \
+          --ephemeral-os-disk \
+          --os-disk-size-gb 100 \
           --eviction-policy Delete \
           --nic-delete-option Delete \
           --output tsv \

From 888101005148703bc06df73265cd310ed7b1e178 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 21 Oct 2024 16:45:36 -0500
Subject: [PATCH 121/656] feat: explicit names for meetecho recordings (#8062)

* feat: explicit names for meetecho recordings

* fix: better regex and kebab-styled-endpoint-name

* fix: spaces around comparison operator
---
 ietf/api/tests.py                             | 66 ++++++++++++++++++-
 ietf/api/urls.py                              |  4 +-
 .../0009_session_meetecho_recording_name.py   | 20 ++++++
 ietf/meeting/models.py                        | 15 +++--
 ietf/meeting/tests_models.py                  |  6 +-
 ietf/meeting/views.py                         | 39 +++++++++++
 6 files changed, 143 insertions(+), 7 deletions(-)
 create mode 100644 ietf/meeting/migrations/0009_session_meetecho_recording_name.py

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 4f2a7f7d3c..97e928dc51 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2015-2020, All Rights Reserved
+# Copyright The IETF Trust 2015-2024, All Rights Reserved
 # -*- coding: utf-8 -*-
 import base64
 import datetime
@@ -222,6 +222,70 @@ def test_api_set_session_video_url(self):
         event = doc.latest_event()
         self.assertEqual(event.by, recman)
 
+    def test_api_set_meetecho_recording_name(self):
+        url = urlreverse("ietf.meeting.views.api_set_meetecho_recording_name")
+        recmanrole = RoleFactory(group__type_id="ietf", name_id="recman")
+        recman = recmanrole.person
+        meeting = MeetingFactory(type_id="ietf")
+        session = SessionFactory(group__type_id="wg", meeting=meeting)
+        apikey = PersonalApiKey.objects.create(endpoint=url, person=recman)
+        name = "testname"
+
+        # error cases
+        r = self.client.post(url, {})
+        self.assertContains(r, "Missing apikey parameter", status_code=400)
+
+        badrole = RoleFactory(group__type_id="ietf", name_id="ad")
+        badapikey = PersonalApiKey.objects.create(endpoint=url, person=badrole.person)
+        badrole.person.user.last_login = timezone.now()
+        badrole.person.user.save()
+        r = self.client.post(url, {"apikey": badapikey.hash()})
+        self.assertContains(r, "Restricted to role: Recording Manager", status_code=403)
+
+        r = self.client.post(url, {"apikey": apikey.hash()})
+        self.assertContains(r, "Too long since last regular login", status_code=400)
+        recman.user.last_login = timezone.now()
+        recman.user.save()
+
+        r = self.client.get(url, {"apikey": apikey.hash()})
+        self.assertContains(r, "Method not allowed", status_code=405)
+
+        r = self.client.post(url, {"apikey": apikey.hash()})
+        self.assertContains(r, "Missing session_id parameter", status_code=400)
+
+        r = self.client.post(url, {"apikey": apikey.hash(), "session_id": session.pk})
+        self.assertContains(r, "Missing name parameter", status_code=400)
+
+        bad_pk = int(Session.objects.order_by("-pk").first().pk) + 1
+        r = self.client.post(
+            url,
+            {
+                "apikey": apikey.hash(),
+                "session_id": bad_pk,
+                "name": name,
+            },
+        )
+        self.assertContains(r, "Session not found", status_code=400)
+
+        r = self.client.post(
+            url,
+            {
+                "apikey": apikey.hash(),
+                "session_id": "foo",
+                "name": name,
+            },
+        )
+        self.assertContains(r, "Invalid session_id", status_code=400)
+
+        r = self.client.post(
+            url, {"apikey": apikey.hash(), "session_id": session.pk, "name": name}
+        )
+        self.assertContains(r, "Done", status_code=200)
+
+        session.refresh_from_db()
+        self.assertEqual(session.meetecho_recording_name, name)
+
+
     def test_api_add_session_attendees_deprecated(self):
         # Deprecated test - should be removed when we stop accepting a simple list of user PKs in
         # the add_session_attendees() view
diff --git a/ietf/api/urls.py b/ietf/api/urls.py
index 48525dfda2..431ad5c5d4 100644
--- a/ietf/api/urls.py
+++ b/ietf/api/urls.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2017, All Rights Reserved
+# Copyright The IETF Trust 2017-2024, All Rights Reserved
 
 from django.conf import settings
 from django.urls import include
@@ -39,6 +39,8 @@
     url(r'^iesg/position', views_ballot.api_set_position),
     # Let Meetecho set session video URLs
     url(r'^meeting/session/video/url$', meeting_views.api_set_session_video_url),
+    # Let Meetecho tell us the name of its recordings
+    url(r'^meeting/session/recording-name$', meeting_views.api_set_meetecho_recording_name),
     # Meeting agenda + floorplan data
     url(r'^meeting/(?P<num>[A-Za-z0-9._+-]+)/agenda-data$', meeting_views.api_get_agenda_data),
     # Meeting session materials
diff --git a/ietf/meeting/migrations/0009_session_meetecho_recording_name.py b/ietf/meeting/migrations/0009_session_meetecho_recording_name.py
new file mode 100644
index 0000000000..79ca4919a3
--- /dev/null
+++ b/ietf/meeting/migrations/0009_session_meetecho_recording_name.py
@@ -0,0 +1,20 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("meeting", "0008_remove_schedtimesessassignment_notes"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="session",
+            name="meetecho_recording_name",
+            field=models.CharField(
+                blank=True, help_text="Name of the meetecho recording", max_length=64
+            ),
+        ),
+    ]
diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index e169077800..8c6fb97413 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -1042,6 +1042,7 @@ class Session(models.Model):
     on_agenda = models.BooleanField(default=True, help_text='Is this session visible on the meeting agenda?')
     has_onsite_tool = models.BooleanField(default=False, help_text="Does this session use the officially supported onsite and remote tooling?")
     chat_room = models.CharField(blank=True, max_length=32, help_text='Name of Zulip stream, if different from group acronym')
+    meetecho_recording_name = models.CharField(blank=True, max_length=64, help_text="Name of the meetecho recording")
 
     tombstone_for = models.ForeignKey('Session', blank=True, null=True, help_text="This session is the tombstone for a session that was rescheduled", on_delete=models.CASCADE)
 
@@ -1332,17 +1333,23 @@ def onsite_tool_url(self):
         return None
 
     def _session_recording_url_label(self):
+        otsa = self.official_timeslotassignment()
+        if otsa is None:
+            return None
         if self.meeting.type.slug == "ietf" and self.has_onsite_tool:
-            session_label = f"IETF{self.meeting.number}-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}"
+            session_label = f"IETF{self.meeting.number}-{self.group.acronym.upper()}-{otsa.timeslot.time.strftime('%Y%m%d-%H%M')}"
         else:
-            session_label = f"IETF-{self.group.acronym.upper()}-{self.official_timeslotassignment().timeslot.time.strftime('%Y%m%d-%H%M')}"
+            session_label = f"IETF-{self.group.acronym.upper()}-{otsa.timeslot.time.strftime('%Y%m%d-%H%M')}"
         return session_label
 
     def session_recording_url(self):
         url_formatter = getattr(settings, "MEETECHO_SESSION_RECORDING_URL", "")
         url = None
-        if url_formatter and self.video_stream_url:
-            url = url_formatter.format(session_label=self._session_recording_url_label())
+        name = self.meetecho_recording_name
+        if name is None or name.strip() == "":
+            name = self._session_recording_url_label()
+        if url_formatter.strip() != "" and name is not None:
+            url = url_formatter.format(session_label=name)
         return url
 
 
diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py
index 8457423c51..03b706e1d7 100644
--- a/ietf/meeting/tests_models.py
+++ b/ietf/meeting/tests_models.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2021, All Rights Reserved
+# Copyright The IETF Trust 2021-2024, All Rights Reserved
 # -*- coding: utf-8 -*-
 """Tests of models in the Meeting application"""
 import datetime
@@ -172,6 +172,10 @@ def test_session_recording_url(self, mock):
                 settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}"
                 self.assertEqual(session.session_recording_url(), "http://player.example.com?LABEL")
 
+                session.meetecho_recording_name="actualname"
+                session.save()
+                self.assertEqual(session.session_recording_url(), "http://player.example.com?actualname")
+
     def test_session_recording_url_label_ietf(self):
         session = SessionFactory(
             meeting__type_id='ietf',
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 6e6c4dfe23..bce6ccd1bf 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -4270,6 +4270,45 @@ class OldUploadRedirect(RedirectView):
     def get_redirect_url(self, **kwargs):
         return reverse_lazy('ietf.meeting.views.session_details',kwargs=self.kwargs)
 
+
+@require_api_key
+@role_required("Recording Manager")
+@csrf_exempt
+def api_set_meetecho_recording_name(request):
+    """Set name for meetecho recording
+
+    parameters:
+        apikey: the poster's personal API key
+        session_id: id of the session to update
+        name: the name to use for the recording at meetecho player
+    """
+    def err(code, text):
+        return HttpResponse(text, status=code, content_type='text/plain')
+
+    if request.method != "POST":
+        return HttpResponseNotAllowed(
+            content="Method not allowed", content_type="text/plain", permitted_methods=('POST',)
+        )
+
+    session_id = request.POST.get('session_id', None)
+    if session_id is None:
+        return err(400, 'Missing session_id parameter')
+    name = request.POST.get('name', None)
+    if name is None:
+        return err(400, 'Missing name parameter')
+
+    try:
+        session = Session.objects.get(pk=session_id)
+    except Session.DoesNotExist:
+        return err(400, f"Session not found with session_id '{session_id}'")
+    except ValueError:
+        return err(400, "Invalid session_id: {session_id}")
+
+    session.meetecho_recording_name = name
+    session.save()
+
+    return HttpResponse("Done", status=200, content_type='text/plain')
+
 @require_api_key
 @role_required('Recording Manager')
 @csrf_exempt

From 1e193377df7cca32656c6bb062cea0a5ac3680f3 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 21 Oct 2024 18:48:51 -0300
Subject: [PATCH 122/656] ci: replace nginx default.conf (#8063)

* ci: squash nginx default.conf

* ci: replace nginx default.conf for auth
---
 k8s/auth.yaml        | 2 +-
 k8s/datatracker.yaml | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/k8s/auth.yaml b/k8s/auth.yaml
index c92ed05163..2bdb064447 100644
--- a/k8s/auth.yaml
+++ b/k8s/auth.yaml
@@ -84,7 +84,7 @@ spec:
               mountPath: /etc/nginx/conf.d/00logging.conf
               subPath: nginx-logging.conf
             - name: dt-cfg
-              mountPath: /etc/nginx/conf.d/auth.conf
+              mountPath: /etc/nginx/conf.d/default.conf
               subPath: nginx-auth.conf
         # -----------------------------------------------------
         # ScoutAPM Container
diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml
index a8a9675687..3d9e86a29d 100644
--- a/k8s/datatracker.yaml
+++ b/k8s/datatracker.yaml
@@ -84,7 +84,8 @@ spec:
               mountPath: /etc/nginx/conf.d/00logging.conf
               subPath: nginx-logging.conf
             - name: dt-cfg
-              mountPath: /etc/nginx/conf.d/datatracker.conf
+              # Replaces the original default.conf
+              mountPath: /etc/nginx/conf.d/default.conf
               subPath: nginx-datatracker.conf
         # -----------------------------------------------------
         # ScoutAPM Container

From c26ba974baa50a8f95e0b112f7cab90d007f3302 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 22 Oct 2024 12:51:16 -0300
Subject: [PATCH 123/656] refactor: use form for search_submission (#8065)

* refactor: use form for search_submission

* chore: remove lint

* style: Black
---
 ietf/submit/forms.py                         | 10 +++
 ietf/submit/views.py                         | 93 ++++++++++++++------
 ietf/templates/submit/search_submission.html |  8 +-
 3 files changed, 77 insertions(+), 34 deletions(-)

diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py
index 2781d3365a..bed87b77c2 100644
--- a/ietf/submit/forms.py
+++ b/ietf/submit/forms.py
@@ -659,3 +659,13 @@ def __init__(self, *args, **kwargs):
         self.fields['frm'].label='From'
         self.fields['frm'].widget.attrs['readonly'] = True
         self.fields['reply_to'].widget.attrs['readonly'] = True
+
+
+class SubmissionSearchForm(forms.Form):
+    """Form used for search_submission"""
+
+    name = forms.CharField(max_length=255, required=True, label="I-D name")
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields["name"].widget.attrs["placeholder"] = "draft-..."
diff --git a/ietf/submit/views.py b/ietf/submit/views.py
index 3f745741e4..f710bdeb0c 100644
--- a/ietf/submit/views.py
+++ b/ietf/submit/views.py
@@ -26,17 +26,47 @@
 from ietf.ietfauth.utils import has_role, role_required
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.person.models import Email
-from ietf.submit.forms import (SubmissionAutoUploadForm, AuthorForm, SubmitterForm, EditSubmissionForm,
-                               PreapprovalForm, ReplacesForm, SubmissionManualUploadForm)
+from ietf.submit.forms import (
+    SubmissionAutoUploadForm,
+    AuthorForm,
+    SubmitterForm,
+    EditSubmissionForm,
+    PreapprovalForm,
+    ReplacesForm,
+    SubmissionManualUploadForm,
+    SubmissionSearchForm,
+)
 from ietf.submit.mail import send_full_url, send_manual_post_request
-from ietf.submit.models import (Submission, Preapproval, SubmissionExtResource,
-    DraftSubmissionStateName )
-from ietf.submit.tasks import process_uploaded_submission_task, process_and_accept_uploaded_submission_task, poke
-from ietf.submit.utils import ( approvable_submissions_for_user, preapprovals_for_user,
-    recently_approved_by_user, validate_submission, create_submission_event, docevent_from_submission,
-    post_submission, cancel_submission, rename_submission_files, remove_submission_files,
-    get_submission, save_files, clear_existing_files, accept_submission, accept_submission_requires_group_approval,
-    accept_submission_requires_prev_auth_approval, update_submission_external_resources)
+from ietf.submit.models import (
+    Submission,
+    Preapproval,
+    SubmissionExtResource,
+    DraftSubmissionStateName,
+)
+from ietf.submit.tasks import (
+    process_uploaded_submission_task,
+    process_and_accept_uploaded_submission_task,
+    poke,
+)
+from ietf.submit.utils import (
+    approvable_submissions_for_user,
+    preapprovals_for_user,
+    recently_approved_by_user,
+    validate_submission,
+    create_submission_event,
+    docevent_from_submission,
+    post_submission,
+    cancel_submission,
+    rename_submission_files,
+    remove_submission_files,
+    get_submission,
+    save_files,
+    clear_existing_files,
+    accept_submission,
+    accept_submission_requires_group_approval,
+    accept_submission_requires_prev_auth_approval,
+    update_submission_external_resources,
+)
 from ietf.stats.utils import clean_country_name
 from ietf.utils.accesstoken import generate_access_token
 from ietf.utils.log import log
@@ -195,24 +225,33 @@ def api_submit_tombstone(request):
 def tool_instructions(request):
     return render(request, 'submit/tool_instructions.html', {'selected': 'instructions'})
 
+
 def search_submission(request):
-    error = None
-    name = None
-    if request.method == 'POST':
-        name = request.POST.get('name', '')
-        submission = Submission.objects.filter(name=name).order_by('-pk').first()
-        if submission:
-            return redirect(submission_status, submission_id=submission.pk)
-        else:
-            if re.search(r'-\d\d$', name):
-                submission = Submission.objects.filter(name=name[:-3]).order_by('-pk').first()
-                if submission:
-                    return redirect(submission_status, submission_id=submission.pk)
-        error = 'No valid submission found for %s' % name
-    return render(request, 'submit/search_submission.html',
-                              {'selected': 'status',
-                               'error': error,
-                               'name': name})
+    if request.method == "POST":
+        form = SubmissionSearchForm(request.POST)
+        if form.is_valid():
+            name = form.cleaned_data["name"]
+            submission = Submission.objects.filter(name=name).order_by("-pk").first()
+            if submission:
+                return redirect(submission_status, submission_id=submission.pk)
+            else:
+                if re.search(r"-\d\d$", name):
+                    submission = (
+                        Submission.objects.filter(name=name[:-3])
+                        .order_by("-pk")
+                        .first()
+                    )
+                    if submission:
+                        return redirect(submission_status, submission_id=submission.pk)
+            form.add_error(None, f"No valid submission found for {name}")
+    else:
+        form = SubmissionSearchForm()
+    return render(
+        request,
+        "submit/search_submission.html",
+        {"selected": "status", "form": form},
+    )
+
 
 def can_edit_submission(user, submission, access_token):
     key_matched = access_token and submission.access_token() == access_token
diff --git a/ietf/templates/submit/search_submission.html b/ietf/templates/submit/search_submission.html
index 3f827abbea..807d6d7d86 100644
--- a/ietf/templates/submit/search_submission.html
+++ b/ietf/templates/submit/search_submission.html
@@ -12,14 +12,8 @@
     <form method="post">
         {% csrf_token %}
         <div class="mb-3">
-            <label class="form-label" for="name">I-D name</label>
-            <input type="text" class="form-control" id="name" name="name" placeholder="draft-..." required>
+            {% bootstrap_form form %}
         </div>
-        {% if error %}
-            <p class="alert alert-danger my-3">
-                {{ error }}
-            </p>
-        {% endif %}
         <button class="btn btn-primary" type="submit">See status</button>
     </form>
 {% endblock %}
\ No newline at end of file

From 2947e286f66880c17f26293cf1f02e83ab7e7e65 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 23 Oct 2024 04:51:11 -0400
Subject: [PATCH 124/656] docs: Update LICENSE

---
 LICENSE | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/LICENSE b/LICENSE
index c180fa3f9f..dc6e0c5663 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2008-2023, The IETF Trust
+Copyright (c) 2008-2024, The IETF Trust
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

From 3130ecd9f56f6b9b0de25b1e832e8d80c50a3c2d Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 25 Oct 2024 10:52:09 -0300
Subject: [PATCH 125/656] fix: add recording-name api key endpoint; appauth url
 fix (#8081)

* fix: add endpoint option for recording-name

* chore: migration

* test: validate PersonalApiKey when used in tests

* fix: limit /api/appauth URLs as intended

* test: fix tests

* chore: fix lint

* test: PersonalApiKey create -> factory

* chore: remove unused import
---
 ietf/api/tests.py                             | 87 ++++++++++---------
 ietf/api/urls.py                              |  2 +-
 ietf/api/views.py                             |  4 +-
 ietf/doc/tests_ballot.py                      |  6 +-
 ietf/ietfauth/tests.py                        | 13 +--
 ietf/meeting/tests_views.py                   |  6 +-
 ietf/person/factories.py                      | 16 +++-
 .../0003_alter_personalapikey_endpoint.py     | 42 +++++++++
 ietf/person/models.py                         |  1 +
 9 files changed, 118 insertions(+), 59 deletions(-)
 create mode 100644 ietf/person/migrations/0003_alter_personalapikey_endpoint.py

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 97e928dc51..7dc9c42cc8 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -33,9 +33,8 @@
 from ietf.meeting.models import Session
 from ietf.nomcom.models import Volunteer
 from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year
-from ietf.person.factories import PersonFactory, random_faker, EmailFactory
+from ietf.person.factories import PersonFactory, random_faker, EmailFactory, PersonalApiKeyFactory
 from ietf.person.models import Email, User
-from ietf.person.models import PersonalApiKey
 from ietf.stats.models import MeetingRegistration
 from ietf.utils.mail import empty_outbox, outbox, get_payload_text
 from ietf.utils.models import DumpInfo
@@ -71,7 +70,7 @@ def test_deprecated_api_set_session_video_url(self):
         meeting = MeetingFactory(type_id='ietf')
         session = SessionFactory(group__type_id='wg', meeting=meeting)
         group = session.group
-        apikey = PersonalApiKey.objects.create(endpoint=url, person=recman)
+        apikey = PersonalApiKeyFactory(endpoint=url, person=recman)
         video = 'https://foo.example.com/bar/beer/'
 
         # error cases
@@ -79,7 +78,7 @@ def test_deprecated_api_set_session_video_url(self):
         self.assertContains(r, "Missing apikey parameter", status_code=400)
 
         badrole  = RoleFactory(group__type_id='ietf', name_id='ad')
-        badapikey = PersonalApiKey.objects.create(endpoint=url, person=badrole.person)
+        badapikey = PersonalApiKeyFactory(endpoint=url, person=badrole.person)
         badrole.person.user.last_login = timezone.now()
         badrole.person.user.save()
         r = self.client.post(url, {'apikey': badapikey.hash()} )
@@ -151,7 +150,7 @@ def test_api_set_session_video_url(self):
         recman = recmanrole.person
         meeting = MeetingFactory(type_id="ietf")
         session = SessionFactory(group__type_id="wg", meeting=meeting)
-        apikey = PersonalApiKey.objects.create(endpoint=url, person=recman)
+        apikey = PersonalApiKeyFactory(endpoint=url, person=recman)
         video = "https://foo.example.com/bar/beer/"
 
         # error cases
@@ -159,7 +158,7 @@ def test_api_set_session_video_url(self):
         self.assertContains(r, "Missing apikey parameter", status_code=400)
 
         badrole = RoleFactory(group__type_id="ietf", name_id="ad")
-        badapikey = PersonalApiKey.objects.create(endpoint=url, person=badrole.person)
+        badapikey = PersonalApiKeyFactory(endpoint=url, person=badrole.person)
         badrole.person.user.last_login = timezone.now()
         badrole.person.user.save()
         r = self.client.post(url, {"apikey": badapikey.hash()})
@@ -228,7 +227,7 @@ def test_api_set_meetecho_recording_name(self):
         recman = recmanrole.person
         meeting = MeetingFactory(type_id="ietf")
         session = SessionFactory(group__type_id="wg", meeting=meeting)
-        apikey = PersonalApiKey.objects.create(endpoint=url, person=recman)
+        apikey = PersonalApiKeyFactory(endpoint=url, person=recman)
         name = "testname"
 
         # error cases
@@ -236,7 +235,7 @@ def test_api_set_meetecho_recording_name(self):
         self.assertContains(r, "Missing apikey parameter", status_code=400)
 
         badrole = RoleFactory(group__type_id="ietf", name_id="ad")
-        badapikey = PersonalApiKey.objects.create(endpoint=url, person=badrole.person)
+        badapikey = PersonalApiKeyFactory(endpoint=url, person=badrole.person)
         badrole.person.user.last_login = timezone.now()
         badrole.person.user.save()
         r = self.client.post(url, {"apikey": badapikey.hash()})
@@ -295,10 +294,10 @@ def test_api_add_session_attendees_deprecated(self):
         recman = recmanrole.person
         meeting = MeetingFactory(type_id='ietf')
         session = SessionFactory(group__type_id='wg', meeting=meeting)  
-        apikey = PersonalApiKey.objects.create(endpoint=url, person=recman)
+        apikey = PersonalApiKeyFactory(endpoint=url, person=recman)
 
         badrole  = RoleFactory(group__type_id='ietf', name_id='ad')
-        badapikey = PersonalApiKey.objects.create(endpoint=url, person=badrole.person)
+        badapikey = PersonalApiKeyFactory(endpoint=url, person=badrole.person)
         badrole.person.user.last_login = timezone.now()
         badrole.person.user.save()
 
@@ -361,10 +360,10 @@ def test_api_add_session_attendees(self):
         recman = recmanrole.person
         meeting = MeetingFactory(type_id="ietf")
         session = SessionFactory(group__type_id="wg", meeting=meeting)
-        apikey = PersonalApiKey.objects.create(endpoint=url, person=recman)
+        apikey = PersonalApiKeyFactory(endpoint=url, person=recman)
 
         badrole = RoleFactory(group__type_id="ietf", name_id="ad")
-        badapikey = PersonalApiKey.objects.create(endpoint=url, person=badrole.person)
+        badapikey = PersonalApiKeyFactory(endpoint=url, person=badrole.person)
         badrole.person.user.last_login = timezone.now()
         badrole.person.user.save()
 
@@ -517,8 +516,8 @@ def test_api_upload_polls_and_chatlog(self):
             ),
         ):
             url = urlreverse(f"ietf.meeting.views.api_upload_{type_id}")
-            apikey = PersonalApiKey.objects.create(endpoint=url, person=recmanrole.person)
-            badapikey = PersonalApiKey.objects.create(endpoint=url, person=badrole.person)
+            apikey = PersonalApiKeyFactory(endpoint=url, person=recmanrole.person)
+            badapikey = PersonalApiKeyFactory(endpoint=url, person=badrole.person)
 
             r = self.client.post(url, {})
             self.assertContains(r, "Missing apikey parameter", status_code=400)
@@ -562,7 +561,7 @@ def test_deprecated_api_upload_bluesheet(self):
         meeting = MeetingFactory(type_id='ietf')
         session = SessionFactory(group__type_id='wg', meeting=meeting)
         group = session.group
-        apikey = PersonalApiKey.objects.create(endpoint=url, person=recman)
+        apikey = PersonalApiKeyFactory(endpoint=url, person=recman)
 
         people = [
             {"name": "Andrea Andreotti", "affiliation": "Azienda"},
@@ -579,7 +578,7 @@ def test_deprecated_api_upload_bluesheet(self):
         self.assertContains(r, "Missing apikey parameter", status_code=400)
 
         badrole = RoleFactory(group__type_id='ietf', name_id='ad')
-        badapikey = PersonalApiKey.objects.create(endpoint=url, person=badrole.person)
+        badapikey = PersonalApiKeyFactory(endpoint=url, person=badrole.person)
         badrole.person.user.last_login = timezone.now()
         badrole.person.user.save()
         r = self.client.post(url, {'apikey': badapikey.hash()})
@@ -654,7 +653,7 @@ def test_api_upload_bluesheet(self):
         meeting = MeetingFactory(type_id="ietf")
         session = SessionFactory(group__type_id="wg", meeting=meeting)
         group = session.group
-        apikey = PersonalApiKey.objects.create(endpoint=url, person=recman)
+        apikey = PersonalApiKeyFactory(endpoint=url, person=recman)
 
         people = [
             {"name": "Andrea Andreotti", "affiliation": "Azienda"},
@@ -671,7 +670,7 @@ def test_api_upload_bluesheet(self):
         self.assertContains(r, "Missing apikey parameter", status_code=400)
 
         badrole = RoleFactory(group__type_id="ietf", name_id="ad")
-        badapikey = PersonalApiKey.objects.create(endpoint=url, person=badrole.person)
+        badapikey = PersonalApiKeyFactory(endpoint=url, person=badrole.person)
         badrole.person.user.last_login = timezone.now()
         badrole.person.user.save()
         r = self.client.post(url, {"apikey": badapikey.hash()})
@@ -781,14 +780,14 @@ def test_api_v2_person_export_view(self):
         url = urlreverse('ietf.api.views.ApiV2PersonExportView')
         robot = PersonFactory(user__is_staff=True)
         RoleFactory(name_id='robot', person=robot, email=robot.email(), group__acronym='secretariat')
-        apikey = PersonalApiKey.objects.create(endpoint=url, person=robot)
+        apikey = PersonalApiKeyFactory(endpoint=url, person=robot)
 
         # error cases
         r = self.client.post(url, {})
         self.assertContains(r, "Missing apikey parameter", status_code=400)
 
         badrole = RoleFactory(group__type_id='ietf', name_id='ad')
-        badapikey = PersonalApiKey.objects.create(endpoint=url, person=badrole.person)
+        badapikey = PersonalApiKeyFactory(endpoint=url, person=badrole.person)
         badrole.person.user.last_login = timezone.now()
         badrole.person.user.save()
         r = self.client.post(url, {'apikey': badapikey.hash()})
@@ -827,7 +826,7 @@ def test_api_new_meeting_registration(self):
         oidcp = PersonFactory(user__is_staff=True)
         # Make sure 'oidcp' has an acceptable role
         RoleFactory(name_id='robot', person=oidcp, email=oidcp.email(), group__acronym='secretariat')
-        key = PersonalApiKey.objects.create(person=oidcp, endpoint=url)
+        key = PersonalApiKeyFactory(person=oidcp, endpoint=url)
         reg['apikey'] = key.hash()
         #
         # Test valid POST
@@ -911,7 +910,7 @@ def test_api_new_meeting_registration_nomcom_volunteer(self):
         oidcp = PersonFactory(user__is_staff=True)
         # Make sure 'oidcp' has an acceptable role
         RoleFactory(name_id='robot', person=oidcp, email=oidcp.email(), group__acronym='secretariat')
-        key = PersonalApiKey.objects.create(person=oidcp, endpoint=url)
+        key = PersonalApiKeyFactory(person=oidcp, endpoint=url)
         reg['apikey'] = key.hash()
 
         # first test is_nomcom_volunteer False
@@ -945,28 +944,30 @@ def test_api_version(self):
 
 
     def test_api_appauth(self):
-        url = urlreverse('ietf.api.views.app_auth')
-        person = PersonFactory()
-        apikey = PersonalApiKey.objects.create(endpoint=url, person=person)
-
-        self.client.login(username=person.user.username,password=f'{person.user.username}+password')
-        self.client.logout()
-
-        # error cases
-        # missing apikey
-        r = self.client.post(url, {})
-        self.assertContains(r, 'Missing apikey parameter', status_code=400)
-
-        # invalid apikey
-        r = self.client.post(url, {'apikey': 'foobar'})
-        self.assertContains(r, 'Invalid apikey', status_code=403)
-
-        # working case
-        r = self.client.post(url, {'apikey': apikey.hash()})
-        self.assertEqual(r.status_code, 200)
-        jsondata = r.json()
-        self.assertEqual(jsondata['success'], True)
+        for app in ["authortools", "bibxml"]:
+            url = urlreverse('ietf.api.views.app_auth', kwargs={"app": app})
+            person = PersonFactory()
+            apikey = PersonalApiKeyFactory(endpoint=url, person=person)
     
+            self.client.login(username=person.user.username,password=f'{person.user.username}+password')
+            self.client.logout()
+    
+            # error cases
+            # missing apikey
+            r = self.client.post(url, {})
+            self.assertContains(r, 'Missing apikey parameter', status_code=400)
+    
+            # invalid apikey
+            r = self.client.post(url, {'apikey': 'foobar'})
+            self.assertContains(r, 'Invalid apikey', status_code=403)
+    
+            # working case
+            r = self.client.post(url, {'apikey': apikey.hash()})
+            self.assertEqual(r.status_code, 200)
+            jsondata = r.json()
+            self.assertEqual(jsondata['success'], True)
+            self.client.logout()
+
     def test_api_get_session_matherials_no_agenda_meeting_url(self):
         meeting = MeetingFactory(type_id='ietf')
         session = SessionFactory(meeting=meeting)
diff --git a/ietf/api/urls.py b/ietf/api/urls.py
index 431ad5c5d4..6846e32747 100644
--- a/ietf/api/urls.py
+++ b/ietf/api/urls.py
@@ -69,7 +69,7 @@
     # Datatracker version
     url(r'^version/?$', api_views.version),
     # Application authentication API key
-    url(r'^appauth/[authortools|bibxml]', api_views.app_auth),
+    url(r'^appauth/(?P<app>authortools|bibxml)$', api_views.app_auth),
     # latest versions
     url(r'^rfcdiff-latest-json/%(name)s(?:-%(rev)s)?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, api_views.rfcdiff_latest_json),
     url(r'^rfcdiff-latest-json/(?P<name>[Rr][Ff][Cc] [0-9]+?)(\.txt|\.html)?/?$', api_views.rfcdiff_latest_json),
diff --git a/ietf/api/views.py b/ietf/api/views.py
index f8662f9a0e..df67cffd56 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -30,7 +30,7 @@
 from tastypie.utils.mime import determine_format, build_content_type
 from textwrap import dedent
 from traceback import format_exception, extract_tb
-from typing import Iterable, Optional
+from typing import Iterable, Optional, Literal
 
 import ietf
 from ietf.api import _api_list
@@ -251,7 +251,7 @@ def version(request):
 
 @require_api_key
 @csrf_exempt
-def app_auth(request):
+def app_auth(request, app: Literal["authortools", "bibxml"]):
     return HttpResponse(
             json.dumps({'success': True}),
             content_type='application/json')
diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py
index f95ba8812b..c7362b58e2 100644
--- a/ietf/doc/tests_ballot.py
+++ b/ietf/doc/tests_ballot.py
@@ -27,8 +27,8 @@
 from ietf.ipr.factories import HolderIprDisclosureFactory
 from ietf.name.models import BallotPositionName
 from ietf.iesg.models import TelechatDate
-from ietf.person.models import Person, PersonalApiKey
-from ietf.person.factories import PersonFactory
+from ietf.person.models import Person
+from ietf.person.factories import PersonFactory, PersonalApiKeyFactory
 from ietf.person.utils import get_active_ads
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
@@ -111,7 +111,7 @@ def test_api_set_position(self):
         create_ballot_if_not_open(None, draft, ad, 'approve')
         ad.user.last_login = timezone.now()
         ad.user.save()
-        apikey = PersonalApiKey.objects.create(endpoint=url, person=ad)
+        apikey = PersonalApiKeyFactory(endpoint=url, person=ad)
 
         # vote
         events_before = draft.docevent_set.count()
diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py
index 722c1e8b6f..fed764e8bd 100644
--- a/ietf/ietfauth/tests.py
+++ b/ietf/ietfauth/tests.py
@@ -35,7 +35,7 @@
 from ietf.meeting.factories import MeetingFactory
 from ietf.nomcom.factories import NomComFactory
 from ietf.person.factories import PersonFactory, EmailFactory, UserFactory, PersonalApiKeyFactory
-from ietf.person.models import Person, Email, PersonalApiKey
+from ietf.person.models import Person, Email
 from ietf.person.tasks import send_apikey_usage_emails_task
 from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory
 from ietf.review.models import ReviewWish, UnavailablePeriod
@@ -788,9 +788,8 @@ def test_apikey_errors(self):
             self.assertContains(r, 'Invalid apikey', status_code=403)
 
             # invalid apikey (invalidated api key)
-            unauthorized_url = urlreverse('ietf.api.views.app_auth')
-            invalidated_apikey = PersonalApiKey.objects.create(
-                        endpoint=unauthorized_url, person=person, valid=False)
+            unauthorized_url = urlreverse('ietf.api.views.app_auth', kwargs={'app': 'authortools'})
+            invalidated_apikey = PersonalApiKeyFactory(endpoint=unauthorized_url, person=person, valid=False)
             r = self.client.post(unauthorized_url, {'apikey': invalidated_apikey.hash()})
             self.assertContains(r, 'Invalid apikey', status_code=403)
 
@@ -803,7 +802,11 @@ def test_apikey_errors(self):
             person.user.save()
 
             # endpoint mismatch
-            key2 = PersonalApiKey.objects.create(person=person, endpoint='/')
+            key2 = PersonalApiKeyFactory(
+                person=person,
+                endpoint='/',
+                validate_model=False,  # allow invalid endpoint
+            )
             r = self.client.post(key.endpoint, {'apikey':key2.hash(), 'dummy':'dummy',})
             self.assertContains(r, 'Apikey endpoint mismatch', status_code=400)
             key2.delete()
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 642edcb9b4..0d291f12bc 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -38,7 +38,7 @@
 from ietf.doc.models import Document, NewRevisionDocEvent
 from ietf.group.models import Group, Role, GroupFeatures
 from ietf.group.utils import can_manage_group
-from ietf.person.models import Person, PersonalApiKey
+from ietf.person.models import Person
 from ietf.meeting.helpers import can_approve_interim_request, can_request_interim_meeting, can_view_interim_request, preprocess_assignments_for_agenda
 from ietf.meeting.helpers import send_interim_approval_request, AgendaKeywordTagger
 from ietf.meeting.helpers import send_interim_meeting_cancellation_notice, send_interim_session_cancellation_notice
@@ -56,7 +56,7 @@
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent
 from ietf.utils.timezone import date_today, time_now
 
-from ietf.person.factories import PersonFactory
+from ietf.person.factories import PersonFactory, PersonalApiKeyFactory
 from ietf.group.factories import GroupFactory, GroupEventFactory, RoleFactory
 from ietf.meeting.factories import (SessionFactory, ScheduleFactory,
     SessionPresentationFactory, MeetingFactory, FloorPlanFactory,
@@ -8743,7 +8743,7 @@ def test_session_attendance(self):
         add_attendees_url = urlreverse('ietf.meeting.views.api_add_session_attendees')
         recmanrole = RoleFactory(group__type_id='ietf', name_id='recman', person__user__last_login=timezone.now())
         recman = recmanrole.person
-        apikey = PersonalApiKey.objects.create(endpoint=add_attendees_url, person=recman)
+        apikey = PersonalApiKeyFactory(endpoint=add_attendees_url, person=recman)
         attendees = [person.user.pk for person in persons]
         self.client.login(username='recman', password='recman+password')
         r = self.client.post(add_attendees_url, {'apikey':apikey.hash(), 'attended':f'{{"session_id":{session.pk},"attendees":{attendees}}}'})
diff --git a/ietf/person/factories.py b/ietf/person/factories.py
index 2012483c0d..45de554766 100644
--- a/ietf/person/factories.py
+++ b/ietf/person/factories.py
@@ -158,10 +158,22 @@ class Meta:
 
 class PersonalApiKeyFactory(factory.django.DjangoModelFactory):
     person = factory.SubFactory(PersonFactory)
-    endpoint = FuzzyChoice(PERSON_API_KEY_ENDPOINTS)
-
+    endpoint = FuzzyChoice(v for v, n in PERSON_API_KEY_ENDPOINTS)
+    
     class Meta:
         model = PersonalApiKey
+        skip_postgeneration_save = True
+
+    @factory.post_generation
+    def validate_model(obj, create, extracted, **kwargs):
+        """Validate the model after creation
+        
+        Passing validate_model=False will disable the validation.
+        """
+        do_clean =  True if extracted is None else extracted
+        if do_clean:
+            obj.full_clean()
+
 
 class PersonApiKeyEventFactory(factory.django.DjangoModelFactory):
     key = factory.SubFactory(PersonalApiKeyFactory)
diff --git a/ietf/person/migrations/0003_alter_personalapikey_endpoint.py b/ietf/person/migrations/0003_alter_personalapikey_endpoint.py
new file mode 100644
index 0000000000..202af4b101
--- /dev/null
+++ b/ietf/person/migrations/0003_alter_personalapikey_endpoint.py
@@ -0,0 +1,42 @@
+# Generated by Django 4.2.16 on 2024-10-24 21:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("person", "0002_alter_historicalperson_ascii_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="personalapikey",
+            name="endpoint",
+            field=models.CharField(
+                choices=[
+                    ("/api/appauth/authortools", "/api/appauth/authortools"),
+                    ("/api/appauth/bibxml", "/api/appauth/bibxml"),
+                    ("/api/iesg/position", "/api/iesg/position"),
+                    (
+                        "/api/meeting/session/recording-name",
+                        "/api/meeting/session/recording-name",
+                    ),
+                    (
+                        "/api/meeting/session/video/url",
+                        "/api/meeting/session/video/url",
+                    ),
+                    ("/api/notify/meeting/bluesheet", "/api/notify/meeting/bluesheet"),
+                    (
+                        "/api/notify/meeting/registration",
+                        "/api/notify/meeting/registration",
+                    ),
+                    ("/api/notify/session/attendees", "/api/notify/session/attendees"),
+                    ("/api/notify/session/chatlog", "/api/notify/session/chatlog"),
+                    ("/api/notify/session/polls", "/api/notify/session/polls"),
+                    ("/api/v2/person/person", "/api/v2/person/person"),
+                ],
+                max_length=128,
+            ),
+        ),
+    ]
diff --git a/ietf/person/models.py b/ietf/person/models.py
index 0c25152361..85989acfc1 100644
--- a/ietf/person/models.py
+++ b/ietf/person/models.py
@@ -376,6 +376,7 @@ def salt():
     ("/api/iesg/position", "/api/iesg/position", "Area Director"),
     ("/api/v2/person/person", "/api/v2/person/person", "Robot"),
     ("/api/meeting/session/video/url", "/api/meeting/session/video/url", "Recording Manager"),
+    ("/api/meeting/session/recording-name", "/api/meeting/session/recording-name", "Recording Manager"),
     ("/api/notify/meeting/registration", "/api/notify/meeting/registration", "Robot"),
     ("/api/notify/meeting/bluesheet", "/api/notify/meeting/bluesheet", "Recording Manager"),
     ("/api/notify/session/attendees", "/api/notify/session/attendees", "Recording Manager"),

From 6e33097f2269544c3b41064424552a5b2ed58ff8 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 25 Oct 2024 10:52:27 -0300
Subject: [PATCH 126/656] chore: remove pyzmail (#8078)

Unused since a00dfc0c2771661d23b7ea413c0f324149abfb2f
---
 pyzmail/__init__.py            |  21 -
 pyzmail/generate.py            | 529 ---------------------
 pyzmail/parse.py               | 817 ---------------------------------
 pyzmail/tests/__init__.py      |   0
 pyzmail/tests/test_both.py     |  99 ----
 pyzmail/tests/test_generate.py |  30 --
 pyzmail/tests/test_parse.py    | 290 ------------
 pyzmail/tests/test_send.py     |  77 ----
 pyzmail/tests/test_utils.py    |  24 -
 pyzmail/utils.py               | 155 -------
 pyzmail/version.py             |   1 -
 11 files changed, 2043 deletions(-)
 delete mode 100644 pyzmail/__init__.py
 delete mode 100644 pyzmail/generate.py
 delete mode 100644 pyzmail/parse.py
 delete mode 100644 pyzmail/tests/__init__.py
 delete mode 100644 pyzmail/tests/test_both.py
 delete mode 100644 pyzmail/tests/test_generate.py
 delete mode 100644 pyzmail/tests/test_parse.py
 delete mode 100644 pyzmail/tests/test_send.py
 delete mode 100644 pyzmail/tests/test_utils.py
 delete mode 100644 pyzmail/utils.py
 delete mode 100644 pyzmail/version.py

diff --git a/pyzmail/__init__.py b/pyzmail/__init__.py
deleted file mode 100644
index f6c8854abc..0000000000
--- a/pyzmail/__init__.py
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# pyzmail/__init__.py
-# (c) Alain Spineux <alain.spineux@gmail.com>
-# http://www.magiksys.net/pyzmail
-# Released under LGPL
-
-from . import utils
-from .generate import compose_mail, send_mail, send_mail2
-from .parse import email_address_re, PyzMessage, PzMessage, decode_text
-from .parse import message_from_string, message_from_file
-from .parse import message_from_bytes, message_from_binary_file # python >= 3.2
-from .version import __version__
-
-# to help epydoc to display functions available from top of the package
-__all__= [ 'compose_mail', 'send_mail', 'send_mail2', 'email_address_re', \
-           'PyzMessage', 'PzMessage', 'decode_text', '__version__', 
-           'utils', 'generate', 'parse', 'version',
-           'message_from_string','message_from_file',
-           'message_from_binary_file', 'message_from_bytes', # python >= 3.2
-           ]
-
diff --git a/pyzmail/generate.py b/pyzmail/generate.py
deleted file mode 100644
index 2ad1f62246..0000000000
--- a/pyzmail/generate.py
+++ /dev/null
@@ -1,529 +0,0 @@
-#
-# pyzmail/generate.py
-# (c) Alain Spineux <alain.spineux@gmail.com>
-# http://www.magiksys.net/pyzmail
-# Released under LGPL
-
-"""
-Useful functions to compose and send emails.
-
-For short:
-
->>> payload, mail_from, rcpt_to, msg_id=compose_mail((u'Me', 'me@foo.com'),
-... [(u'Him', 'him@bar.com')], u'the subject', 'iso-8859-1', ('Hello world', 'us-ascii'),
-... attachments=[('attached', 'text', 'plain', 'text.txt', 'us-ascii')])
-... #doctest: +SKIP
->>> error=send_mail(payload, mail_from, rcpt_to, 'localhost', smtp_port=25)
-... #doctest: +SKIP
-"""
-
-import os, sys
-import time
-import base64
-import smtplib, socket
-import email
-import email.encoders
-import email.header
-import email.utils
-import email.mime
-import email.mime.base
-import email.mime.text
-import email.mime.image
-import email.mime.multipart
-
-from . import utils
-
-def format_addresses(addresses, header_name=None, charset=None):
-    """
-    Convert a list of addresses into a MIME-compliant header for a From, To, Cc,
-    or any other I{address} related field. 
-    This mixes the use of email.utils.formataddr() and email.header.Header().
-    
-    @type addresses: list
-    @param addresses: list of addresses, can be a mix of string a tuple  of the form 
-        C{[ 'address@domain', (u'Name', 'name@domain'), ...]}. 
-        If C{u'Name'} contains non us-ascii characters, it must be a 
-        unicode string or encoded using the I{charset} argument.
-    @type header_name: string or None
-    @keyword header_name: the name of the header. Its length is used to limit 
-        the length of the first line of the header according the RFC's 
-        requirements. (not very important, but it's better to match the 
-        requirements when possible)
-    @type charset: str
-    @keyword charset: the encoding charset for non unicode I{name} and a B{hint}
-        for encoding of unicode string. In other words,
-        if the I{name} of an address in a byte string containing non 
-        I{us-ascii} characters, then C{name.decode(charset)}
-        must generate the expected result. If a unicode string
-        is used instead, charset will be tried to encode the
-        string, if it fail, I{utf-8} will be used.
-        With B{Python 3.x} I{charset} is no more a hint and an exception will
-        be raised instead of using I{utf-8} has a fall back.
-    @rtype: str    
-    @return: the encoded list of formated addresses separated by commas, 
-    ready to use as I{Header} value.
-
-    >>> print format_addresses([('John', 'john@foo.com') ], 'From', 'us-ascii').encode()
-    John <john@foo.com>
-    >>> print format_addresses([(u'l\\xe9o', 'leo@foo.com') ], 'To', 'iso-8859-1').encode()
-    =?iso-8859-1?q?l=E9o?= <leo@foo.com>
-    >>> print format_addresses([(u'l\\xe9o', 'leo@foo.com') ], 'To', 'us-ascii').encode() 
-    ... # don't work in 3.X because charset is more than a hint 
-    ... #doctest: +SKIP
-    =?utf-8?q?l=C3=A9o?= <leo@foo.com>
-    >>> # because u'l\xe9o' cannot be encoded into us-ascii, utf8 is used instead
-    >>> print format_addresses([('No\\xe9', 'noe@f.com'), (u'M\\u0101ori', 'maori@b.com')  ], 'Cc', 'iso-8859-1').encode()
-    ... # don't work in 3.X because charset is more than a hint 
-    ... #doctest: +SKIP
-    =?iso-8859-1?q?No=E9?= <noe@f.com> , =?utf-8?b?TcSBb3Jp?= <maori@b.com>
-    >>> # 'No\xe9' is already encoded into iso-8859-1, but u'M\\u0101ori' cannot be encoded into iso-8859-1 
-    >>> # then utf8 is used here
-    >>> print format_addresses(['a@bar.com', ('John', 'john@foo.com') ], 'From', 'us-ascii').encode()
-    a@bar.com , John <john@foo.com>
-    """
-    header=email.header.Header(charset=charset, header_name=header_name)
-    for i, address in enumerate(addresses):
-        if i!=0:
-            # add separator between addresses
-            header.append(',', charset='us-ascii') 
-
-        try:
-            name, addr=address
-        except ValueError:
-            # address is not a tuple, their is no name, only email address
-            header.append(address, charset='us-ascii') 
-        else:
-            # check if address name is a unicode or byte string in "pure" us-ascii
-            if utils.is_usascii(name):
-                # name is a us-ascii byte string, i can use formataddr
-                formated_addr=email.utils.formataddr((name, addr))
-                # us-ascii must be used and not default 'charset'  
-                header.append(formated_addr, charset='us-ascii')
-            else:
-                # this is not as "pure" us-ascii string
-                # Header will use "RFC2047" to encode the address name
-                # if name is byte string, charset will be used to decode it first
-                header.append(name)
-                # here us-ascii must be used and not default 'charset'  
-                header.append('<%s>' % (addr,), charset='us-ascii') 
-                    
-    return header
-
-
-def build_mail(text, html=None, attachments=None, embeddeds=None):
-    """
-    Generate the core of the email message regarding the parameters.
-    The structure of the MIME email may vary, but the general one is as follow::
-
-        multipart/mixed (only if attachments are included)
-         |
-         +-- multipart/related (only if embedded contents are included) 
-         |    |
-         |    +-- multipart/alternative (only if text AND html are available)
-         |    |    |
-         |    |    +-- text/plain (text version of the message)
-         |    |    +-- text/html  (html version of the message)
-         |    |     
-         |    +-- image/gif  (where to include embedded contents)
-         |
-         +-- application/msword (where to add attachments)
-
-    @param text: the text version of the message, under the form of a tuple:
-        C{(encoded_content, encoding)} where I{encoded_content} is a byte string
-        encoded using I{encoding}.
-        I{text} can be None if the message has no text version. 
-    @type text: tuple or None
-    @keyword html: the HTML version of the message, under the form of a tuple:
-        C{(encoded_content, encoding)} where I{encoded_content} is a byte string
-        encoded using I{encoding}
-        I{html} can be None if the message has no HTML version. 
-    @type html: tuple or None
-    @keyword attachments: the list of attachments to include into the mail, in the
-        form [(data, maintype, subtype, filename, charset), ..] where :
-            - I{data} : is the raw data, or a I{charset} encoded string for 'text'
-            content. 
-            - I{maintype} : is a MIME main type like : 'text', 'image', 'application' ....
-            - I{subtype} : is a MIME sub type of the above I{maintype} for example :
-            'plain', 'png', 'msword' for respectively 'text/plain', 'image/png',
-            'application/msword'.
-            - I{filename} this is the filename of the attachment, it must be a
-            'us-ascii' string or a tuple of the form
-            C{(encoding, language, encoded_filename)} 
-            following the RFC2231 requirement, for example 
-            C{('iso-8859-1', 'fr', u'r\\xe9pertoir.png'.encode('iso-8859-1'))}
-            - I{charset} : if I{maintype} is 'text', then I{data} must be encoded
-            using this I{charset}. It can be None for non 'text' content.
-    @type attachments: list
-    @keyword embeddeds: is a list of documents embedded inside the HTML or text 
-        version of the message. It is similar to the I{attachments} list,  
-        but I{filename} is replaced by I{content_id} that is related to
-        the B{cid} reference into the HTML or text version of the message.
-    @type embeddeds: list
-    @rtype: inherit from email.Message 
-    @return: the message in a MIME object 
-
-    >>> mail=build_mail(('Hello world', 'us-ascii'), attachments=[('attached', 'text', 'plain', 'text.txt', 'us-ascii')])
-    >>> mail.set_boundary('===limit1==')
-    >>> print mail.as_string(unixfrom=False)    
-    Content-Type: multipart/mixed; boundary="===limit1=="
-    MIME-Version: 1.0
-    <BLANKLINE>
-    --===limit1==
-    Content-Type: text/plain; charset="us-ascii"
-    MIME-Version: 1.0
-    Content-Transfer-Encoding: 7bit
-    <BLANKLINE>
-    Hello world
-    --===limit1==
-    Content-Type: text/plain; charset="us-ascii"
-    MIME-Version: 1.0
-    Content-Transfer-Encoding: 7bit
-    Content-Disposition: attachment; filename="text.txt"
-    <BLANKLINE>
-    attached
-    --===limit1==--
-    """
-
-    if attachments is None:
-        attachments = []
-    if embeddeds is None:
-        embeddeds = []
-
-    main=text_part=html_part=None
-    if text:
-        content, charset=text
-        main=text_part=email.mime.text.MIMEText(content, 'plain', charset)
-    
-    if html:
-        content, charset=html
-        main=html_part=email.mime.text.MIMEText(content, 'html', charset)
-        
-    if not text_part and not html_part:
-        main=text_part=email.mime.text.MIMEText('', 'plain', 'us-ascii')
-    elif text_part and html_part:
-        # need to create a multipart/alternative to include text and html version
-        main=email.mime.multipart.MIMEMultipart('alternative', None, [text_part, html_part])
-    
-    if embeddeds:
-        related=email.mime.multipart.MIMEMultipart('related')
-        related.attach(main)
-        for part in embeddeds:
-            if not isinstance(part, email.mime.base.MIMEBase):
-                data, maintype, subtype, content_id, charset=part
-                if (maintype=='text'):
-                    part=email.mime.text.MIMEText(data, subtype, charset)
-                else:
-                    part=email.mime.base.MIMEBase(maintype, subtype)
-                    part.set_payload(data)
-                    email.encoders.encode_base64(part)
-                part.add_header('Content-ID', '<'+content_id+'>')
-                part.add_header('Content-Disposition', 'inline')
-            related.attach(part)
-        main=related
-            
-    if attachments:
-        mixed=email.mime.multipart.MIMEMultipart('mixed')
-        mixed.attach(main)
-        for part in attachments:
-            if not isinstance(part, email.mime.base.MIMEBase):
-                data, maintype, subtype, filename, charset=part
-                if (maintype=='text'):
-                    part=email.mime.text.MIMEText(data, subtype, charset)
-                else:
-                    part=email.mime.base.MIMEBase(maintype, subtype)
-                    part.set_payload(data)
-                    email.encoders.encode_base64(part)
-                part.add_header('Content-Disposition', 'attachment', filename=filename)
-            mixed.attach(part)
-        main=mixed
-        
-    return main
-
-def complete_mail(message, sender, recipients, subject, default_charset, cc=None, bcc=None, message_id_string=None, date=None, headers=None):
-    """
-    Fill in the From, To, Cc, Subject, Date and Message-Id I{headers} of 
-    one existing message regarding the parameters.
-
-    @type message:email.Message 
-    @param message: the message to fill in
-    @type sender: tuple
-    @param sender: a tuple of the form (u'Sender Name', 'sender.address@domain.com')
-    @type recipients: list
-    @param recipients: a list of addresses. Address can be tuple or string like
-    expected by L{format_addresses()}, for example: C{[ 'address@dmain.com', 
-    (u'Recipient Name', 'recipient.address@domain.com'), ... ]}
-    @type subject: str
-    @param subject: The subject of the message, can be a unicode string or a
-    string encoded using I{default_charset} encoding. Prefert unicode to
-    byte string here.
-    @type default_charset: str  
-    @param default_charset: The default charset for this email. Arguments
-    that are non unicode string are supposed to be encoded using this charset. 
-    This I{charset} will be used has an hint when encoding mail content. 
-    @type cc: list
-    @keyword cc: The I{carbone copy} addresses. Same format as the I{recipients}
-    argument.
-    @type bcc: list
-    @keyword bcc: The I{blind carbone copy} addresses. Same format as the I{recipients}
-    argument.
-    @type message_id_string: str or None
-    @keyword message_id_string: if None, don't append any I{Message-ID} to the 
-    mail, let the SMTP do the job, else use the string to generate a unique 
-    I{ID} using C{email.utils.make_msgid()}. The generated value is 
-    returned as last argument. For example use the name of your application.
-    @type date: int or None
-    @keyword date: utc time in second from the epoch or None. If None then
-    use curent time C{time.time()} instead.
-    @type headers: list of tuple
-    @keyword headers: a list of C{(field, value)} tuples to fill in the mail 
-    header fields. Values are encoded using I{default_charset}.
-    @rtype: tuple
-    @return: B{(payload, mail_from, rcpt_to, msg_id)}
-        - I{payload} (str) is the content of the email, generated from the message
-        - I{mail_from} (str) is the address of the sender to pass to the SMTP host
-        - I{rcpt_to} (list) is a list of the recipients addresses to pass to the SMTP host
-        of the form C{[ 'a@b.com', c@d.com', ]}. This combine all recipients, 
-        I{carbone copy} addresses and I{blind carbone copy} addresses.
-        - I{msg_id} (None or str) None if message_id_string==None else the generated value for 
-        the message-id. If not None, this I{Message-ID} is already written 
-        into the payload.
-        
-    >>> import email.mime.text
-    >>> msg=email.mime.text.MIMEText('The text.', 'plain', 'us-ascii')
-    >>> # I could use build_mail() instead
-    >>> payload, mail_from, rcpt_to, msg_id=complete_mail(msg, ('Me', 'me@foo.com'),
-    ... [ ('Him', 'him@bar.com'), ], 'Non unicode subject', 'iso-8859-1',
-    ... cc=['her@bar.com',], date=1313558269, headers=[('User-Agent', u'pyzmail'), ])
-    >>> print payload
-    ... # 3.X encode  User-Agent: using 'iso-8859-1' even if it contains only us-asccii
-    ... # doctest: +ELLIPSIS  
-    Content-Type: text/plain; charset="us-ascii"
-    MIME-Version: 1.0
-    Content-Transfer-Encoding: 7bit
-    From: Me <me@foo.com>
-    To: Him <him@bar.com>
-    Cc: her@bar.com
-    Subject: =?iso-8859-1?q?Non_unicode_subject?=
-    Date: ...
-    User-Agent: ...pyzmail...
-    <BLANKLINE>
-    The text.
-    >>> print 'mail_from=%r rcpt_to=%r' % (mail_from, rcpt_to)
-    mail_from='me@foo.com' rcpt_to=['him@bar.com', 'her@bar.com']
-    """
-    def getaddr(address):
-        if isinstance(address, tuple):
-            return address[1]
-        else:
-            return address
-
-    if cc is None:
-        cc=[]
-    if bcc is None:
-        bcc=[]
-    if headers is None:
-        headers=[]
-
-    mail_from=getaddr(sender[1])
-    rcpt_to=list(map(getaddr, recipients))
-    rcpt_to.extend(list(map(getaddr, cc)))
-    rcpt_to.extend(list(map(getaddr, bcc)))
-
-    message['From'] = format_addresses([ sender, ], header_name='from', charset=default_charset)
-    if recipients:
-        message['To'] = format_addresses(recipients, header_name='to', charset=default_charset)
-    if cc:
-        message['Cc'] = format_addresses(cc, header_name='cc', charset=default_charset)
-    message['Subject'] = email.header.Header(subject, default_charset)
-    if date:
-        utc_from_epoch=date
-    else:
-        utc_from_epoch=time.time()
-    message['Date'] = email.utils.formatdate(utc_from_epoch, localtime=True)
-    
-    if message_id_string:
-        msg_id=message['Message-Id']=email.utils.make_msgid(message_id_string)
-    else:
-        msg_id=None
-        
-    for field, value in headers:
-        message[field]=email.header.Header(value, default_charset)
-
-    payload=message.as_string()
-    
-    return payload, mail_from, rcpt_to, msg_id
-
-def compose_mail(sender, recipients, subject, default_charset, text, html=None, attachments=None, embeddeds=None, cc=None, bcc=None, message_id_string=None, date=None, headers=None):
-    """
-    Compose an email regarding the arguments. Call L{build_mail()} and 
-    L{complete_mail()} at once.
-    
-    Read the B{parameters} descriptions of both functions L{build_mail()} and L{complete_mail()}. 
-    
-    Returned value is the same as for L{build_mail()} and L{complete_mail()}.
-    You can pass the returned values to L{send_mail()} or L{send_mail2()}.
-
-    @rtype: tuple
-    @return: B{(payload, mail_from, rcpt_to, msg_id)}
-
-    >>> payload, mail_from, rcpt_to, msg_id=compose_mail((u'Me', 'me@foo.com'), [(u'Him', 'him@bar.com')], u'the subject', 'iso-8859-1', ('Hello world', 'us-ascii'), attachments=[('attached', 'text', 'plain', 'text.txt', 'us-ascii')])
-    """
-    if attachments is None:
-        attachments=[]
-    if embeddeds is None:
-        embeddeds=[]
-    if cc is None:
-        cc=[]
-    if bcc is None:
-        bcc = []
-    if headers is None:
-        headers=[]
-
-    message=build_mail(text, html, attachments, embeddeds)
-    return complete_mail(message, sender, recipients, subject, default_charset, cc, bcc, message_id_string, date, headers)
-        
-
-def send_mail2(payload, mail_from, rcpt_to, smtp_host, smtp_port=25, smtp_mode='normal', smtp_login=None, smtp_password=None):
-    """
-    Send the message to a SMTP host. Look at the L{send_mail()} documentation.
-    L{send_mail()} call this function and catch all exceptions to convert them
-    into a user friendly error message. The returned value 
-    is always a dictionary. It can be empty if all recipients have been
-    accepted. 
-
-    @rtype: dict
-    @return: This function return the value returnd by C{smtplib.SMTP.sendmail()}
-    or raise the same exceptions.  
-    
-    This method will return normally if the mail is accepted for at least one 
-    recipient. Otherwise it will raise an exception. That is, if this 
-    method does not raise an exception, then someone should get your mail. 
-    If this method does not raise an exception, it returns a dictionary, 
-    with one entry for each recipient that was refused. Each entry contains a
-    tuple of the SMTP error code and the accompanying error message sent by the server.
-    
-    @raise smtplib.SMTPException: Look at the standard C{smtplib.SMTP.sendmail()} documentation.
-    
-    """
-    if smtp_mode=='ssl':
-        smtp=smtplib.SMTP_SSL(smtp_host, smtp_port)
-    else:
-        smtp=smtplib.SMTP(smtp_host, smtp_port)
-        if smtp_mode=='tls':
-            smtp.starttls()
-            
-    if smtp_login and smtp_password:
-        if sys.version_info<(3, 0):
-            # python 2.x  
-            # login and password must be encoded 
-            # because HMAC used in CRAM_MD5 require non unicode string
-            smtp.login(smtp_login.encode('utf-8'), smtp_password.encode('utf-8'))
-        else:
-            #python 3.x
-            smtp.login(smtp_login, smtp_password)
-    try:
-        ret=smtp.sendmail(mail_from, rcpt_to, payload)
-    finally:
-        try:
-            smtp.quit()
-        except Exception as e:
-            pass
-            
-    return ret
-
-def send_mail(payload, mail_from, rcpt_to, smtp_host, smtp_port=25, smtp_mode='normal', smtp_login=None, smtp_password=None):
-    """
-    Send the message to a SMTP host. Handle SSL, TLS and authentication.
-    I{payload}, I{mail_from} and I{rcpt_to} can come from values returned by
-    L{complete_mail()}. This function call L{send_mail2()} but catch all
-    exceptions and return friendly error message instead. 
-     
-    @type payload: str
-    @param payload: the mail content.
-    @type mail_from: str
-    @param mail_from: the sender address, for example: C{'me@domain.com'}.
-    @type rcpt_to: list
-    @param rcpt_to: The list of the recipient addresses in the form 
-    C{[ 'a@b.com', c@d.com', ]}. No names here, only email addresses.
-    @type smtp_host: str
-    @param smtp_host: the IP address or the name of the SMTP host.
-    @type smtp_port: int
-    @keyword smtp_port: the port to connect to on the SMTP host. Default is C{25}.
-    @type smtp_mode: str
-    @keyword smtp_mode: the way to connect to the SMTP host, can be:  
-                      C{'normal'}, C{'ssl'} or C{'tls'}. default is C{'normal'}
-    @type smtp_login: str or None
-    @keyword smtp_login: If authentication is required, this is the login. 
-                       Be carefull to I{UTF8} encode your login if it contains
-                       non I{us-ascii} characters.
-    @type smtp_password: str or None
-    @keyword smtp_password: If authentication is required, this is the password.
-                          Be carefull to I{UTF8} encode your password if it 
-                          contains non I{us-ascii} characters.
-
-    @rtype: dict or str
-    @return: This function return a dictionary of failed recipients 
-    or a string with an error message. 
-    
-    If all recipients have been accepted the dictionary is empty. If the
-    returned value is a string, none of the recipients will get the message.
-    
-    The dictionary is exactly of the same sort as 
-    smtplib.SMTP.sendmail() returns with one entry for each recipient that 
-    was refused. Each entry contains a tuple of the SMTP error code and 
-    the accompanying error message sent by the server. 
-    
-    Example:
-    
-    >>> send_mail('Subject: hello\\n\\nmessage', 'a@foo.com', [ 'b@bar.com', ], 'localhost') #doctest: +SKIP
-    {}
-    
-    Here is how to use the returned value::
-        if isinstance(ret, dict):
-          if ret:
-            print 'failed' recipients:
-            for recipient, (code, msg) in ret.iteritems():
-                print 'code=%d recipient=%s\terror=%s' % (code, recipient, msg)
-          else:
-            print 'success'
-        else:
-          print 'Error:', ret
-
-    To use your GMail account to send your mail::
-        smtp_host='smtp.gmail.com'
-        smtp_port=587
-        smtp_mode='tls'
-        smtp_login='your.gmail.addresse@gmail.com'
-        smtp_password='your.gmail.password'
-        
-    Use your GMail address for the sender ! 
-
-    """
-
-    error=dict()
-    try:
-        ret=send_mail2(payload, mail_from, rcpt_to, smtp_host, smtp_port, smtp_mode, smtp_login, smtp_password)
-    except (socket.error, ) as e:
-        error='server %s:%s not responding: %s' % (smtp_host, smtp_port, e)
-    except smtplib.SMTPAuthenticationError as e:
-        error='authentication error: %s' % (e, )
-    except smtplib.SMTPRecipientsRefused as e:
-        # code, error=e.recipients[recipient_addr]
-        error='all recipients refused: '+', '.join(list(e.recipients.keys()))
-    except smtplib.SMTPSenderRefused as e:
-        # e.sender, e.smtp_code, e.smtp_error
-        error='sender refused: %s' % (e.sender, )
-    except smtplib.SMTPDataError as e:
-        error='SMTP protocol mismatch: %s' % (e, )
-    except smtplib.SMTPHeloError as e:
-        error="server didn't reply properly to the HELO greeting: %s" % (e, )
-    except smtplib.SMTPException as e:
-        error='SMTP error: %s' % (e, )
-#    except Exception, e:
-#        raise # unknown error
-    else:
-        # failed addresses and error messages
-        error=ret
-        
-    return error
-
diff --git a/pyzmail/parse.py b/pyzmail/parse.py
deleted file mode 100644
index 9f86c37177..0000000000
--- a/pyzmail/parse.py
+++ /dev/null
@@ -1,817 +0,0 @@
-#
-# pyzmail/parse.py
-# (c) Alain Spineux <alain.spineux@gmail.com>
-# http://www.magiksys.net/pyzmail
-# Released under LGPL
-
-"""
-Useful functions to parse emails
-
-@var email_address_re: a regex that match well formed email address (from perlfaq9)
-@undocumented: atom_rfc2822
-@undocumented: atom_posfix_restricted
-@undocumented: atom
-@undocumented: dot_atom
-@undocumented: local
-@undocumented: domain_lit
-@undocumented: domain
-@undocumented: addr_spec
-"""
-
-import re
-import io
-import email
-import email.errors
-import email.header
-import email.message
-import mimetypes
-
-from .utils import *
-
-# email address REGEX matching the RFC 2822 spec from perlfaq9
-#    my $atom       = qr{[a-zA-Z0-9_!#\$\%&'*+/=?\^`{}~|\-]+};
-#    my $dot_atom   = qr{$atom(?:\.$atom)*};
-#    my $quoted     = qr{"(?:\\[^\r\n]|[^\\"])*"};
-#    my $local      = qr{(?:$dot_atom|$quoted)};
-#    my $domain_lit = qr{\[(?:\\\S|[\x21-\x5a\x5e-\x7e])*\]};
-#    my $domain     = qr{(?:$dot_atom|$domain_lit)};
-#    my $addr_spec  = qr{$local\@$domain};
-# 
-# Python's translation
-atom_rfc2822=r"[a-zA-Z0-9_!#\$\%&'*+/=?\^`{}~|\-]+"
-atom_posfix_restricted=r"[a-zA-Z0-9_#\$&'*+/=?\^`{}~|\-]+" # without '!' and '%'
-atom=atom_rfc2822
-dot_atom=atom  +  r"(?:\."  +  atom  +  ")*"
-quoted=r'"(?:\\[^\r\n]|[^\\"])*"'
-local="(?:"  +  dot_atom  +  "|"  +  quoted  +  ")"
-domain_lit=r"\[(?:\\\S|[\x21-\x5a\x5e-\x7e])*\]"
-domain="(?:"  +  dot_atom  +  "|"  +  domain_lit  +  ")"
-addr_spec=local  +  "@"  +  domain
-# and the result
-email_address_re=re.compile('^'+addr_spec+'$')
-
-class MailPart:
-    """
-    Data related to a mail part (aka message content, attachment or  
-    embedded content in an email)
-    
-    @type charset: str or None
-    @ivar charset: the encoding of the I{get_payload()} content if I{type} is 'text/*'
-    and charset has been specified in the message
-    @type content_id: str or None
-    @ivar content_id: the MIME Content-ID if specified in the message.
-    @type description: str or None
-    @ivar description: the MIME Content-Description if specified in the message.
-    @type disposition: str or None
-    @ivar disposition: C{None}, C{'inline'} or C{'attachment'} depending
-    the MIME Content-Disposition value
-    @type filename: unicode or None
-    @ivar filename: the name of the file, if specified in the message. 
-    @type part: inherit from email.mime.base.MIMEBase
-    @ivar part: the related part inside the message.
-    @type is_body: str or None
-    @ivar is_body: None if this part is not the mail content itself (an 
-        attachment or embedded content), C{'text/plain'} if this part is the 
-        text content or C{'text/html'} if this part is the HTML version. 
-    @type sanitized_filename: str or None
-    @ivar sanitized_filename: This field is filled by L{PyzMessage} to store 
-    a valid unique filename related or not with the original filename.
-    @type type: str
-    @ivar type: the MIME type, like 'text/plain', 'image/png', 'application/msword' ...
-    """
-            
-    def __init__(self, part, filename=None, type=None, charset=None, content_id=None, description=None, disposition=None, sanitized_filename=None, is_body=None):
-        """
-        Create an mail part and initialize all attributes
-        """
-        self.part=part          # original python part
-        self.filename=filename  # filename in unicode (if any) 
-        self.type=type          # the mime-type
-        self.charset=charset    # the charset (if any) 
-        self.description=description    # if any 
-        self.disposition=disposition    # 'inline', 'attachment' or None
-        self.sanitized_filename=sanitized_filename # cleanup your filename here (TODO)  
-        self.is_body=is_body        # usually in (None, 'text/plain' or 'text/html')
-        self.content_id=content_id  # if any
-        if self.content_id:
-            # strip '<>' to ease search and replace in "root" content (TODO) 
-            if self.content_id.startswith('<') and self.content_id.endswith('>'):
-                self.content_id=self.content_id[1:-1]
-
-    def get_payload(self):
-        """
-        decode and return part payload. if I{type} is 'text/*' and I{charset} 
-        not C{None}, be careful to take care of the text encoding. Use 
-        something like C{part.get_payload().decode(part.charset)}
-        """
-        
-        payload=None
-        if self.type.startswith('message/'): 
-            # I don't use msg.as_string() because I want to use mangle_from_=False
-            if sys.version_info<(3, 0):
-                # python 2.x  
-                from email.generator import Generator
-                fp = io.StringIO()
-                g = Generator(fp, mangle_from_=False)
-                g.flatten(self.part, unixfrom=False)
-                payload=fp.getvalue()
-            else:
-                # support only for python >= 3.2
-                from email.generator import BytesGenerator
-                import io
-                fp = io.BytesIO()
-                g = BytesGenerator(fp, mangle_from_=False)
-                g.flatten(self.part, unixfrom=False)
-                payload=fp.getvalue()
-                
-        else:
-            payload=self.part.get_payload(decode=True)
-        return payload
-                        
-    def __repr__(self):
-        st='MailPart<'
-        if self.is_body:
-            st+='*'
-        st+=self.type
-        if self.charset:
-            st+=' charset='+self.charset
-        if self.filename:
-            st+=' filename='+self.filename
-        if self.content_id:
-            st+=' content_id='+self.content_id
-        st+=' len=%d' % (len(self.get_payload()), )
-        st+='>'
-        return st
-
-
-
-_line_end_re=re.compile('\r\n|\n\r|\n|\r')
-
-def _friendly_header(header):
-    """
-    Convert header returned by C{email.message.Message.get()} into a 
-    user friendly string. 
-
-    Py3k C{email.message.Message.get()} return C{header.Header()} with charset 
-    set to C{charset.UNKNOWN8BIT} when the header contains invalid characters, 
-    else it return I{str} as  Python 2.X does   
-    
-    @type header: str or email.header.Header
-    @param header: the header to convert into a user friendly string
-    
-    @rtype: str
-    @returns: the converter header 
-    """
-    
-    save=header
-    if isinstance(header, email.header.Header):
-        header=str(header)
-        
-    return re.sub(_line_end_re, ' ', header)
-        
-def decode_mail_header(value, default_charset='us-ascii'):
-    """
-    Decode a header value into a unicode string. 
-    Works like a more smarter python 
-    C{u"".join(email.header.decode_header()} function
-    
-    @type value: str
-    @param value: the value of the header. 
-    @type default_charset: str
-    @keyword default_charset: if one charset used in the header (multiple charset 
-    can be mixed) is unknown, then use this charset instead.  
-    
-    >>> decode_mail_header('=?iso-8859-1?q?Courrier_=E8lectronique_en_Fran=E7ais?=')
-    u'Courrier \\xe8lectronique en Fran\\xe7ais'
-    """
-
-#    value=_friendly_header(value)
-    try:
-        headers=email.header.decode_header(value)
-    except email.errors.HeaderParseError:
-        # this can append in email.base64mime.decode(), for example for this value:
-        # '=?UTF-8?B?15HXmdeh15jXqNeVINeY15DXpteUINeTJ9eV16jXlSDXkdeg15XXldeUINem15PXpywg15TXptei16bXldei15nXnSDXqdecINek15zXmdeZ?==?UTF-8?B?157XldeR15nXnCwg157Xldek16Ig157Xl9eV15wg15HXodeV15bXnyDXk9ec15DXnCDXldeh15gg157Xl9eR16rXldeqINep15wg15HXmdeQ?==?UTF-8?B?15zXmNeZ?='
-        # then return a sanitized ascii string
-        # TODO: some improvements are possible here, but a failure here is
-        # unlikely
-        return value.encode('us-ascii', 'replace').decode('us-ascii')
-    else:
-        for i, (text, charset) in enumerate(headers):
-            # python 3.x
-            # email.header.decode_header('a') -> [('a', None)]
-            # email.header.decode_header('a =?ISO-8859-1?Q?foo?= b')
-            # --> [(b'a', None), (b'foo', 'iso-8859-1'), (b'b', None)]
-            # in Py3 text is sometime str and sometime byte :-(
-            # python 2.x
-            # email.header.decode_header('a') -> [('a', None)]
-            # email.header.decode_header('a =?ISO-8859-1?Q?foo?= b')
-            # --> [('a', None), ('foo', 'iso-8859-1'), ('b', None)]
-            if (charset is None and sys.version_info>=(3, 0)):
-                # Py3
-                if isinstance(text, str):
-                    # convert Py3 string into bytes string to be sure their is no 
-                    # non us-ascii chars and because next line expect byte string
-                    text=text.encode('us-ascii', 'replace')
-            try:
-                headers[i]=text.decode(charset or 'us-ascii', 'replace')
-            except LookupError:
-                # if the charset is unknown, force default 
-                headers[i]=text.decode(default_charset, 'replace')
-
-        return "".join(headers)
-    
-def get_mail_addresses(message, header_name):
-    """
-    retrieve all email addresses from one message header
-
-    @type message: email.message.Message
-    @param message: the email message
-    @type header_name: str
-    @param header_name: the name of the header, can be 'from', 'to', 'cc' or 
-    any other header containing one or more email addresses
-    @rtype: list
-    @returns: a list of the addresses in the form of tuples 
-    C{[(u'Name', 'addresse@domain.com'), ...]}
-
-    >>> import email
-    >>> import email.mime.text
-    >>> msg=email.mime.text.MIMEText('The text.', 'plain', 'us-ascii')
-    >>> msg['From']=email.email.utils.formataddr(('Me', 'me@foo.com'))
-    >>> msg['To']=email.email.utils.formataddr(('A', 'a@foo.com'))+', '+email.email.utils.formataddr(('B', 'b@foo.com'))
-    >>> print msg.as_string(unixfrom=False)
-    Content-Type: text/plain; charset="us-ascii"
-    MIME-Version: 1.0
-    Content-Transfer-Encoding: 7bit
-    From: Me <me@foo.com>
-    To: A <a@foo.com>, B <b@foo.com>
-    <BLANKLINE>
-    The text.
-    >>> get_mail_addresses(msg, 'from')
-    [(u'Me', 'me@foo.com')]
-    >>> get_mail_addresses(msg, 'to')
-    [(u'A', 'a@foo.com'), (u'B', 'b@foo.com')]
-    """ 
-    addrs=email.utils.getaddresses([ _friendly_header(h) for h in message.get_all(header_name, [])])
-    for i, (addr_name, addr) in enumerate(addrs):
-        if not addr_name and addr:
-            # only one string! Is it the address or the  address name ?
-            # use the same for both and see later
-            addr_name=addr
-            
-        if is_usascii(addr):
-            # address must be ascii only and must match address regex
-            if not email_address_re.match(addr):
-                addr=''
-        else:
-            addr=''
-        addrs[i]=(decode_mail_header(addr_name), addr)
-    return addrs
-
-def get_filename(part):
-    """
-    Find the filename of a mail part. Many MUA send attachments with the 
-    filename in the I{name} parameter of the I{Content-type} header instead 
-    of in the I{filename} parameter of the I{Content-Disposition} header.
-    
-    @type part: inherit from email.mime.base.MIMEBase
-    @param part: the mail part 
-    @rtype: None or unicode
-    @returns: the filename or None if not found
-    
-    >>> import email.mime.image
-    >>> attach=email.mime.image.MIMEImage('data', 'png')
-    >>> attach.add_header('Content-Disposition', 'attachment', filename='image.png')
-    >>> get_filename(attach)
-    u'image.png'
-    >>> print attach.as_string(unixfrom=False)
-    Content-Type: image/png
-    MIME-Version: 1.0
-    Content-Transfer-Encoding: base64
-    Content-Disposition: attachment; filename="image.png"
-    <BLANKLINE>
-    ZGF0YQ==
-    >>> import email.mime.text
-    >>> attach=email.mime.text.MIMEText('The text.', 'plain', 'us-ascii')
-    >>> attach.add_header('Content-Disposition', 'attachment', filename=('iso-8859-1', 'fr', u'Fran\\xe7ais.txt'.encode('iso-8859-1')))
-    >>> get_filename(attach)
-    u'Fran\\xe7ais.txt'
-    >>> print attach.as_string(unixfrom=False)
-    Content-Type: text/plain; charset="us-ascii"
-    MIME-Version: 1.0
-    Content-Transfer-Encoding: 7bit
-    Content-Disposition: attachment; filename*="iso-8859-1'fr'Fran%E7ais.txt"
-    <BLANKLINE>
-    The text.
-    """
-    filename=part.get_param('filename', None, 'content-disposition')
-    if not filename:
-        filename=part.get_param('name', None) # default is 'content-type'
-        
-    if filename:
-        if isinstance(filename, tuple):
-            # RFC 2231 must be used to encode parameters inside MIME header
-            filename=email.utils.collapse_rfc2231_value(filename).strip()
-        else:
-            # But a lot of MUA erroneously use RFC 2047 instead of RFC 2231
-            # in fact anybody missuse RFC2047 here !!!
-            filename=decode_mail_header(filename)
-        
-    return filename
-
-def _search_message_content(contents, part):
-    """
-    recursive search of message content (text or HTML) inside 
-    the structure of the email. Used by L{search_message_content()}
-    
-    @type contents: dict
-    @param contents: contents already found in parents or brothers I{parts}. 
-    The dictionary will be completed as and when. key is the MIME type of the part.  
-    @type part: inherit email.mime.base.MIMEBase
-    @param part: the part of the mail to look inside recursively.    
-    """
-    type=part.get_content_type()
-    if part.is_multipart(): # type.startswith('multipart/'):
-        # explore only True 'multipart/*' 
-        # because 'messages/rfc822' are 'multipart/*' too but
-        # must not be explored here 
-        if type=='multipart/related':
-            # the first part or the one pointed by start 
-            start=part.get_param('start', None)
-            related_type=part.get_param('type', None)
-            for i, subpart in enumerate(part.get_payload()):
-                if (not start and i==0) or (start and start==subpart.get('Content-Id')):
-                    _search_message_content(contents, subpart)
-                    return
-        elif type=='multipart/alternative':
-            # all parts are candidates and latest is the best
-            for subpart in part.get_payload():
-                _search_message_content(contents, subpart)
-        elif type in ('multipart/report',  'multipart/signed'):
-            # only the first part is candidate
-            try:
-                subpart=part.get_payload()[0]
-            except IndexError:
-                return
-            else:
-                _search_message_content(contents, subpart)
-                return
-
-        elif type=='multipart/encrypted':
-            # the second part is the good one, but we need to de-crypt it 
-            # using the first part. Do nothing
-            return
-            
-        else: 
-            # unknown types must be handled as 'multipart/mixed'
-            # This is the peace of code that could probably be improved, 
-            # I use a heuristic : if not already found, use first valid non 
-            # 'attachment' parts found
-            for subpart in part.get_payload():
-                tmp_contents=dict()
-                _search_message_content(tmp_contents, subpart)
-                for k, v in tmp_contents.items():
-                    if not subpart.get_param('attachment', None, 'content-disposition')=='':
-                        # if not an attachment, initiate value if not already found
-                        contents.setdefault(k, v)
-            return
-    else:
-        contents[part.get_content_type().lower()]=part
-        return
-    
-    return
-
-def search_message_content(mail):
-    """
-    search of message content (text or HTML) inside 
-    the structure of the mail. This function is used by L{get_mail_parts()}
-    to set the C{is_body} part of the L{MailPart}s
-    
-    @type mail: inherit from email.message.Message
-    @param mail: the message to search in.
-    @rtype: dict
-    @returns: a dictionary of the form C{{'text/plain': text_part, 'text/html': html_part}}
-    where text_part and html_part inherite from C{email.mime.text.MIMEText} 
-    and are respectively the I{text} and I{HTML} version of the message content. 
-    One part can be missing. The dictionay can aven be empty if none of the
-    parts math the requirements to be considered as the content.     
-    """
-    contents=dict()
-    _search_message_content(contents, mail)
-    return contents
-
-def get_mail_parts(msg):
-    """
-    return a list of all parts of the message as a list of L{MailPart}.
-    Retrieve parts attributes to fill in L{MailPart} object.
-    
-    @type msg: inherit email.message.Message
-    @param msg: the message
-    @rtype: list
-    @returns: list of mail parts
-
-    >>> import email.mime.multipart
-    >>> msg=email.mime.multipart.MIMEMultipart(boundary='===limit1==')
-    >>> import email.mime.text
-    >>> txt=email.mime.text.MIMEText('The text.', 'plain', 'us-ascii')
-    >>> msg.attach(txt)
-    >>> import email.mime.image
-    >>> image=email.mime.image.MIMEImage('data', 'png')
-    >>> image.add_header('Content-Disposition', 'attachment', filename='image.png')
-    >>> msg.attach(image)
-    >>> print msg.as_string(unixfrom=False)    
-    Content-Type: multipart/mixed; boundary="===limit1=="
-    MIME-Version: 1.0
-    <BLANKLINE>
-    --===limit1==
-    Content-Type: text/plain; charset="us-ascii"
-    MIME-Version: 1.0
-    Content-Transfer-Encoding: 7bit
-    <BLANKLINE>
-    The text.
-    --===limit1==
-    Content-Type: image/png
-    MIME-Version: 1.0
-    Content-Transfer-Encoding: base64
-    Content-Disposition: attachment; filename="image.png"
-    <BLANKLINE>
-    ZGF0YQ==
-    --===limit1==--
-    >>> parts=get_mail_parts(msg)
-    >>> parts
-    [MailPart<*text/plain charset=us-ascii len=9>, MailPart<image/png filename=image.png len=4>]
-    >>> # the star "*" means this is the mail content, not an attachment 
-    >>> parts[0].get_payload().decode(parts[0].charset)
-    u'The text.'
-    >>> parts[1].filename, len(parts[1].get_payload())
-    (u'image.png', 4)
-
-    """
-    mailparts=[]
-
-    # retrieve messages of the email
-    contents=search_message_content(msg)
-    # reverse contents dict
-    parts=dict((v,k) for k, v in contents.items())
-
-    # organize the stack to handle deep first search
-    stack=[ msg, ]
-    while stack:
-        part=stack.pop(0)
-        type=part.get_content_type()
-        if type.startswith('message/'): 
-            # ('message/delivery-status', 'message/rfc822', 'message/disposition-notification'):
-            # I don't want to explore the tree deeper her and just save source using msg.as_string()
-            # but I don't use msg.as_string() because I want to use mangle_from_=False 
-            filename='message.eml'
-            mailparts.append(MailPart(part, filename=filename, type=type, charset=part.get_param('charset'), description=part.get('Content-Description')))
-        elif part.is_multipart():
-            # insert new parts at the beginning of the stack (deep first search)
-            stack[:0]=part.get_payload()
-        else:
-            charset=part.get_param('charset')
-            filename=get_filename(part)
-                
-            disposition=None
-            if part.get_param('inline', None, 'content-disposition')=='':
-                disposition='inline'
-            elif part.get_param('attachment', None, 'content-disposition')=='':
-                disposition='attachment'
-                
-            mailparts.append(MailPart(part, filename=filename, type=type, charset=charset, content_id=part.get('Content-Id'), description=part.get('Content-Description'), disposition=disposition, is_body=parts.get(part, False)))
-
-    return mailparts
-
-
-def decode_text(payload, charset, default_charset):
-    """
-    Try to decode text content by trying multiple charset until success.
-    First try I{charset}, else try I{default_charset} finally
-    try popular charsets in order : ascii, utf-8, utf-16, windows-1252, cp850
-    If all fail then use I{default_charset} and replace wrong characters
-    
-    @type payload: str
-    @param payload: the content to decode
-    @type charset: str or None
-    @param charset: the first charset to try if != C{None}
-    @type default_charset: str or None
-    @param default_charset: the second charset to try if != C{None}
-    
-    @rtype: tuple
-    @returns: a tuple of the form C{(payload, charset)}
-        - I{payload}: this is the decoded payload if charset is not None and
-        payload is a unicode string
-        - I{charset}: the charset that was used to decode I{payload} If charset is
-        C{None} then something goes wrong: if I{payload} is unicode then
-        invalid characters have been replaced and the used charset is I{default_charset}
-        else, if I{payload} is still byte string then nothing has been done. 
-             
-     
-    """
-    for chset in [ charset, default_charset, 'ascii', 'utf-8', 'utf-16', 'windows-1252', 'cp850' ]:
-        if chset:
-            try: 
-                return payload.decode(chset), chset
-            except UnicodeError:
-                pass
-
-    if default_charset:
-        return payload.decode(chset, 'replace'), None 
-
-    return payload, None
-
-class PyzMessage(email.message.Message):
-    """
-    Inherit from email.message.Message. Combine L{get_mail_parts()},
-    L{get_mail_addresses()} and L{decode_mail_header()} into a
-    B{convenient} object to access mail contents and attributes.
-    This class also B{sanitize} part filenames.
-    
-    @type mailparts: list of L{MailPart}
-    @ivar mailparts: list of L{MailPart} objects composing the email, I{text_part}
-    and I{html_part} are part of this list as are other attachements and embedded
-    contents.
-    @type text_part: L{MailPart} or None
-    @ivar text_part: the L{MailPart} object that contains the I{text}
-    version of the message, None if the mail has not I{text} content.
-    @type html_part: L{MailPart} or None
-    @ivar html_part: the L{MailPart} object that contains the I{HTML}
-    version of the message, None if the mail has not I{HTML} content.
-
-    @note: Sample:
-    
-    >>> raw='''Content-Type: text/plain; charset="us-ascii"
-    ... MIME-Version: 1.0
-    ... Content-Transfer-Encoding: 7bit
-    ... Subject: The subject
-    ... From: Me <me@foo.com>
-    ... To: A <a@foo.com>, B <b@foo.com>
-    ...  
-    ... The text.
-    ... '''
-    >>> msg=PyzMessage.factory(raw)
-    >>> print 'Subject: %r' % (msg.get_subject(), )
-    Subject: u'The subject'
-    >>> print 'From: %r' % (msg.get_address('from'), )
-    From: (u'Me', 'me@foo.com')
-    >>> print 'To: %r' % (msg.get_addresses('to'), )
-    To: [(u'A', 'a@foo.com'), (u'B', 'b@foo.com')]
-    >>> print 'Cc: %r' % (msg.get_addresses('cc'), )
-    Cc: []
-    >>> for mailpart in msg.mailparts:
-    ...   print '    %sfilename=%r sanitized_filename=%r type=%s charset=%s desc=%s size=%d' % ('*'if mailpart.is_body else ' ', mailpart.filename, mailpart.sanitized_filename, mailpart.type, mailpart.charset, mailpart.part.get('Content-Description'), 0 if mailpart.get_payload()==None else len(mailpart.get_payload()))
-    ...   if mailpart.is_body=='text/plain':
-    ...     payload, used_charset=decode_text(mailpart.get_payload(), mailpart.charset, None) 
-    ...     print '        >', payload.split('\\n')[0]
-    ...
-        *filename=None sanitized_filename='text.txt' type=text/plain charset=us-ascii desc=None size=10
-            > The text.
-    """
-
-    @staticmethod
-    def smart_parser(input):
-        """
-        Use the appropriate parser and return a email.message.Message object
-        (this is not a L{PyzMessage} object)
-        
-        @type input: string, file, bytes, binary_file or  email.message.Message
-        @param input: the source of the message
-        @rtype: email.message.Message
-        @returns: the message
-        """
-        if isinstance(input, email.message.Message):
-            return input
-        
-        if sys.version_info<(3, 0):
-            # python 2.x 
-            if isinstance(input, str):
-                return email.message_from_string(input)
-            elif hasattr(input, 'read') and hasattr(input, 'readline'):
-                return email.message_from_file(input)
-            else:
-                raise ValueError('input must be a string, a file or a Message')
-        else:
-            # python 3.x 
-            if isinstance(input, str):
-                return email.message_from_string(input)
-            elif isinstance(input, bytes):
-                # python >= 3.2 only
-                return email.message_from_bytes(input)
-            elif hasattr(input, 'read') and hasattr(input, 'readline'):
-                if hasattr(input, 'encoding'):
-                    # python >= 3.2 only
-                    return email.message_from_file(input)
-                else:
-                    return email.message_from_binary_file(input)
-            else:
-                raise ValueError('input must be a string a bytes, a file or a Message')
-    
-    @staticmethod
-    def factory(input):
-        """
-        Use the appropriate parser and return a L{PyzMessage} object
-        see L{smart_parser}
-        @type input: string, file, bytes, binary_file or  email.message.Message
-        @param input: the source of the message
-        @rtype: L{PyzMessage}
-        @returns: the L{PyzMessage} message
-        """
-        return PyzMessage(PyzMessage.smart_parser(input))
-        
-    
-    def __init__(self, message):
-        """
-        Initialize the object with data coming from I{message}. 
-        
-        @type message: inherit email.message.Message
-        @param message: The message
-        """
-        if not isinstance(message, email.message.Message):
-            raise ValueError("message must inherit from email.message.Message use PyzMessage.factory() instead")
-        self.__dict__.update(message.__dict__)  
-
-        self.mailparts=get_mail_parts(self)
-        self.text_part=None
-        self.html_part=None
-        
-        filenames=[]
-        for part in self.mailparts:
-            ext=mimetypes.guess_extension(part.type)
-            if not ext:
-                 # default to .bin
-                ext='.bin'
-            elif ext=='.ksh':
-                # guess_extension() is not very accurate, .txt is more
-                # appropriate than .ksh 
-                ext='.txt'
-            
-            sanitized_filename=sanitize_filename(part.filename, part.type.split('/', 1)[0], ext)
-            sanitized_filename=handle_filename_collision(sanitized_filename, filenames)
-            filenames.append(sanitized_filename.lower())
-            part.sanitized_filename=sanitized_filename
-            
-            if part.is_body=='text/plain': 
-                self.text_part=part
-                
-            if part.is_body=='text/html': 
-                self.html_part=part
-    
-    def get_addresses(self, name):
-        """
-        return the I{name} header value as an list of addresses tuple as 
-        returned by L{get_mail_addresses()}
-        
-        @type name: str
-        @param name: the name of the header to read value from: 'to', 'cc' are
-        valid I{name} here.
-        @rtype: tuple
-        @returns: a tuple of the form C{('Sender Name', 'sender.address@domain.com')}
-        or C{('', '')} if no header match that I{name}.
-        """ 
-        return get_mail_addresses(self, name)
-    
-    def get_address(self, name):
-        """
-        return the I{name} header value as an address tuple as returned by 
-        L{get_mail_addresses()} 
-
-        @type name: str
-        @param name: the name of the header to read value from: : C{'from'} can
-        be used to return the sender address. 
-        @rtype: list of tuple
-        @returns: a list of tuple of the form C{[('Recipient Name', 'recipient.address@domain.com'), ...]}
-        or an empty list if no header match that I{name}.
-        """ 
-        value=get_mail_addresses(self, name)
-        if value:
-            return value[0]
-        else:
-            return ('', '')
-    
-    def get_subject(self, default=''):
-        """
-        return the RFC2047 decoded subject.
-        
-        @type default: any
-        @param default: The value to return if the message has no I{Subject}
-        @rtype: unicode
-        @returns: the subject or C{default}
-        """
-        return self.get_decoded_header('subject', default)
-
-    def get_decoded_header(self, name, default=''):
-        """
-        return decoded header I{name} using RFC2047. Always use this function 
-        to access header, because any header can contain invalid characters 
-        and this function sanitize the string and avoid unicode exception later
-        in your program. 
-        EVEN for date, I already saw a "Center box bar horizontal" instead
-        of a minus character. 
-
-        @type name: str
-        @param name: the name of the header to read value from.
-        @type default: any
-        @param default: The value to return if the I{name} field don't exist
-        in this message.
-        @rtype: unicode
-        @returns: the value of the header having that I{name} or C{default} if no
-        header have that name.
-        """
-        value=self.get(name)
-        if value==None:
-            value=default
-        else:
-            value=decode_mail_header(value)
-        return value
-    
-class PzMessage(PyzMessage):
-    """
-    Old name and interface for PyzMessage. 
-    B{Deprecated}
-    """
-
-    def __init__(self, input):
-        """
-        Initialize the object with data coming from I{input}. 
-        
-        @type input: str or file or email.message.Message
-        @param input: used as the raw content for the email, can be a string,  
-        a file object or an email.message.Message object. 
-        """
-        PyzMessage.__init__(self, self.smart_parser(input))
-        
-
-def message_from_string(s, *args, **kws):
-    """
-    Parse a string into a L{PyzMessage} object model.
-    @type s: str
-    @param s: the input string 
-    @rtype: L{PyzMessage}
-    @return: the L{PyzMessage} object
-    """
-    return PyzMessage(email.message_from_string(s, *args, **kws))
-
-def message_from_file(fp, *args, **kws):
-    """
-    Read a file and parse its contents into a L{PyzMessage} object model.
-    @type fp: text_file
-    @param fp: the input file (must be open in text mode if Python >= 3.0)
-    @rtype: L{PyzMessage}
-    @return: the L{PyzMessage} object
-    """
-    return PyzMessage(email.message_from_file(fp, *args, **kws))
-
-def message_from_bytes(s, *args, **kws):
-    """
-    Parse a bytes string into a L{PyzMessage} object model.
-    B{(Python >= 3.2)}
-    @type s: bytes
-    @param s: the input bytes string 
-    @rtype: L{PyzMessage}
-    @return: the L{PyzMessage} object
-    """
-    return PyzMessage(email.message_from_bytes(s, *args, **kws))
-
-def message_from_binary_file(fp, *args, **kws):
-    """
-    Read a binary file and parse its contents into a L{PyzMessage} object model.
-    B{(Python >= 3.2)}
-    @type fp: binary_file
-    @param fp: the input file, must be open in binary mode 
-    @rtype: L{PyzMessage}
-    @return: the L{PyzMessage} object
-    """
-    return PyzMessage(email.message_from_binary_file(fp, *args, **kws))
-
-
-if __name__ == "__main__":
-    import sys
-
-    if len(sys.argv)<=1:
-        print('usage : %s filename' % sys.argv[0])
-        print('read an email from file and display a resume of its content')
-        sys.exit(1)
-        
-    msg=PyzMessage.factory(open(sys.argv[1], 'rb'))
-    
-    print('Subject: %r' % (msg.get_subject(), ))
-    print('From: %r' % (msg.get_address('from'), ))
-    print('To: %r' % (msg.get_addresses('to'), ))
-    print('Cc: %r' % (msg.get_addresses('cc'), ))
-    print('Date: %r' % (msg.get_decoded_header('date', ''), ))
-    print('Message-Id: %r' % (msg.get_decoded_header('message-id', ''), ))
-    
-    for mailpart in msg.mailparts:
-        # dont forget to be careful to sanitize 'filename' and be carefull
-        # for filename collision, to before to save :
-        print('   %sfilename=%r type=%s charset=%s desc=%s size=%d' % ('*'if mailpart.is_body else ' ', mailpart.filename, mailpart.type, mailpart.charset, mailpart.part.get('Content-Description'), 0 if mailpart.get_payload()==None else len(mailpart.get_payload())))
-
-        if mailpart.is_body=='text/plain':
-            # print first 3 lines
-            payload, used_charset=decode_text(mailpart.get_payload(), mailpart.charset, None) 
-            for line in payload.split('\n')[:3]:
-                # be careful console can be unable to display unicode characters
-                if line:
-                    print('       >', line)
-
-        
-
diff --git a/pyzmail/tests/__init__.py b/pyzmail/tests/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/pyzmail/tests/test_both.py b/pyzmail/tests/test_both.py
deleted file mode 100644
index 2607ed8d7f..0000000000
--- a/pyzmail/tests/test_both.py
+++ /dev/null
@@ -1,99 +0,0 @@
-import unittest
-import pyzmail
-from pyzmail.generate import *
-from pyzmail.parse import *
-
-class TestBoth(unittest.TestCase):
-
-    def setUp(self):
-        pass
-    
-    def test_compose_and_parse(self):
-        """test generate and parse"""
-        
-        sender=('Me', 'me@foo.com')
-        recipients=[('Him', 'him@bar.com'), 'just@me.com']
-        subject='Le sujet en Fran\xe7ais'
-        text_content='Bonjour aux Fran\xe7ais'
-        prefered_encoding='iso-8859-1'
-        text_encoding='iso-8859-1'
-        attachments=[('attached content', 'text', 'plain', 'textfile1.txt', 'us-ascii'),
-                     ('Fran\xe7ais', 'text', 'plain', 'textfile2.txt', 'iso-8859-1'),
-                     ('Fran\xe7ais', 'text', 'plain', 'textfile3.txt', 'iso-8859-1'),
-                     (b'image', 'image', 'jpg', 'imagefile.jpg', None),
-                     ]
-        embeddeds=[('embedded content', 'text', 'plain', 'embedded', 'us-ascii'),
-                   (b'picture', 'image', 'png', 'picture', None),
-                   ]
-        headers=[ ('X-extra', 'extra value'), ('X-extra2', "Seconde ent\xe8te"), ('X-extra3', 'last extra'),]
-        
-        message_id_string='pyzmail'
-        date=1313558269
-        
-        payload, mail_from, rcpt_to, msg_id=pyzmail.compose_mail(\
-            sender, \
-            recipients, \
-            subject, \
-            prefered_encoding, \
-            (text_content, text_encoding), \
-            html=None, \
-            attachments=attachments, \
-            embeddeds=embeddeds, \
-            headers=headers, \
-            message_id_string=message_id_string, \
-            date=date\
-            )
-        
-        msg=PyzMessage.factory(payload)
-        
-        self.assertEqual(sender, msg.get_address('from'))
-        self.assertEqual(recipients[0], msg.get_addresses('to')[0])
-        self.assertEqual(recipients[1], msg.get_addresses('to')[1][1])
-        self.assertEqual(subject, msg.get_subject())
-        self.assertEqual(subject, msg.get_decoded_header('subject'))
-        
-        # try to handle different timezone carefully
-        mail_date=list(email.utils.parsedate(msg.get_decoded_header('date')))
-        self.assertEqual(mail_date[:6], list(time.localtime(date))[:6])
-        
-        self.assertNotEqual(msg.get('message-id').find(message_id_string), -1)
-        for name, value in headers:
-            self.assertEqual(value, msg.get_decoded_header(name))
-        
-        for mailpart in msg.mailparts:
-            if mailpart.is_body:
-                self.assertEqual(mailpart.content_id, None)
-                self.assertEqual(mailpart.filename, None)
-                self.assertEqual(type(mailpart.sanitized_filename), str)
-                if mailpart.type=='text/plain':
-                    self.assertEqual(mailpart.get_payload(), text_content.encode(text_encoding))
-                else:
-                    self.fail('found unknown body part')
-            else:
-                if mailpart.filename:
-                    lst=attachments
-                    self.assertEqual(mailpart.filename, mailpart.sanitized_filename)
-                    self.assertEqual(mailpart.content_id, None)
-                elif mailpart.content_id:
-                    lst=embeddeds
-                    self.assertEqual(mailpart.filename, None)
-                else:
-                    self.fail('found unknown part')
-                    
-                found=False
-                for attach in lst:
-                    found=(mailpart.filename and attach[3]==mailpart.filename) \
-                        or (mailpart.content_id and attach[3]==mailpart.content_id)
-                    if found:
-                        break
-                        
-                if found:
-                    self.assertEqual(mailpart.type, attach[1]+'/'+attach[2])
-                    payload=mailpart.get_payload()
-                    if attach[1]=='text' and attach[4] and isinstance(attach[0], str):
-                        payload=payload.decode(attach[4])
-                    self.assertEqual(payload, attach[0])
-                else:
-                    self.fail('found unknown attachment')
-
-
diff --git a/pyzmail/tests/test_generate.py b/pyzmail/tests/test_generate.py
deleted file mode 100644
index 7afdebc593..0000000000
--- a/pyzmail/tests/test_generate.py
+++ /dev/null
@@ -1,30 +0,0 @@
-import unittest, doctest
-import pyzmail
-from pyzmail.generate import *
-
-class TestGenerate(unittest.TestCase):
-
-    def setUp(self):
-        pass
-    
-    def test_format_addresses(self):
-        """test format_addresse"""
-        self.assertEqual('foo@example.com', str(format_addresses([ 'foo@example.com', ])))
-        self.assertEqual('Foo <foo@example.com>', str(format_addresses([ ('Foo', 'foo@example.com'), ])))
-        # notice the space around the comma
-        self.assertEqual('foo@example.com , bar@example.com', str(format_addresses([ 'foo@example.com', 'bar@example.com'])))
-        # notice the space around the comma
-        self.assertEqual('Foo <foo@example.com> , Bar <bar@example.com>', str(format_addresses([ ('Foo', 'foo@example.com'), ( 'Bar', 'bar@example.com')])))
-
-# Add doctest 
-def load_tests(loader, tests, ignore):
-    # this works with python 2.7 and 3.x
-    tests.addTests(doctest.DocTestSuite(pyzmail.generate))
-    return tests
-
-def additional_tests():
-    # Add doctest for python 2.6 and below
-    if sys.version_info<(2, 7):
-        return doctest.DocTestSuite(pyzmail.generate)
-    else:
-        return unittest.TestSuite()
diff --git a/pyzmail/tests/test_parse.py b/pyzmail/tests/test_parse.py
deleted file mode 100644
index f7a5adb9e2..0000000000
--- a/pyzmail/tests/test_parse.py
+++ /dev/null
@@ -1,290 +0,0 @@
-import unittest, doctest
-import pyzmail
-from pyzmail.parse import *
-
-
-class Msg:
-    """mimic a email.Message"""
-    def __init__(self, value):
-        self.value=value
-        
-    def get_all(self, header_name, default):
-        if self.value:
-            return [self.value, ]
-        else:
-            return []
-
-class TestParse(unittest.TestCase):
-
-    def setUp(self):
-        pass
-    
-    def test_decode_mail_header(self):
-        """test decode_mail_header()"""
-        self.assertEqual(decode_mail_header(''), '')
-        self.assertEqual(decode_mail_header('hello'), 'hello')
-        self.assertEqual(decode_mail_header('hello '), 'hello ')
-        self.assertEqual(decode_mail_header('=?iso-8859-1?q?Courrier_=E8lectronique_Fran=E7ais?='), 'Courrier \xe8lectronique Fran\xe7ais')
-        self.assertEqual(decode_mail_header('=?utf8?q?Courrier_=C3=A8lectronique_Fran=C3=A7ais?='), 'Courrier \xe8lectronique Fran\xe7ais')
-        self.assertEqual(decode_mail_header('=?utf-8?b?RnJhbsOnYWlz?='), 'Fran\xe7ais')
-        self.assertEqual(decode_mail_header('=?iso-8859-1?q?Courrier_=E8lectronique_?= =?utf8?q?Fran=C3=A7ais?='), 'Courrier \xe8lectronique Fran\xe7ais')
-        self.assertEqual(decode_mail_header('=?iso-8859-1?q?Courrier_=E8lectronique_?= =?utf-8?b?RnJhbsOnYWlz?='), 'Courrier \xe8lectronique Fran\xe7ais')
-        self.assertEqual(decode_mail_header('h_subject_q_iso_8858_1 : =?ISO-8859-1?Q?Fran=E7ais=E20accentu=E9?= !'), 'h_subject_q_iso_8858_1 :Fran\xe7ais\xe20accentu\xe9!')
-   
-    def test_get_mail_addresses(self):
-        """test get_mail_addresses()"""
-        self.assertEqual([ ('foo@example.com', 'foo@example.com') ], get_mail_addresses(Msg('foo@example.com'), 'to'))
-        self.assertEqual([ ('Foo', 'foo@example.com'), ], get_mail_addresses(Msg('Foo <foo@example.com>'), 'to'))
-        # notice the space around the comma
-        self.assertEqual([ ('foo@example.com', 'foo@example.com'), ('bar@example.com', 'bar@example.com')], get_mail_addresses(Msg('foo@example.com , bar@example.com'), 'to'))
-        self.assertEqual([ ('Foo', 'foo@example.com'), ( 'Bar', 'bar@example.com')], get_mail_addresses(Msg('Foo <foo@example.com> , Bar <bar@example.com>'), 'to'))
-        self.assertEqual([ ('Foo', 'foo@example.com'), ('bar@example.com', 'bar@example.com')], get_mail_addresses(Msg('Foo <foo@example.com> , bar@example.com'), 'to'))
-        self.assertEqual([ ('Mr Foo', 'foo@example.com'), ('bar@example.com', 'bar@example.com')], get_mail_addresses(Msg('Mr\nFoo <foo@example.com> , bar@example.com'), 'to'))
-        
-        self.assertEqual([ ('Beno\xeet', 'benoit@example.com')], get_mail_addresses(Msg('=?utf-8?q?Beno=C3=AEt?= <benoit@example.com>'), 'to'))
-        
-        # address already encoded into utf8 (bad)
-        address='Ant\xf3nio Foo <a.foo@example.com>'.encode('utf8')
-        if sys.version_info<(3, 0):
-            self.assertEqual([('Ant\ufffd\ufffdnio Foo', 'a.foo@example.com')], get_mail_addresses(Msg(address), 'to'))
-        else:
-            # Python 3.2 return header when surrogate characters are used in header
-            self.assertEqual([('Ant??nio Foo', 'a.foo@example.com'), ], get_mail_addresses(Msg(email.header.Header(address, charset=email.charset.UNKNOWN8BIT, header_name='to')), 'to'))
-        
-    def test_get_filename(self):
-        """test get_filename()"""
-        import email.mime.image
-
-        filename='Fran\xe7ais.png'
-        if sys.version_info<(3, 0):
-            encoded_filename=filename.encode('iso-8859-1')
-        else:
-            encoded_filename=filename
-               
-        payload=b'data'
-        attach=email.mime.image.MIMEImage(payload, 'png')
-        attach.add_header('Content-Disposition', 'attachment', filename='image.png')
-        self.assertEqual('image.png', get_filename(attach))
-
-        attach=email.mime.image.MIMEImage(payload, 'png')
-        attach.add_header('Content-Disposition', 'attachment', filename=('iso-8859-1', 'fr', encoded_filename))
-        self.assertEqual('Fran\xe7ais.png', get_filename(attach))
-        
-        attach=email.mime.image.MIMEImage(payload, 'png')
-        attach.set_param('name', 'image.png')
-        self.assertEqual('image.png', get_filename(attach))
-
-        attach=email.mime.image.MIMEImage(payload, 'png')
-        attach.set_param('name', ('iso-8859-1', 'fr', encoded_filename))
-        self.assertEqual('Fran\xe7ais.png', get_filename(attach))
-
-        attach=email.mime.image.MIMEImage(payload, 'png')
-        attach.add_header('Content-Disposition', 'attachment', filename='image.png')
-        attach.set_param('name', 'image_wrong.png')
-        self.assertEqual('image.png', get_filename(attach))
-
-    def test_get_mailparts(self):
-        """test get_mailparts()"""
-        import email.mime.multipart
-        import email.mime.text
-        import email.mime.image
-        msg=email.mime.multipart.MIMEMultipart(boundary='===limit1==')
-        txt=email.mime.text.MIMEText('The text.', 'plain', 'us-ascii')
-        msg.attach(txt)
-        image=email.mime.image.MIMEImage(b'data', 'png')
-        image.add_header('Content-Disposition', 'attachment', filename='image.png')
-        image.add_header('Content-Description', 'the description')
-        image.add_header('Content-ID', '<this.is.the.normaly.unique.contentid>')
-        msg.attach(image)
-        
-        raw=msg.as_string(unixfrom=False)
-        expected_raw="""Content-Type: multipart/mixed; boundary="===limit1=="
-MIME-Version: 1.0
-
---===limit1==
-Content-Type: text/plain; charset="us-ascii"
-MIME-Version: 1.0
-Content-Transfer-Encoding: 7bit
-
-The text.
---===limit1==
-Content-Type: image/png
-MIME-Version: 1.0
-Content-Transfer-Encoding: base64
-Content-Disposition: attachment; filename="image.png"
-Content-Description: the description
-Content-ID: <this.is.the.normaly.unique.contentid>
-
-ZGF0YQ==<HERE1>
---===limit1==--"""
-    
-        if sys.version_info<(3, 0):
-            expected_raw=expected_raw.replace('<HERE1>','')
-        else:
-            expected_raw=expected_raw.replace('<HERE1>','\n')
-
-        self.assertEqual(raw, expected_raw)
-        
-        parts=get_mail_parts(msg)
-        # [MailPart<*text/plain charset=us-ascii len=9>, MailPart<image/png filename=image.png len=4>]
-
-        self.assertEqual(len(parts), 2)
-
-        self.assertEqual(parts[0].type, 'text/plain')
-        self.assertEqual(parts[0].is_body, 'text/plain') # not a error, is_body must be type 
-        self.assertEqual(parts[0].charset, 'us-ascii')
-        self.assertEqual(parts[0].get_payload().decode(parts[0].charset), 'The text.')
-
-        self.assertEqual(parts[1].type, 'image/png')
-        self.assertEqual(parts[1].is_body, False)
-        self.assertEqual(parts[1].charset, None)
-        self.assertEqual(parts[1].filename, 'image.png')
-        self.assertEqual(parts[1].description, 'the description')
-        self.assertEqual(parts[1].content_id, 'this.is.the.normaly.unique.contentid')
-        self.assertEqual(parts[1].get_payload(), b'data')
-
-    
-    raw_1='''Content-Type: text/plain; charset="us-ascii"
-MIME-Version: 1.0
-Content-Transfer-Encoding: 7bit
-Subject: simple test
-From: Me <me@foo.com>
-To: A <a@foo.com>, B <b@foo.com>
-Cc: C <c@foo.com>, d@foo.com
-User-Agent: pyzmail
-
-The text.
-'''
-
-    def check_message_1(self, msg):
-        self.assertEqual(msg.get_subject(), 'simple test')
-        self.assertEqual(msg.get_decoded_header('subject'), 'simple test')
-        self.assertEqual(msg.get_decoded_header('User-Agent'), 'pyzmail')
-        self.assertEqual(msg.get('User-Agent'), 'pyzmail')
-        self.assertEqual(msg.get_address('from'), ('Me', 'me@foo.com'))
-        self.assertEqual(msg.get_addresses('to'), [('A', 'a@foo.com'), ('B', 'b@foo.com')])
-        self.assertEqual(msg.get_addresses('cc'), [('C', 'c@foo.com'), ('d@foo.com', 'd@foo.com')])
-        self.assertEqual(len(msg.mailparts), 1)
-        self.assertEqual(msg.text_part, msg.mailparts[0])
-        self.assertEqual(msg.html_part, None)
-
-    # use 8bits encoding and 2 different charsets ! python 3.0 & 3.1 are not eable to parse this sample 
-    raw_2=b"""From: sender@domain.com
-To: recipient@domain.com
-Date: Tue, 7 Jun 2011 16:32:17 +0200
-Subject: contains 8bits attachments using different encoding
-Content-Type: multipart/mixed; boundary=mixed
-
---mixed
-Content-Type: text/plain; charset="us-ascii"
-MIME-Version: 1.0
-Content-Transfer-Encoding: 7bit
-
-body
---mixed
-Content-Type: text/plain; charset="windows-1252"
-MIME-Version: 1.0
-Content-Transfer-Encoding: 8bit
-Content-Disposition: attachment; filename="file1.txt"
-
-bo\xeete mail = mailbox
---mixed
-Content-Type: text/plain; charset="utf-8"
-MIME-Version: 1.0
-Content-Transfer-Encoding: 8bit
-Content-Disposition: attachment; filename="file2.txt"
-
-bo\xc3\xaete mail = mailbox
---mixed--
-"""
-    
-    def check_message_2(self, msg):
-        self.assertEqual(msg.get_subject(), 'contains 8bits attachments using different encoding')
-    
-        body, file1, file2=msg.mailparts
-
-        self.assertEqual('file1.txt', file1.filename)
-        self.assertEqual('file2.txt', file2.filename)
-        self.assertEqual('windows-1252', file1.charset)
-        self.assertEqual('utf-8', file2.charset)
-        content=b'bo\xeete mail = mailbox'.decode("windows-1252")
-        content1=file1.get_payload().decode(file1.charset)
-        content2=file2.get_payload().decode(file2.charset)
-        self.assertEqual(content, content1)
-        self.assertEqual(content, content2)
-
-    # this one contain non us-ascii chars in the header 
-    # py 2x and py3k return different value here  
-    raw_3=b'Content-Type: text/plain; charset="us-ascii"\n' \
-          b'MIME-Version: 1.0\n' \
-          b'Content-Transfer-Encoding: 7bit\n' \
-          + 'Subject: Beno\xeet & Ant\xf3nio\n'.encode('utf8') +\
-          b'From: =?utf-8?q?Beno=C3=AEt?= <benoit@example.com>\n' \
-          + 'To: Ant\xf3nio Foo <a.foo@example.com>\n'.encode('utf8') \
-          + 'Cc: Beno\xeet <benoit@foo.com>, d@foo.com\n'.encode('utf8') +\
-          b'User-Agent: pyzmail\n' \
-          b'\n' \
-          b'The text.\n'
-
-    def check_message_3(self, msg):
-        subject='Beno\ufffd\ufffdt & Ant\ufffd\ufffdnio' #  if sys.version_info<(3, 0) else u'Beno??t & Ant??nio'
-        self.assertEqual(msg.get_subject(), subject)
-        self.assertEqual(msg.get_decoded_header('subject'), subject)
-        self.assertEqual(msg.get_decoded_header('User-Agent'), 'pyzmail')
-        self.assertEqual(msg.get('User-Agent'), 'pyzmail')
-        self.assertEqual(msg.get_address('from'), ('Beno\xeet', 'benoit@example.com'))
-
-        to=msg.get_addresses('to')
-        self.assertEqual(to[0][1], 'a.foo@example.com')
-        self.assertEqual(to[0][0], 'Ant\ufffd\ufffdnio Foo' if sys.version_info<(3, 0) else 'Ant??nio Foo')
-        
-        cc=msg.get_addresses('cc')
-        self.assertEqual(cc[0][1], 'benoit@foo.com')
-        self.assertEqual(cc[0][0], 'Beno\ufffd\ufffdt' if sys.version_info<(3, 0) else 'Beno??t')
-        self.assertEqual(cc[1], ('d@foo.com', 'd@foo.com'))
-        
-        self.assertEqual(len(msg.mailparts), 1)
-        self.assertEqual(msg.text_part, msg.mailparts[0])
-        self.assertEqual(msg.html_part, None)
-
-
-    def check_pyzmessage_factories(self, input, check):
-        """test PyzMessage from different sources"""
-        if isinstance(input, bytes) and sys.version_info>=(3, 2):
-            check(PyzMessage.factory(input))
-            check(message_from_bytes(input))
-
-            import io
-            check(PyzMessage.factory(io.BytesIO(input)))
-            check(message_from_binary_file(io.BytesIO(input)))
-
-        if isinstance(input, str):
-
-            check(PyzMessage.factory(input))
-            check(message_from_string(input))
-
-            import io
-            check(PyzMessage.factory(io.StringIO(input)))
-            check(message_from_file(io.StringIO(input)))
-        
-    def test_pyzmessage_factories(self):
-        """test PyzMessage class different sources"""
-        self.check_pyzmessage_factories(self.raw_1, self.check_message_1)
-        self.check_pyzmessage_factories(self.raw_2, self.check_message_2)
-        self.check_pyzmessage_factories(self.raw_3, self.check_message_3)
-
-
-# Add doctest 
-def load_tests(loader, tests, ignore):
-    # this works with python 2.7 and 3.x
-    if sys.version_info<(3, 0): 
-        tests.addTests(doctest.DocTestSuite(pyzmail.parse))
-    return tests
-
-def additional_tests():
-    # Add doctest for python 2.6 and below
-    if sys.version_info<(2, 7):
-        return doctest.DocTestSuite(pyzmail.parse)
-    else:
-        return unittest.TestSuite()
-
diff --git a/pyzmail/tests/test_send.py b/pyzmail/tests/test_send.py
deleted file mode 100644
index 554f8549a4..0000000000
--- a/pyzmail/tests/test_send.py
+++ /dev/null
@@ -1,77 +0,0 @@
-import threading, smtpd, asyncore, socket, smtplib, time
-import unittest
-import pyzmail
-from pyzmail.generate import *
-
-
-smtpd_addr='127.0.0.1' 
-smtpd_port=32525
-smtp_bad_port=smtpd_port-1
-
-smtp_mode='normal'
-smtp_login=None
-smtp_password=None
-    
-
-class SMTPServer(smtpd.SMTPServer):
-    def __init__(self, localaddr, remoteaddr, received):
-        smtpd.SMTPServer.__init__(self, localaddr, remoteaddr)
-        self.set_reuse_addr()
-        # put the received mail into received list 
-        self.received=received
-        
-    def process_message(self, peer, mail_from, rcpt_to, data):
-        ret=None
-        if mail_from.startswith('data_error'):
-            ret='552 Requested mail action aborted: exceeded storage allocation'
-        self.received.append((ret, peer, mail_from, rcpt_to, data))
-        return ret
-
-class TestSend(unittest.TestCase):
-
-    def setUp(self):
-        self.received=[]
-        self.smtp_server=SMTPServer((smtpd_addr, smtpd_port), None, self.received)
-
-        def asyncloop():
-            # check every sec if all channel are close
-            asyncore.loop(1)
-
-                    
-        self.payload, self.mail_from, self.rcpt_to, self.msg_id=compose_mail(('Me', 'me@foo.com'), [('Him', 'him@bar.com')], 'the subject', 'iso-8859-1', ('Hello world', 'us-ascii'))
-
-        # start the server after having built the payload, to handle failure in 
-        # the code above
-        self.smtpd_thread=threading.Thread(target=asyncloop)
-        self.smtpd_thread.daemon=True
-        self.smtpd_thread.start()
-
-
-    def tearDown(self):
-        self.smtp_server.close()
-        self.smtpd_thread.join()
-        
-    def test_simple_send(self):
-        """simple send"""
-        ret=send_mail(self.payload, self.mail_from, self.rcpt_to, smtpd_addr, smtpd_port, smtp_mode=smtp_mode, smtp_login=smtp_login, smtp_password=smtp_password)
-        self.assertEqual(ret, dict())
-        (ret, peer, mail_from, rcpt_to, payload)=self.received[0]
-        self.assertEqual(self.payload, payload)
-        self.assertEqual(self.mail_from, mail_from)
-        self.assertEqual(self.rcpt_to, rcpt_to)
-        self.assertEqual('127.0.0.1', peer[0])
-
-    def test_send_to_a_wrong_port(self):
-        """send to a wrong port"""
-        self.smtp_server.close()
-        ret=send_mail(self.payload, self.mail_from, self.rcpt_to, smtpd_addr, smtpd_port, smtp_mode=smtp_mode, smtp_login=smtp_login, smtp_password=smtp_password)
-        self.assertEqual(type(ret), str)
-
-    def test_send_data_error(self):
-        """smtp server return error code"""
-        ret=send_mail(self.payload, 'data_error@foo.com', self.rcpt_to, smtpd_addr, smtp_bad_port, smtp_mode=smtp_mode, smtp_login=smtp_login, smtp_password=smtp_password)
-        self.assertEqual(type(ret), str)
-
-if __name__ == '__main__':
-    unittest.main()
-
diff --git a/pyzmail/tests/test_utils.py b/pyzmail/tests/test_utils.py
deleted file mode 100644
index e03d07d0d9..0000000000
--- a/pyzmail/tests/test_utils.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import unittest, doctest
-import pyzmail
-from pyzmail.utils import *
-
-class TestUtils(unittest.TestCase):
-
-    def setUp(self):
-        pass
-    
-    def test_nothing(self):
-        pass
-
-# Add doctest 
-def load_tests(loader, tests, ignore):
-    # this works with python 2.7 and 3.x
-    tests.addTests(doctest.DocTestSuite(pyzmail.utils))
-    return tests
-
-def additional_tests():
-    # Add doctest for python 2.6 and below
-    if sys.version_info<(2, 7):
-        return doctest.DocTestSuite(pyzmail.utils)
-    else:
-        return unittest.TestSuite()
diff --git a/pyzmail/utils.py b/pyzmail/utils.py
deleted file mode 100644
index 436e2a4c34..0000000000
--- a/pyzmail/utils.py
+++ /dev/null
@@ -1,155 +0,0 @@
-#
-# pyzmail/utils.py
-# (c) Alain Spineux <alain.spineux@gmail.com>
-# http://www.magiksys.net/pyzmail
-# Released under LGPL
-
-"""
-Various functions used by other modules
-@var invalid_chars_in_filename: a mix of characters not permitted in most used filesystems
-@var invalid_windows_name: a list of unauthorized filenames under Windows
-"""
-
-import sys
-
-invalid_chars_in_filename=b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f' \
-                          b'\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f' \
-                          b'<>:"/\\|?*%\''
-
-invalid_windows_name=[b'CON', b'PRN', b'AUX', b'NUL', b'COM1', b'COM2', b'COM3', 
-                      b'COM4', b'COM5', b'COM6', b'COM7', b'COM8', b'COM9', 
-                      b'LPT1', b'LPT2', b'LPT3', b'LPT4', b'LPT5', b'LPT6', b'LPT7',
-                      b'LPT8', b'LPT9' ]
-
-def sanitize_filename(filename, alt_name, alt_ext):
-    """
-    Convert the given filename into a name that should work on all 
-    platform. Remove non us-ascii characters, and drop invalid filename.
-    Use the I{alternative} filename if needed.
-    
-    @type filename: unicode or None
-    @param filename: the originale filename or None. Can be unicode.
-    @type alt_name: str
-    @param alt_name: the alternative filename if filename is None or useless
-    @type alt_ext: str
-    @param alt_ext: the alternative filename extension (including the '.')
-
-    @rtype: str
-    @returns: a valid filename.
-     
-    >>> sanitize_filename('document.txt', 'file', '.txt')
-    'document.txt'
-    >>> sanitize_filename('number1.txt', 'file', '.txt')
-    'number1.txt'
-    >>> sanitize_filename(None, 'file', '.txt')
-    'file.txt'
-    >>> sanitize_filename(u'R\\xe9pertoir.txt', 'file', '.txt')
-    'Rpertoir.txt'
-    >>> # the '\\xe9' has been removed
-    >>> sanitize_filename(u'\\xe9\\xe6.html', 'file', '.txt')
-    'file.html'
-    >>> # all non us-ascii characters have been removed, the alternative name
-    >>> # has been used the replace empty string. The originale extention
-    >>> # is still valid  
-    >>> sanitize_filename(u'COM1.txt', 'file', '.txt')
-    'COM1A.txt'
-    >>> # if name match an invalid name or assimilated then a A is added
-    """
-    
-    if not filename:
-        return alt_name+alt_ext
-
-    if ((sys.version_info<(3, 0) and isinstance(filename, str)) or \
-        (sys.version_info>=(3, 0) and isinstance(filename, str))):
-        filename=filename.encode('ascii', 'ignore')
-    
-    filename=filename.translate(None, invalid_chars_in_filename)
-    filename=filename.strip()
-        
-    upper=filename.upper()
-    for name in invalid_windows_name:
-        if upper==name:
-            filename=filename+b'A'
-            break
-        if upper.startswith(name+b'.'):
-            filename=filename[:len(name)]+b'A'+filename[len(name):]
-            break
-
-    if sys.version_info>=(3, 0):
-        # back to string
-        filename=filename.decode('us-ascii')
-
-    if filename.rfind('.')==0:
-        filename=alt_name+filename
-
-    return filename
-
-def handle_filename_collision(filename, filenames):
-    """
-    Avoid filename collision, add a sequence number to the name when required.
-    'file.txt' will be renamed into 'file-01.txt' then 'file-02.txt' ... 
-    until their is no more collision. The file is not added to the list.
-     
-    Windows don't make the difference between lower and upper case. To avoid
-    "case" collision, the function compare C{filename.lower()} to the list.
-    If you provide a list in lower case only, then any collisions will be avoided.     
-    
-    @type filename: str
-    @param filename: the filename
-    @type filenames: list or set
-    @param filenames: a list of filenames. 
-
-    @rtype: str
-    @returns: the I{filename} or the appropriately I{indexed} I{filename} 
-     
-    >>> handle_filename_collision('file.txt', [ ])
-    'file.txt'
-    >>> handle_filename_collision('file.txt', [ 'file.txt' ])
-    'file-01.txt'
-    >>> handle_filename_collision('file.txt', [ 'file.txt', 'file-01.txt',])
-    'file-02.txt'
-    >>> handle_filename_collision('foo', [ 'foo',])
-    'foo-01'
-    >>> handle_filename_collision('foo', [ 'foo', 'foo-01',])
-    'foo-02'
-    >>> handle_filename_collision('FOO', [ 'foo', 'foo-01',])
-    'FOO-02'
-    """
-    if filename.lower() in filenames:
-        try:
-            basename, ext=filename.rsplit('.', 1)
-            ext='.'+ext
-        except ValueError:
-            basename, ext=filename, '' 
-
-        i=1
-        while True:
-            filename='%s-%02d%s' % (basename, i, ext)
-            if filename.lower() not in filenames:
-                break
-            i+=1
-        
-    return filename
-
-def is_usascii(value):
-    """"
-    test if string contains us-ascii characters only
-    
-    >>> is_usascii('foo')
-    True
-    >>> is_usascii(u'foo')
-    True
-    >>> is_usascii(u'Fran\xe7ais')
-    False
-    >>> is_usascii('bad\x81')
-    False
-    """
-    try:
-        # if value is byte string, it will be decoded first using us-ascii
-        # and will generate UnicodeEncodeError, this is fine too
-        value.encode('us-ascii')
-    except UnicodeError:
-        return False
-    
-    return True
- 
\ No newline at end of file
diff --git a/pyzmail/version.py b/pyzmail/version.py
deleted file mode 100644
index 03e1b0c6dc..0000000000
--- a/pyzmail/version.py
+++ /dev/null
@@ -1 +0,0 @@
-__version__='1.0.3'

From 94f95c0ebacdd1c3a1f4748b8a45b9f7305bbab7 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 28 Oct 2024 11:57:06 -0500
Subject: [PATCH 127/656] fix: correct label for objectively false ipr
 disclosures (#8101)

---
 ietf/ipr/tests.py                 | 26 ++++++++++++++++++++++----
 ietf/templates/ipr/ipr_table.html |  7 +++++--
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py
index d72018f10b..67be278dbb 100644
--- a/ietf/ipr/tests.py
+++ b/ietf/ipr/tests.py
@@ -32,7 +32,8 @@
     GenericIprDisclosureFactory,
     IprDisclosureBaseFactory,
     IprDocRelFactory,
-    IprEventFactory
+    IprEventFactory,
+    ThirdPartyIprDisclosureFactory
 )
 from ietf.ipr.forms import DraftForm, HolderIprDisclosureForm
 from ietf.ipr.mail import (process_response_email, get_reply_to, get_update_submitter_emails,
@@ -104,9 +105,26 @@ def test_get_update_submitter_emails(self):
         self.assertTrue(messages[0].startswith('To: %s' % ipr.submitter_email))
         
     def test_showlist(self):
-        ipr = HolderIprDisclosureFactory()
-        r = self.client.get(urlreverse("ietf.ipr.views.showlist"))
-        self.assertContains(r, ipr.title)
+        for disc_factory_type in (HolderIprDisclosureFactory, GenericIprDisclosureFactory, ThirdPartyIprDisclosureFactory):
+            ipr = disc_factory_type(state_id="removed")
+            r = self.client.get(urlreverse("ietf.ipr.views.showlist"))
+            self.assertContains(r, ipr.title)
+            self.assertContains(r, "removed at the request of the submitter")
+            self.assertNotContains(r, "removed as objectively false")
+            ipr.state_id="posted"
+            ipr.save()
+            r = self.client.get(urlreverse("ietf.ipr.views.showlist"))
+            self.assertContains(r, ipr.title)
+            self.assertNotContains(r, "removed at the request of the submitter")
+            self.assertNotContains(r, "removed as objectively false")
+            ipr.state_id="removed_objfalse"
+            ipr.save()
+            r = self.client.get(urlreverse("ietf.ipr.views.showlist"))
+            self.assertContains(r, ipr.title)
+            self.assertNotContains(r, "removed at the request of the submitter")
+            self.assertContains(r, "removed as objectively false")
+            ipr.delete()
+
 
     def test_show_posted(self):
         ipr = HolderIprDisclosureFactory()
diff --git a/ietf/templates/ipr/ipr_table.html b/ietf/templates/ipr/ipr_table.html
index ce3982c987..d8247ea587 100644
--- a/ietf/templates/ipr/ipr_table.html
+++ b/ietf/templates/ipr/ipr_table.html
@@ -39,10 +39,13 @@
                                     </small>
                                 {% endif %}
                             {% endfor %}
-                        {% else %}
+                        {% elif ipr.state_id == 'removed' %}
                             <div>{{ ipr.title }}</div>
                             <small><i>This IPR disclosure was removed at the request of the submitter.</i></small>
-                        {% endif %}
+                        {% elif ipr.state_id == 'removed_objfalse' %}
+                            <div>{{ ipr.title }}</div>
+                            <small><i>This IPR disclosure was removed as objectively false.</i></small>
+                        {% endif %} {# Intentionally not emitting anything for any other states #}
                     </td>
                     {% if administrative_list == 'pending' %}
                         {% with ipr.get_latest_event_msgout as latest_msgout %}

From 8ec185cbb9b984aa526fffd620b347d5c094a0ce Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 28 Oct 2024 17:24:14 -0300
Subject: [PATCH 128/656] ci: fixed base image for prod (#8103)

---
 dev/build/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index e7cc510bc7..bede86fac9 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:latest
+FROM ghcr.io/ietf-tools/datatracker-app-base:2024102800
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive

From 50d4dc61fe9a6a8bb78e4af71f45b41081b396a5 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Mon, 28 Oct 2024 22:10:40 -0400
Subject: [PATCH 129/656] ci: Update build-base-app.yml with image versioning

---
 .github/workflows/build-base-app.yml | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index 3995b4b49b..3cea907fdd 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -14,12 +14,17 @@ jobs:
   publish:
     runs-on: ubuntu-latest
     permissions:
-      contents: read
+      contents: write
       packages: write
 
     steps:
     - uses: actions/checkout@v4
 
+    - name: Set Version
+      run: |
+        printf -v CURDATE '%(%Y%m%d%H%M)' -1
+        echo "IMGVERSION=$CURDATE" >> $GITHUB_OUTPUT
+
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3
     
@@ -42,4 +47,17 @@ jobs:
         file: docker/base.Dockerfile
         platforms: linux/amd64,linux/arm64
         push: true
-        tags: ghcr.io/ietf-tools/datatracker-app-base:latest
+        tags: |
+          ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }}
+          ghcr.io/ietf-tools/datatracker-app-base:latest
+
+    - name: Update version references
+      run: |
+        sed -i "1s/.*/FROM ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile
+
+    - name: Commit CHANGELOG.md
+      uses: stefanzweifel/git-auto-commit-action@v5
+      with:
+        branch: main
+        commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}'
+        file_pattern: dev/build/Dockerfile

From 88dfeff3f765b79d6086362cbdd250e889fe0815 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Mon, 28 Oct 2024 22:40:12 -0400
Subject: [PATCH 130/656] ci: read target base image from TARGET_BASE file

---
 .github/workflows/build-base-app.yml |  5 +++--
 .github/workflows/build.yml          | 10 +++++++++-
 .github/workflows/tests.yml          |  9 +++++++--
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index 3cea907fdd..f5bcc49033 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -23,7 +23,7 @@ jobs:
     - name: Set Version
       run: |
         printf -v CURDATE '%(%Y%m%d%H%M)' -1
-        echo "IMGVERSION=$CURDATE" >> $GITHUB_OUTPUT
+        echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV
 
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3
@@ -54,10 +54,11 @@ jobs:
     - name: Update version references
       run: |
         sed -i "1s/.*/FROM ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile
+        echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE
 
     - name: Commit CHANGELOG.md
       uses: stefanzweifel/git-auto-commit-action@v5
       with:
         branch: main
         commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}'
-        file_pattern: dev/build/Dockerfile
+        file_pattern: dev/build/Dockerfile dev/build/TARGET_BASE
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 756600cc1e..6041fe474b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -63,6 +63,7 @@ jobs:
       pkg_version: ${{ steps.buildvars.outputs.pkg_version }}
       from_tag: ${{ steps.semver.outputs.nextStrict }}
       to_tag: ${{ steps.semver.outputs.current }}
+      base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }}
 
     steps:   
     - uses: actions/checkout@v4
@@ -122,6 +123,11 @@ jobs:
           echo "::notice::Non-production build ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME"
         fi
 
+    - name: Get Base Image Target Version
+      id: baseimgversion
+      run: |
+        echo "base_image_version=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT
+
   # -----------------------------------------------------------------
   # TESTS
   # -----------------------------------------------------------------
@@ -133,6 +139,7 @@ jobs:
     needs: [prepare]
     with:
       ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }}
+      targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }}
 
   # -----------------------------------------------------------------
   # RELEASE
@@ -200,6 +207,7 @@ jobs:
         SKIP_TESTS: ${{ github.event.inputs.skiptests }}
         DEBIAN_FRONTEND: noninteractive
         BROWSERSLIST_IGNORE_OLD_DATA: 1
+        TARGETBASE: ${{ needs.prepare.outputs.base_image_version }}
       with:
         host: ${{ steps.azlaunch.outputs.ipaddr }}
         port: 22
@@ -310,7 +318,7 @@ jobs:
           echo "=========================================================================="
           echo "Collecting statics..."
           echo "=========================================================================="
-          sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:latest sh dev/build/collectstatics.sh
+          sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:$TARGETBASE sh dev/build/collectstatics.sh
           echo "Pushing statics..."
           cd static
           aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 64db03db09..7da7ec897f 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -8,12 +8,17 @@ on:
         default: false
         required: true
         type: boolean
+      targetBaseVersion:
+        description: 'Target Base Image Version'
+        default: latest
+        required: true
+        type: string
 
 jobs:
   tests-python:
     name: Python Tests
     runs-on: ubuntu-latest
-    container: ghcr.io/ietf-tools/datatracker-app-base:latest
+    container: ghcr.io/ietf-tools/datatracker-app-base:${{ github.event.inputs.targetBaseVersion }}
     
     services:
       db:
@@ -114,7 +119,7 @@ jobs:
   tests-playwright-legacy:
     name: Playwright Legacy Tests
     runs-on: ubuntu-latest
-    container: ghcr.io/ietf-tools/datatracker-app-base:latest
+    container: ghcr.io/ietf-tools/datatracker-app-base:${{ github.event.inputs.targetBaseVersion }}
     strategy:
       fail-fast: false
       matrix:

From d1cffe7a145f61d366404574ab0aeac82eb148a9 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Mon, 28 Oct 2024 22:45:07 -0400
Subject: [PATCH 131/656] ci: fix base image version command

---
 .github/workflows/build-base-app.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index f5bcc49033..0c3d07a378 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -22,7 +22,7 @@ jobs:
 
     - name: Set Version
       run: |
-        printf -v CURDATE '%(%Y%m%d%H%M)' -1
+        printf -v CURDATE '%(%Y%m%dT%H%M)T' -1
         echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV
 
     - name: Set up QEMU

From 96060fcf6f863e4bcb117db6cd95a9ef5f918f1a Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Mon, 28 Oct 2024 22:59:29 -0400
Subject: [PATCH 132/656] ci: fix build-base-app workflow

---
 .github/workflows/build-base-app.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index 0c3d07a378..11958dda72 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -53,7 +53,7 @@ jobs:
 
     - name: Update version references
       run: |
-        sed -i "1s/.*/FROM ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile
+        sed -i "1s/.*/FROM ghcr.io\/ietf-tools\/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile
         echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE
 
     - name: Commit CHANGELOG.md

From 8a4d02026853a8118e4472e766447bf8bcae8952 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Mon, 28 Oct 2024 23:27:30 -0400
Subject: [PATCH 133/656] ci: use build account to push base image changes

---
 .github/workflows/build-base-app.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index 11958dda72..609b641ef9 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -19,6 +19,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GH_COMMON_TOKEN }}
 
     - name: Set Version
       run: |

From b926178e62e030db126f7aa291ca86e921d20af1 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 29 Oct 2024 11:18:31 -0500
Subject: [PATCH 134/656] fix: quicker calculation of status from draft text
 (#8111)

* fix: quicker calculation of status from draft text

* chore: remove unused import

* fix: only read a small prefix of draft text when needed
---
 ietf/doc/models.py    | 15 +++++++++++----
 ietf/doc/views_doc.py | 11 +++++------
 ietf/utils/draft.py   | 18 ++++++++++++++++++
 3 files changed, 34 insertions(+), 10 deletions(-)

diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index 639e6ca857..077502db11 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -530,7 +530,7 @@ def replaces(self):
     def replaced_by(self):
         return set([ r.document for r in self.related_that("replaces") ])
 
-    def text(self):
+    def text(self, size = -1):
         path = self.get_file_name()
         root, ext =  os.path.splitext(path)
         txtpath = root+'.txt'
@@ -538,14 +538,21 @@ def text(self):
             path = txtpath
         try:
             with io.open(path, 'rb') as file:
-                raw = file.read()
+                raw = file.read(size)
         except IOError:
             return None
+        text = None
         try:
             text = raw.decode('utf-8')
         except UnicodeDecodeError:
-            text = raw.decode('latin-1')
-        #
+            for back in range(1,4):
+                try:
+                    text = raw[:-back].decode('utf-8')
+                    break
+                except UnicodeDecodeError:
+                    pass
+            if text is None:
+                text = raw.decode('latin-1')
         return text
 
     def text_or_error(self):
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 915dcebde5..50c60aefc5 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -84,7 +84,7 @@
 from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
 from ietf.utils import markup_txt, log, markdown
-from ietf.utils.draft import PlaintextDraft
+from ietf.utils.draft import get_status_from_draft_text
 from ietf.utils.meetecho import MeetechoAPIError, SlidesManager
 from ietf.utils.response import permission_denied
 from ietf.utils.text import maybe_split
@@ -2261,12 +2261,11 @@ def idnits2_state(request, name, rev=None):
     elif doc.intended_std_level:
         doc.deststatus = doc.intended_std_level.name
     else:
-        text = doc.text()
+         # 10000 is a conservative prefix on number of utf-8 encoded bytes to 
+         # cover at least the first 10 lines of characters
+        text = doc.text(size=10000)
         if text:
-            parsed_draft = PlaintextDraft(
-                text=doc.text(), source=name, name_from_source=False
-            )
-            doc.deststatus = parsed_draft.get_status()
+            doc.deststatus = get_status_from_draft_text(text)
         else:
             doc.deststatus = "Unknown"
     return render(
diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py
index a1e79760ea..50add5abba 100755
--- a/ietf/utils/draft.py
+++ b/ietf/utils/draft.py
@@ -131,6 +131,24 @@ def acronym_match(s, l):
     #_debug(" s:%s; l:%s => %s; %s" % (s, l, acronym, s==acronym)) 
     return s == acronym
 
+def get_status_from_draft_text(text):
+
+    # Take prefix to shortcut work over very large drafts
+    # 5000 is conservatively much more than a full page of characters and we
+    # only want the first 10 lines.
+    text = text.strip()[:5000] # Take prefix to shortcut work over very large drafts 
+    text = re.sub(".\x08", "", text)    # Get rid of inkribbon backspace-emphasis
+    text = text.replace("\r\n", "\n")   # Convert DOS to unix
+    text = text.replace("\r", "\n")     # Convert MAC to unix
+    lines = text.split("\n")[:10]
+    status = None
+    for line in lines:
+        status_match = re.search(r"^\s*Intended [Ss]tatus:\s*(.*?)   ", line)
+        if status_match:
+            status = status_match.group(1)
+            break
+    return status
+
 class Draft:
     """Base class for drafts
 

From 0d43663e9bb99b45ee779b469f4c38ca14cfb208 Mon Sep 17 00:00:00 2001
From: NGPixel <NGPixel@users.noreply.github.com>
Date: Tue, 29 Oct 2024 16:43:21 +0000
Subject: [PATCH 135/656] ci: update base image target version to 20241029T1632

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 dev/build/TARGET_BASE

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index bede86fac9..728b37156c 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:2024102800
+FROM ghcr.io/ietf-tools/datatracker-app-base:20241029T1632
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
new file mode 100644
index 0000000000..2f8aac20f9
--- /dev/null
+++ b/dev/build/TARGET_BASE
@@ -0,0 +1 @@
+20241029T1632

From 5f685f45fb5f94f714a9c7a650d111d2e2bc16eb Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 29 Oct 2024 12:52:18 -0400
Subject: [PATCH 136/656] ci: fix tests.yml workflow

---
 .github/workflows/tests.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 7da7ec897f..4e0b31fb14 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -18,7 +18,7 @@ jobs:
   tests-python:
     name: Python Tests
     runs-on: ubuntu-latest
-    container: ghcr.io/ietf-tools/datatracker-app-base:${{ github.event.inputs.targetBaseVersion }}
+    container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }}
     
     services:
       db:
@@ -48,7 +48,7 @@ jobs:
           exit 1
         fi
         echo "Running tests..."
-        if [[ "x${{ github.event.inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then
+        if [[ "x${{ inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then
           echo "Lower coverage failures will be ignored."
           HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage
         else
@@ -119,7 +119,7 @@ jobs:
   tests-playwright-legacy:
     name: Playwright Legacy Tests
     runs-on: ubuntu-latest
-    container: ghcr.io/ietf-tools/datatracker-app-base:${{ github.event.inputs.targetBaseVersion }}
+    container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }}
     strategy:
       fail-fast: false
       matrix:

From dd3b9bd2f9f7fe6b558df3be500447a6ac609a29 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 29 Oct 2024 14:06:57 -0400
Subject: [PATCH 137/656] ci: Update build.yml workflow to support parallel
 builds

---
 .github/workflows/build.yml | 58 ++++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 6041fe474b..2e38e125d3 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -166,13 +166,15 @@ jobs:
 
     - name: Launch build VM
       id: azlaunch
+      timeout-minutes: 10
       run: |
         echo "Authenticating to Azure..."
         az login --service-principal -u ${{ secrets.AZ_BUILD_APP_ID }} -p ${{ secrets.AZ_BUILD_PWD }} --tenant ${{ secrets.AZ_BUILD_TENANT_ID }}
+        
         echo "Creating VM..."
         vminfo=$(az vm create \
           --resource-group ghaDatatracker \
-          --name tmpGhaBuildVM \
+          --name tmpGhaBuildVM-${{ github.run_number }} \
           --image Ubuntu2204 \
           --admin-username azureuser \
           --generate-ssh-keys \
@@ -183,11 +185,19 @@ jobs:
           --os-disk-size-gb 100 \
           --eviction-policy Delete \
           --nic-delete-option Delete \
+          --os-disk-delete-option Delete \
           --output tsv \
           --query "publicIpAddress")
         echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT"
         echo "VM Public IP: $vminfo"
         cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key
+        
+        echo "Fetching SSH host public keys..."
+        until ssh-keyscan -t rsa $vminfo 2> /dev/null
+        do
+          echo "Will try again in 5 seconds..."
+          sleep 5
+        done
         ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts
 
     - name: Remote SSH into Build VM
@@ -353,29 +363,29 @@ jobs:
       shell: pwsh
       run: |
         echo "Destroying VM..."        
-        az vm delete -g ghaDatatracker -n tmpGhaBuildVM --yes --force-deletion true
-
-        $resourceOrderRemovalOrder = [ordered]@{
-            "Microsoft.Compute/virtualMachines" = 0
-            "Microsoft.Compute/disks" = 1
-            "Microsoft.Network/networkInterfaces" = 2
-            "Microsoft.Network/publicIpAddresses" = 3
-            "Microsoft.Network/networkSecurityGroups" = 4
-            "Microsoft.Network/virtualNetworks" = 5
-        }
-        echo "Fetching remaining resources..."
-        $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json
-
-        $orderedResources = $resources 
-            | Sort-Object @{
-                Expression = {$resourceOrderRemovalOrder[$_.type]}
-                Descending = $False
-            }
-
-        echo "Deleting remaining resources..."
-        $orderedResources | ForEach-Object {
-            az resource delete --resource-group ghaDatatracker --ids $_.id --verbose
-        }
+        az vm delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }} --yes --force-deletion true
+
+        # $resourceOrderRemovalOrder = [ordered]@{
+        #     "Microsoft.Compute/virtualMachines" = 0
+        #     "Microsoft.Compute/disks" = 1
+        #     "Microsoft.Network/networkInterfaces" = 2
+        #     "Microsoft.Network/publicIpAddresses" = 3
+        #     "Microsoft.Network/networkSecurityGroups" = 4
+        #     "Microsoft.Network/virtualNetworks" = 5
+        # }
+        # echo "Fetching remaining resources..."
+        # $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json
+
+        # $orderedResources = $resources 
+        #     | Sort-Object @{
+        #         Expression = {$resourceOrderRemovalOrder[$_.type]}
+        #         Descending = $False
+        #     }
+
+        # echo "Deleting remaining resources..."
+        # $orderedResources | ForEach-Object {
+        #     az resource delete --resource-group ghaDatatracker --ids $_.id --verbose
+        # }
 
         echo "Logout from Azure..."
         az logout

From e530e0af6aa2a4fe60e77a015e1969cc642abd55 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 29 Oct 2024 15:24:56 -0400
Subject: [PATCH 138/656] ci: add disable selenium toggle to build workflow

---
 .github/workflows/build.yml | 1 +
 .github/workflows/tests.yml | 9 +++++++++
 2 files changed, 10 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2e38e125d3..71730ebb52 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -139,6 +139,7 @@ jobs:
     needs: [prepare]
     with:
       ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }}
+      skipSelenium: true
       targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }}
 
   # -----------------------------------------------------------------
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 4e0b31fb14..682300f2ab 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -8,6 +8,11 @@ on:
         default: false
         required: true
         type: boolean
+      skipSelenium:
+        description: 'Skip Selenium Tests'
+        default: false
+        required: true
+        type: boolean
       targetBaseVersion:
         description: 'Target Base Image Version'
         default: latest
@@ -47,6 +52,10 @@ jobs:
           echo "Model changes without migrations found."
           exit 1
         fi
+        if [[ "x${{ inputs.skipSelenium }}" == "xtrue" ]]; then
+          echo "Disable selenium tests..."
+          sudo rm /usr/bin/geckodriver
+        fi
         echo "Running tests..."
         if [[ "x${{ inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then
           echo "Lower coverage failures will be ignored."

From 0e392af870dea31496074aec0d26b1b838f337b7 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 29 Oct 2024 15:32:14 -0400
Subject: [PATCH 139/656] ci: remove sudo from tests workflow

---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 682300f2ab..7ad3c82364 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -54,7 +54,7 @@ jobs:
         fi
         if [[ "x${{ inputs.skipSelenium }}" == "xtrue" ]]; then
           echo "Disable selenium tests..."
-          sudo rm /usr/bin/geckodriver
+          rm /usr/bin/geckodriver
         fi
         echo "Running tests..."
         if [[ "x${{ inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then

From 84f2affad9b211545f5e72f708f12db2ee5d461a Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 29 Oct 2024 16:07:05 -0400
Subject: [PATCH 140/656] ci: Update build.yml workflow

---
 .github/workflows/build.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 71730ebb52..96ba962a78 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -158,6 +158,7 @@ jobs:
       PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
       FROM_TAG: ${{needs.prepare.outputs.from_tag}}
       TO_TAG: ${{needs.prepare.outputs.to_tag}}
+      TARGET_BASE: ${{needs.prepare.outputs.base_image_version}}
 
     steps:
     - uses: actions/checkout@v4
@@ -218,7 +219,7 @@ jobs:
         SKIP_TESTS: ${{ github.event.inputs.skiptests }}
         DEBIAN_FRONTEND: noninteractive
         BROWSERSLIST_IGNORE_OLD_DATA: 1
-        TARGETBASE: ${{ needs.prepare.outputs.base_image_version }}
+        TARGET_BASE: ${{ env.TARGET_BASE }}
       with:
         host: ${{ steps.azlaunch.outputs.ipaddr }}
         port: 22
@@ -329,7 +330,7 @@ jobs:
           echo "=========================================================================="
           echo "Collecting statics..."
           echo "=========================================================================="
-          sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:$TARGETBASE sh dev/build/collectstatics.sh
+          sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:$TARGET_BASE sh dev/build/collectstatics.sh
           echo "Pushing statics..."
           cd static
           aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors

From 01700ab37bef2a40a7abf0a810841263badc0513 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 29 Oct 2024 16:16:45 -0400
Subject: [PATCH 141/656] ci: fix build workflow

---
 .github/workflows/build.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 96ba962a78..d613328312 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -330,7 +330,8 @@ jobs:
           echo "=========================================================================="
           echo "Collecting statics..."
           echo "=========================================================================="
-          sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:$TARGET_BASE sh dev/build/collectstatics.sh
+          echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}"
+          sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh
           echo "Pushing statics..."
           cd static
           aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors

From d13b47064f9c592603a82a9f31d0d6cb2f7b5b4f Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 29 Oct 2024 22:47:31 -0400
Subject: [PATCH 142/656] docs: update dev/tests/debug.sh

---
 dev/tests/debug.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/tests/debug.sh b/dev/tests/debug.sh
index 8ed28371ae..d87c504bb9 100644
--- a/dev/tests/debug.sh
+++ b/dev/tests/debug.sh
@@ -3,7 +3,7 @@
 # This script recreate the same environment used during tests on GitHub Actions
 # and drops you into a terminal at the point where the actual tests would be run.
 #
-# Refer to https://github.com/ietf-tools/datatracker/blob/main/.github/workflows/build.yml#L141-L155
+# Refer to https://github.com/ietf-tools/datatracker/blob/main/.github/workflows/tests.yml#L47-L66
 # for the commands to run next.
 #
 # Simply type "exit" + ENTER to exit and shutdown this test environment.

From 11fd2a560123070e9dea91e9a13bd28ad2fa412e Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 29 Oct 2024 23:55:47 -0400
Subject: [PATCH 143/656] ci: Update build.yml workflow

---
 .github/workflows/build.yml | 32 +++++++-------------------------
 1 file changed, 7 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d613328312..bf340c5233 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -363,33 +363,15 @@ jobs:
 
     - name: Destroy Build VM + resources
       if: always()
-      shell: pwsh
       run: |
-        echo "Destroying VM..."        
+        echo "Terminate VM..."        
         az vm delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }} --yes --force-deletion true
-
-        # $resourceOrderRemovalOrder = [ordered]@{
-        #     "Microsoft.Compute/virtualMachines" = 0
-        #     "Microsoft.Compute/disks" = 1
-        #     "Microsoft.Network/networkInterfaces" = 2
-        #     "Microsoft.Network/publicIpAddresses" = 3
-        #     "Microsoft.Network/networkSecurityGroups" = 4
-        #     "Microsoft.Network/virtualNetworks" = 5
-        # }
-        # echo "Fetching remaining resources..."
-        # $resources = az resource list --resource-group ghaDatatracker | ConvertFrom-Json
-
-        # $orderedResources = $resources 
-        #     | Sort-Object @{
-        #         Expression = {$resourceOrderRemovalOrder[$_.type]}
-        #         Descending = $False
-        #     }
-
-        # echo "Deleting remaining resources..."
-        # $orderedResources | ForEach-Object {
-        #     az resource delete --resource-group ghaDatatracker --ids $_.id --verbose
-        # }
-
+        echo "Delete Public IP..."
+        az resource delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }}PublicIP --resource-type "Microsoft.Network/publicIPAddresses"
+        echo "Delete Network Security Group..."
+        az resource delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }}NSG --resource-type "Microsoft.Network/networkSecurityGroups"
+        echo "Delete Virtual Network..."
+        az resource delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }}VNET --resource-type "Microsoft.Network/virtualNetworks"
         echo "Logout from Azure..."
         az logout
         

From 1b4f481fbe32847eb81b6ae9a746b022f8c78ae3 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 30 Oct 2024 18:56:32 -0400
Subject: [PATCH 144/656] ci: fix tests.yml workflow

---
 .github/workflows/tests.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 7ad3c82364..bb767513c9 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -11,12 +11,12 @@ on:
       skipSelenium:
         description: 'Skip Selenium Tests'
         default: false
-        required: true
+        required: false
         type: boolean
       targetBaseVersion:
         description: 'Target Base Image Version'
         default: latest
-        required: true
+        required: false
         type: string
 
 jobs:

From 087bf1e77c7e677e3db402737d686ab5b1c6ec75 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Sun, 3 Nov 2024 14:34:16 +0000
Subject: [PATCH 145/656] fix: prefer PDF in (+refactor) materials_document
 view (#8136)

* fix: prefer PDF in materials_document(); refactor

* style: Black

* refactor: split urls entry into simpler pair

It seems Django cannot reverse a URL pattern that uses "|" to
combine options when one inclues a named variable and the other
does not.

* test: test extension choice

* fix: fix test failures

* refactor: get rid of io.open()

* refactor: reunite url patterns

Adds option for a trailing "/" when using an extension,
which was not previously supported but should be somewhere
between harmless and a feature.
---
 ietf/meeting/tests_views.py | 57 +++++++++++++++++++++++++++++++++++--
 ietf/meeting/urls.py        |  2 +-
 ietf/meeting/views.py       | 54 +++++++++++++++++++++--------------
 3 files changed, 88 insertions(+), 25 deletions(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 0d291f12bc..d227708aa0 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -125,8 +125,12 @@ def tearDown(self):
         settings.MEETINGHOST_LOGO_PATH = self.saved_meetinghost_logo_path
         super().tearDown()
 
-    def write_materials_file(self, meeting, doc, content, charset="utf-8"):
-        path = os.path.join(self.materials_dir, "%s/%s/%s" % (meeting.number, doc.type_id, doc.uploaded_filename))
+    def write_materials_file(self, meeting, doc, content, charset="utf-8", with_ext=None):
+        if with_ext is None:
+            filename = doc.uploaded_filename
+        else:
+            filename = Path(doc.uploaded_filename).with_suffix(with_ext)
+        path = os.path.join(self.materials_dir, "%s/%s/%s" % (meeting.number, doc.type_id, filename))
 
         dirname = os.path.dirname(path)
         if not os.path.exists(dirname):
@@ -753,7 +757,56 @@ def test_materials_has_edit_links(self):
         )
         self.assertEqual(len(q(f'a[href^="{edit_url}#session"]')), 1, f'Link to session_details page for {acro}')
 
+    def test_materials_document_extension_choice(self):
+        def _url(**kwargs):
+            return urlreverse("ietf.meeting.views.materials_document", kwargs=kwargs)
+
+        presentation = SessionPresentationFactory(
+            document__rev="00",
+            document__name="slides-whatever",
+            document__uploaded_filename="slides-whatever-00.txt",
+            document__type_id="slides",
+            document__states=(("reuse_policy", "single"),)
+        )
+        session = presentation.session
+        meeting = session.meeting
+        # This is not a realistic set of files to exist, but is useful for testing. Normally,
+        # we'd have _either_ txt, pdf, or pptx + pdf.
+        self.write_materials_file(meeting, presentation.document, "Hi I'm a txt", with_ext=".txt")
+        self.write_materials_file(meeting, presentation.document, "Hi I'm a pptx", with_ext=".pptx")
+
+        # with no rev, prefers the uploaded_filename
+        r = self.client.get(_url(document="slides-whatever", num=meeting.number))  # no rev
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content.decode(), "Hi I'm a txt")
+        
+        # with a rev, prefers pptx because it comes first alphabetically
+        r = self.client.get(_url(document="slides-whatever-00", num=meeting.number))
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content.decode(), "Hi I'm a pptx")
 
+        # now create a pdf
+        self.write_materials_file(meeting, presentation.document, "Hi I'm a pdf", with_ext=".pdf")
+
+        # with no rev, still prefers uploaded_filename
+        r = self.client.get(_url(document="slides-whatever", num=meeting.number))  # no rev
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content.decode(), "Hi I'm a txt")
+
+        # pdf should be preferred with a rev
+        r = self.client.get(_url(document="slides-whatever-00", num=meeting.number))
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.content.decode(), "Hi I'm a pdf")
+        
+        # and explicit extensions should, of course, be respected
+        for ext in ["pdf", "pptx", "txt"]:
+            r = self.client.get(_url(document="slides-whatever-00", num=meeting.number, ext=f".{ext}"))
+            self.assertEqual(r.status_code, 200)
+            self.assertEqual(r.content.decode(), f"Hi I'm a {ext}")
+        
+        # and 404 should come up if the ext is not found
+        r = self.client.get(_url(document="slides-whatever-00", num=meeting.number, ext=".docx"))
+        self.assertEqual(r.status_code, 404)
 
     def test_materials_editable_groups(self):
         meeting = make_meeting_test_data()
diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py
index f2e65578ec..42a5de623c 100644
--- a/ietf/meeting/urls.py
+++ b/ietf/meeting/urls.py
@@ -83,7 +83,7 @@ def get_redirect_url(self, *args, **kwargs):
     url(r'^week-view(?:.html)?/?$', AgendaRedirectView.as_view(pattern_name='agenda', permanent=True)),
     url(r'^materials(?:.html)?/?$', views.materials),
     url(r'^request_minutes/?$', views.request_minutes),
-    url(r'^materials/%(document)s((?P<ext>\.[a-z0-9]+)|/)?$' % settings.URL_REGEXPS, views.materials_document),
+    url(r'^materials/%(document)s(?P<ext>\.[a-z0-9]+)?/?$' % settings.URL_REGEXPS, views.materials_document),
     url(r'^session/?$', views.materials_editable_groups),
     url(r'^proceedings(?:.html)?/?$', views.proceedings),
     url(r'^proceedings(?:.html)?/finalize/?$', views.finalize_proceedings),
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index bce6ccd1bf..64fe73f484 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -4,7 +4,6 @@
 
 import csv
 import datetime
-import glob
 import io
 import itertools
 import json
@@ -20,6 +19,7 @@
 from collections import OrderedDict, Counter, deque, defaultdict, namedtuple
 from functools import partialmethod
 import jsonschema
+from pathlib import Path
 from urllib.parse import parse_qs, unquote, urlencode, urlsplit, urlunsplit
 from tempfile import mkstemp
 from wsgiref.handlers import format_date_time
@@ -250,7 +250,14 @@ def _get_materials_doc(meeting, name):
 
 @cache_page(1 * 60)
 def materials_document(request, document, num=None, ext=None):
-    meeting=get_meeting(num,type_in=['ietf','interim'])
+    """Materials document view
+
+    :param request: Django request
+    :param document: Name of document without an extension
+    :param num: meeting number
+    :param ext: extension including preceding '.'
+    """
+    meeting = get_meeting(num, type_in=["ietf", "interim"])
     num = meeting.number
     try:
         doc, rev = _get_materials_doc(meeting=meeting, name=document)
@@ -258,32 +265,34 @@ def materials_document(request, document, num=None, ext=None):
         raise Http404("No such document for meeting %s" % num)
 
     if not rev:
-        filename = doc.get_file_name()
+        filename = Path(doc.get_file_name())
     else:
-        filename = os.path.join(doc.get_file_path(), document)
+        filename = Path(doc.get_file_path()) / document
     if ext:
-        if not filename.endswith(ext):
-            name, _ = os.path.splitext(filename)
-            filename = name + ext
-    else:
-        filenames = glob.glob(filename+'.*')
-        if filenames:
-            filename = filenames[0]
-    _, basename = os.path.split(filename)
-    if not os.path.exists(filename):
-        raise Http404("File not found: %s" % filename)
+        filename = filename.with_suffix(ext)
+    elif filename.suffix == "":
+        # If we don't already have an extension, try to add one
+        ext_choices = {
+            # Construct a map from suffix to full filename
+            fn.suffix: fn
+            for fn in sorted(filename.parent.glob(filename.stem + ".*"))
+        }
+        if len(ext_choices) > 0:
+            if ".pdf" in ext_choices:
+                filename = ext_choices[".pdf"]
+            else:
+                filename = list(ext_choices.values())[0]
+    if not filename.exists():
+        raise Http404(f"File not found: {filename}")
 
     old_proceedings_format = meeting.number.isdigit() and int(meeting.number) <= 96
     if settings.MEETING_MATERIALS_SERVE_LOCALLY or old_proceedings_format:
-        with io.open(filename, 'rb') as file:
-            bytes = file.read()
-        
+        bytes = filename.read_bytes()
         mtype, chset = get_mime_type(bytes)
         content_type = "%s; charset=%s" % (mtype, chset)
 
-        file_ext = os.path.splitext(filename)
-        if len(file_ext) == 2 and file_ext[1] == '.md' and mtype == 'text/plain':
-            sorted_accept = sort_accept_tuple(request.META.get('HTTP_ACCEPT'))
+        if filename.suffix == ".md" and mtype == "text/plain":
+            sorted_accept = sort_accept_tuple(request.META.get("HTTP_ACCEPT"))
             for atype in sorted_accept:
                 if atype[0] == "text/markdown":
                     content_type = content_type.replace("plain", "markdown", 1)
@@ -293,7 +302,7 @@ def materials_document(request, document, num=None, ext=None):
                         "minimal.html",
                         {
                             "content": markdown.markdown(bytes.decode(encoding=chset)),
-                            "title": basename,
+                            "title": filename.name,
                         },
                     )
                     content_type = content_type.replace("plain", "html", 1)
@@ -302,11 +311,12 @@ def materials_document(request, document, num=None, ext=None):
                     break
 
         response = HttpResponse(bytes, content_type=content_type)
-        response['Content-Disposition'] = 'inline; filename="%s"' % basename
+        response["Content-Disposition"] = f'inline; filename="{filename.name}"'
         return response
     else:
         return HttpResponseRedirect(redirect_to=doc.get_href(meeting=meeting))
 
+
 @login_required
 def materials_editable_groups(request, num=None):
     meeting = get_meeting(num)

From deb58a1a632fe2a4e40e16e2a869f977da881e3e Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Sun, 3 Nov 2024 16:23:29 +0000
Subject: [PATCH 146/656] fix: disallow header wrapping (#8134)

---
 ietf/templates/base.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/templates/base.html b/ietf/templates/base.html
index ceb1d2df08..aa44955527 100644
--- a/ietf/templates/base.html
+++ b/ietf/templates/base.html
@@ -46,7 +46,7 @@
                     {% endif %}
                 </a>
                 <div class="collapse navbar-collapse" id="navbar-collapse">
-                    <ul class="nav navbar-nav">
+                    <ul class="nav navbar-nav flex-nowrap">
                         {% include "base/menu.html" with flavor="top" %}
                     </ul>
                 </div>

From 19d80ffc833ba05e351f170b22e04ebe1aa6e493 Mon Sep 17 00:00:00 2001
From: Tero Kivinen <kivinen@iki.fi>
Date: Sun, 3 Nov 2024 18:29:53 +0200
Subject: [PATCH 147/656] fix: fix statistics for RFCs to show properly.
 (#8139)

---
 ietf/stats/views.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/ietf/stats/views.py b/ietf/stats/views.py
index 23e90ea23c..ea73d9f4fc 100644
--- a/ietf/stats/views.py
+++ b/ietf/stats/views.py
@@ -596,14 +596,17 @@ def build_document_stats_url(stats_type_override=Ellipsis, get_overrides=None):
 
         elif any(stats_type == t[0] for t in possible_yearly_stats_types):
 
-            person_filters = Q(documentauthor__document__type="draft")
-
             # filter persons
             rfc_state = State.objects.get(type="rfc", slug="published")
             if document_type == "rfc":
+                person_filters = Q(documentauthor__document__type="rfc")
                 person_filters &= Q(documentauthor__document__states=rfc_state)
             elif document_type == "draft":
+                person_filters = Q(documentauthor__document__type="draft")
                 person_filters &= ~Q(documentauthor__document__states=rfc_state)
+            else:
+                person_filters = Q(documentauthor__document__type="rfc")
+                person_filters |= Q(documentauthor__document__type="draft")
 
             doc_years = defaultdict(set)
 
@@ -625,8 +628,8 @@ def build_document_stats_url(stats_type_override=Ellipsis, get_overrides=None):
                         ).values_list("source", flat=True)[:1]
                     )
                 )
-                .values_list("draft", "time")
-                .order_by("draft")
+                .values_list("doc", "time")
+                .order_by("doc")
             )
 
             for doc_id, time in rfcevent_qs.iterator():
@@ -663,9 +666,11 @@ def build_document_stats_url(stats_type_override=Ellipsis, get_overrides=None):
                 for name, affiliation, doc in name_affiliation_doc_set:
                     a = aliases.get(affiliation, affiliation)
                     if a:
-                        for year in doc_years.get(doc):
-                            if years_from <= year <= years_to:
-                                bins[(year, a)].add(name)
+                        years = doc_years.get(doc)
+                        if years:
+                            for year in years:
+                                if years_from <= year <= years_to:
+                                    bins[(year, a)].add(name)
 
                 add_labeled_top_series_from_bins(chart_data, bins, limit=8)
 
@@ -724,9 +729,11 @@ def build_document_stats_url(stats_type_override=Ellipsis, get_overrides=None):
                     continent_name = country_to_continent.get(country_name, "")
 
                     if continent_name:
-                        for year in doc_years.get(doc):
-                            if years_from <= year <= years_to:
-                                bins[(year, continent_name)].add(name)
+                        years = doc_years.get(doc)
+                        if years:
+                            for year in years:
+                                if years_from <= year <= years_to:
+                                    bins[(year, continent_name)].add(name)
 
                 add_labeled_top_series_from_bins(chart_data, bins, limit=8)
 

From 7a5b152c5e7e5da7780705c6a3d51f16b51bb79a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 4 Nov 2024 15:24:35 +0000
Subject: [PATCH 148/656] feat: async refresh of agenda data (#8146)

---
 ietf/meeting/tasks.py       | 12 ++++++++++++
 ietf/meeting/tests_views.py | 27 ++++++++++++++++-----------
 ietf/meeting/views.py       | 33 ++++++++++++++++++++++++++-------
 3 files changed, 54 insertions(+), 18 deletions(-)
 create mode 100644 ietf/meeting/tasks.py

diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py
new file mode 100644
index 0000000000..43cbb0a75f
--- /dev/null
+++ b/ietf/meeting/tasks.py
@@ -0,0 +1,12 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+#
+# Celery task definitions
+#
+from celery import shared_task
+
+from .views import generate_agenda_data
+
+
+@shared_task
+def agenda_data_refresh():
+    generate_agenda_data(force_refresh=True)
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index d227708aa0..4f83cf0cd7 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -26,6 +26,7 @@
 from django.urls import reverse as urlreverse
 from django.conf import settings
 from django.contrib.auth.models import User
+from django.core.serializers.json import DjangoJSONEncoder
 from django.test import Client, override_settings
 from django.db.models import F, Max
 from django.http import QueryDict, FileResponse
@@ -49,7 +50,7 @@
 from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting
 from ietf.meeting.utils import create_recording, get_next_sequence, bluesheet_data
 from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule
-from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose
+from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose, generate_agenda_data
 from ietf.name.models import SessionStatusName, ImportantDateName, RoleName, ProceedingsMaterialTypeName
 from ietf.utils.decorators import skip_coverage
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
@@ -245,30 +246,34 @@ def test_meeting_agenda(self):
 
         # Agenda API tests
         # -> Meeting data
-        r = self.client.get(urlreverse("ietf.meeting.views.api_get_agenda_data", kwargs=dict(num=meeting.number)))
-        self.assertEqual(r.status_code, 200)  
-        rjson = json.loads(r.content.decode("utf8"))
-        self.assertJSONEqual(
-            r.content.decode("utf8"),
+        # First, check that the generation function does the right thing
+        generated_data = generate_agenda_data(meeting.number)
+        self.assertEqual(
+            generated_data,
             {
                 "meeting": {
                     "number": meeting.number,
                     "city": meeting.city,
                     "startDate": meeting.date.isoformat(),
                     "endDate": meeting.end_date().isoformat(),
-                    "updated": rjson.get("meeting").get("updated"), # Just expect the value to exist
+                    "updated": generated_data.get("meeting").get("updated"),  # Just expect the value to exist
                     "timezone": meeting.time_zone,
                     "infoNote": meeting.agenda_info_note,
                     "warningNote": meeting.agenda_warning_note
                 },
-                "categories": rjson.get("categories"), # Just expect the value to exist
+                "categories": generated_data.get("categories"),  # Just expect the value to exist
                 "isCurrentMeeting": True,
-                "usesNotes": False, # make_meeting_test_data sets number=72
-                "schedule": rjson.get("schedule"), # Just expect the value to exist
+                "usesNotes": False,  # make_meeting_test_data sets number=72
+                "schedule": generated_data.get("schedule"),  # Just expect the value to exist
                 "floors": []
             }
         )
-        # -> Session Materials
+        with patch("ietf.meeting.views.generate_agenda_data", return_value=generated_data):
+            r = self.client.get(urlreverse("ietf.meeting.views.api_get_agenda_data", kwargs=dict(num=meeting.number)))
+        self.assertEqual(r.status_code, 200)  
+        # json.dumps using the DjangoJSONEncoder to handle timestamps consistently
+        self.assertJSONEqual(r.content.decode("utf8"), json.dumps(generated_data, cls=DjangoJSONEncoder))
+        # -> Session MaterialM
         r = self.client.get(urlreverse("ietf.meeting.views.api_get_session_materials", kwargs=dict(session_id=session.id)))
         self.assertEqual(r.status_code, 200)  
         rjson = json.loads(r.content.decode("utf8"))
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 64fe73f484..628125776f 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -25,6 +25,7 @@
 from wsgiref.handlers import format_date_time
 
 from django import forms
+from django.core.cache import caches
 from django.shortcuts import render, redirect, get_object_or_404
 from django.http import (HttpResponse, HttpResponseRedirect, HttpResponseForbidden,
                          HttpResponseNotFound, Http404, HttpResponseBadRequest,
@@ -1657,8 +1658,16 @@ def agenda(request, num=None, name=None, base=None, ext=None, owner=None, utc=""
         }
     })
 
-@cache_page(5 * 60)
-def api_get_agenda_data (request, num=None):
+
+def generate_agenda_data(num=None, force_refresh=False):
+    """Generate data for the api_get_agenda_data endpoint
+    
+    :num: meeting number
+    :force_refresh: True to force a refresh of the cache
+    """
+    cache = caches["default"]
+    cache_timeout = 6 * 60
+
     meeting = get_ietf_meeting(num)
     if meeting is None:
         raise Http404("No such full IETF meeting")
@@ -1667,6 +1676,12 @@ def api_get_agenda_data (request, num=None):
     else:
         pass
 
+    cache_key = f"generate_agenda_data_{meeting.number}"
+    if not force_refresh:
+        cached_value = cache.get(cache_key)
+        if cached_value is not None:
+            return cached_value
+
     # Select the schedule to show
     schedule = get_schedule(meeting, None)
 
@@ -1685,10 +1700,8 @@ def api_get_agenda_data (request, num=None):
 
     # Get Floor Plans
     floors = FloorPlan.objects.filter(meeting=meeting).order_by('order')
-
-    #debug.show('all([(item.acronym,item.session.order_number,item.session.order_in_meeting()) for item in filtered_assignments])')
-
-    return JsonResponse({
+    
+    result = {
         "meeting": {
             "number": schedule.meeting.number,
             "city": schedule.meeting.city,
@@ -1704,7 +1717,13 @@ def api_get_agenda_data (request, num=None):
         "usesNotes": meeting.uses_notes(),
         "schedule": list(map(agenda_extract_schedule, filtered_assignments)),
         "floors": list(map(agenda_extract_floorplan, floors))
-    })
+    }
+    cache.set(cache_key, result, timeout=cache_timeout)
+    return result
+
+
+def api_get_agenda_data(request, num=None):
+    return JsonResponse(generate_agenda_data(num, force_refresh=False))
 
 
 def api_get_session_materials(request, session_id=None):

From 7293e10a640a81851b0f3b12493f55f0f03114f7 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 4 Nov 2024 16:46:23 +0000
Subject: [PATCH 149/656] feat: log start of gunicorn request (#8154)

* feat: log start of gunicorn request

* feat: add query string
---
 dev/build/gunicorn.conf.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py
index 48661ef77d..89d3dd58d3 100644
--- a/dev/build/gunicorn.conf.py
+++ b/dev/build/gunicorn.conf.py
@@ -47,3 +47,18 @@
         }
     }
 }
+
+def pre_request(worker, req):
+    client_ip = "-"
+    cf_ray = "-"
+    for (header, value) in req.headers:
+        header = header.lower()
+        if header == "cf-connecting-ip":
+            client_ip = value
+        elif header == "cf-ray":
+            cf_ray = value
+    if req.query:
+        path = f"{req.path}?{req.query}"
+    else:
+        path = req.path
+    worker.log.info(f"gunicorn starting to process {req.method} {path} (client_ip={client_ip}, cf_ray={cf_ray})")

From de494790b643f1875e125c149459f7bbadf245a0 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Tue, 5 Nov 2024 15:03:21 +0000
Subject: [PATCH 150/656] fix: Selenium tests via scroll_and_click (#8150)

* fix: selenium tests scroll_and_click

* fix: reduce default timeout to 5 seconds

* fix: also use scroll_and_click on test_upcoming_materials_modal

* fix: remove conditional check on restoring scroll CSS

* fix: restore conditional check on restoring scroll CSS

* chore: code comments and adding jstest.py to coverage ignore
---
 ietf/meeting/tests_js.py | 17 +++++++---------
 ietf/settings.py         |  1 +
 ietf/utils/jstest.py     | 44 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 10 deletions(-)

diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py
index b15aa70e73..a184a7c6d0 100644
--- a/ietf/meeting/tests_js.py
+++ b/ietf/meeting/tests_js.py
@@ -249,7 +249,9 @@ def test_edit_meeting_schedule(self):
         self.assertTrue(s1_element.is_displayed())  # should still be displayed
         self.assertIn('hidden-parent', s1_element.get_attribute('class'),
                       'Session should be hidden when parent disabled')
-        s1_element.click()  # try to select
+        
+        self.scroll_and_click((By.CSS_SELECTOR, '#session{}'.format(s1.pk)))
+
         self.assertNotIn('selected', s1_element.get_attribute('class'),
                          'Session should not be selectable when parent disabled')
 
@@ -299,9 +301,9 @@ def test_edit_meeting_schedule(self):
                         'Session s1 should have moved to second meeting day')
 
         # swap timeslot column - put session in a differently-timed timeslot
-        self.driver.find_element(By.CSS_SELECTOR,
+        self.scroll_and_click((By.CSS_SELECTOR,
             '.day .swap-timeslot-col[data-timeslot-pk="{}"]'.format(slot1b.pk)
-        ).click()  # open modal on the second timeslot for room1
+        ))  # open modal on the second timeslot for room1
         self.assertTrue(self.driver.find_element(By.CSS_SELECTOR, "#swap-timeslot-col-modal").is_displayed())
         self.driver.find_element(By.CSS_SELECTOR,
             '#swap-timeslot-col-modal input[name="target_timeslot"][value="{}"]'.format(slot4.pk)
@@ -1373,13 +1375,8 @@ def test_upcoming_materials_modal(self):
         self.assertFalse(modal_div.is_displayed())
 
         # Click the 'materials' button
-        open_modal_button = self.wait.until(
-            expected_conditions.element_to_be_clickable(
-                (By.CSS_SELECTOR, '[data-bs-target="#modal-%s"]' % slug)
-            ),
-            'Modal open button not found or not clickable',
-        )
-        open_modal_button.click()
+        open_modal_button_locator = (By.CSS_SELECTOR, '[data-bs-target="#modal-%s"]' % slug)
+        self.scroll_and_click(open_modal_button_locator)
         self.wait.until(
             expected_conditions.visibility_of(modal_div),
             'Modal did not become visible after clicking open button',
diff --git a/ietf/settings.py b/ietf/settings.py
index a1a7fee102..0c57d87d11 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -598,6 +598,7 @@ def skip_unreadable_post(record):
     "ietf/review/import_from_review_tool.py",
     "ietf/utils/patch.py",
     "ietf/utils/test_data.py",
+    "ietf/utils/jstest.py",
 ]
 
 # These are code line regex patterns
diff --git a/ietf/utils/jstest.py b/ietf/utils/jstest.py
index 157f97912b..215d78d65f 100644
--- a/ietf/utils/jstest.py
+++ b/ietf/utils/jstest.py
@@ -12,6 +12,8 @@
     from selenium import webdriver
     from selenium.webdriver.firefox.service import Service
     from selenium.webdriver.firefox.options import Options
+    from selenium.webdriver.support.ui import WebDriverWait
+    from selenium.webdriver.support import expected_conditions
     from selenium.webdriver.common.by import By
 except ImportError as e:
     skip_selenium = True
@@ -87,6 +89,48 @@ def scroll_to_element(self, element):
         # actions = ActionChains(self.driver)
         # actions.move_to_element(element).perform()
 
+    def scroll_and_click(self, element_locator, timeout_seconds=5):
+        """
+        Selenium has restrictions around clicking elements outside the viewport, so
+        this wrapper encapsulates the boilerplate of forcing scrolling and clicking.
+
+        :param element_locator: A two item tuple of a Selenium locator eg `(By.CSS_SELECTOR, '#something')`
+        """
+
+        # so that we can restore the state of the webpage after clicking
+        original_html_scroll_behaviour_to_restore = self.driver.execute_script('return document.documentElement.style.scrollBehavior')
+        original_html_overflow_to_restore = self.driver.execute_script('return document.documentElement.style.overflow')
+
+        original_body_scroll_behaviour_to_restore = self.driver.execute_script('return document.body.style.scrollBehavior')
+        original_body_overflow_to_restore = self.driver.execute_script('return document.body.style.overflow')
+
+        self.driver.execute_script('document.documentElement.style.scrollBehavior = "auto"')
+        self.driver.execute_script('document.documentElement.style.overflow = "auto"')
+
+        self.driver.execute_script('document.body.style.scrollBehavior = "auto"')
+        self.driver.execute_script('document.body.style.overflow = "auto"')
+
+        element = self.driver.find_element(element_locator[0], element_locator[1])
+        self.scroll_to_element(element)
+
+        # Note that Selenium itself seems to have multiple definitions of 'clickable'.
+        # You might expect that the following wait for the 'element_to_be_clickable'
+        # would confirm that the following .click() would succeed but it doesn't.
+        # That's why the preceeding code attempts to force scrolling to bring the
+        # element into the viewport to allow clicking.
+        WebDriverWait(self.driver, timeout_seconds).until(expected_conditions.element_to_be_clickable(element_locator))
+
+        element.click()
+
+        if original_html_scroll_behaviour_to_restore:
+            self.driver.execute_script(f'document.documentElement.style.scrollBehavior = "{original_html_scroll_behaviour_to_restore}"')
+        if original_html_overflow_to_restore:
+            self.driver.execute_script(f'document.documentElement.style.overflow = "{original_html_overflow_to_restore}"')
+
+        if original_body_scroll_behaviour_to_restore:
+            self.driver.execute_script(f'document.body.style.scrollBehavior = "{original_body_scroll_behaviour_to_restore}"')
+        if original_body_overflow_to_restore:
+            self.driver.execute_script(f'document.body.style.overflow = "{original_body_overflow_to_restore}"')
 
 class presence_of_element_child_by_css_selector:
     """Wait for presence of a child of a WebElement matching a CSS selector

From 7b749f1623212411717de9567581b53b61c46aa3 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Wed, 6 Nov 2024 08:42:54 +0000
Subject: [PATCH 151/656] fix: Agenda mobile goto now (#8160)

* fix: agenda mobile goto now

* fix: manually close the dropdown
---
 client/agenda/AgendaMobileBar.vue | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/client/agenda/AgendaMobileBar.vue b/client/agenda/AgendaMobileBar.vue
index 78af96e255..63611e21c2 100644
--- a/client/agenda/AgendaMobileBar.vue
+++ b/client/agenda/AgendaMobileBar.vue
@@ -3,11 +3,13 @@
   n-dropdown(
     :options='jumpToDayOptions'
     size='huge'
+    :show='isDropdownOpenRef'
     :show-arrow='true'
     trigger='click'
     @select='jumpToDay'
+    @clickoutside='handleCloseDropdown'
     )
-    button
+    button(@click='handleOpenDropdown')
       i.bi.bi-arrow-down-circle
   button(@click='agendaStore.$patch({ filterShown: true })')
     i.bi.bi-funnel
@@ -28,7 +30,7 @@
 </template>
 
 <script setup>
-import { computed, h } from 'vue'
+import { computed, h, ref } from 'vue'
 import {
   NBadge,
   NDropdown,
@@ -61,7 +63,8 @@ function optionToLink(opts){
       {
         class: 'dropdown-link',
         'data-testid': 'mobile-link',
-        href: `#${key}`
+        href: `#${key}`,
+        onClick: () => jumpToDay(key)
       },
       [
         h(
@@ -77,6 +80,12 @@ function optionToLink(opts){
   }
 }
 
+const isDropdownOpenRef = ref(false)
+
+const handleOpenDropdown = () => isDropdownOpenRef.value = true
+
+const handleCloseDropdown = () => isDropdownOpenRef.value = false
+
 const jumpToDayOptions = computed(() => {
   const days = []
   if (agendaStore.isMeetingLive) {
@@ -124,6 +133,7 @@ function jumpToDay (dayId) {
   } else {
     document.getElementById(dayId)?.scrollIntoView(true)
   }
+  isDropdownOpenRef.value = false
 }
 
 function downloadIcs (key) {

From afbe6aa429bbc1499e1d6ba54e19359fa7ccccff Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 6 Nov 2024 08:49:54 +0000
Subject: [PATCH 152/656] fix: queue tasks in on_commit hook (#8149)

* fix: dispatch tasks in on_commit hook

* test: fix test
---
 ietf/community/models.py |  7 +++++--
 ietf/community/tests.py  |  4 +++-
 ietf/sync/views.py       | 21 +++++++++++++++++----
 3 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/ietf/community/models.py b/ietf/community/models.py
index 4d820eb0f1..834f596355 100644
--- a/ietf/community/models.py
+++ b/ietf/community/models.py
@@ -3,7 +3,7 @@
 
 
 from django.conf import settings
-from django.db import models
+from django.db import models, transaction
 from django.db.models import signals
 from django.urls import reverse as urlreverse
 
@@ -117,7 +117,10 @@ def notify_events(sender, instance, **kwargs):
     # start a Celery task during tests. To prevent this, don't queue a celery task if we're running
     # tests.
     if settings.SERVER_MODE != "test":
-        notify_event_to_subscribers_task.delay(event_id=instance.pk)
+        # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
+        transaction.on_commit(
+            lambda: notify_event_to_subscribers_task.delay(event_id=instance.pk)
+        )
 
 
 signals.post_save.connect(notify_events)
diff --git a/ietf/community/tests.py b/ietf/community/tests.py
index 5329267d8e..743242f11b 100644
--- a/ietf/community/tests.py
+++ b/ietf/community/tests.py
@@ -431,8 +431,10 @@ def test_subscription_for_group(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
 
+    # Mock out the on_commit call so we can tell whether the task was actually queued
+    @mock.patch("ietf.submit.views.transaction.on_commit", side_effect=lambda x: x())
     @mock.patch("ietf.community.models.notify_event_to_subscribers_task")
-    def test_notification_signal_receiver(self, mock_notify_task):
+    def test_notification_signal_receiver(self, mock_notify_task, mock_on_commit):
         """Saving a DocEvent should notify subscribers
         
         This implicitly tests that notify_events is hooked up to the post_save signal.
diff --git a/ietf/sync/views.py b/ietf/sync/views.py
index a2b5f62427..1b7ff2a36c 100644
--- a/ietf/sync/views.py
+++ b/ietf/sync/views.py
@@ -8,6 +8,7 @@
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
+from django.db import transaction
 from django.http import HttpResponse, HttpResponseRedirect, Http404
 from django.shortcuts import render
 from django.utils import timezone
@@ -79,16 +80,28 @@ def notify(request, org, notification):
     if request.method == "POST":
         if notification == "index":
             log("Queuing RFC Editor index sync from notify view POST")
-            tasks.rfc_editor_index_update_task.delay()
+            # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
+            transaction.on_commit(
+                lambda: tasks.rfc_editor_index_update_task.delay()
+            )
         elif notification == "queue":
             log("Queuing RFC Editor queue sync from notify view POST")
-            tasks.rfc_editor_queue_updates_task.delay()
+            # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
+            transaction.on_commit(
+                lambda: tasks.rfc_editor_queue_updates_task.delay()
+            )
         elif notification == "changes":
             log("Queuing IANA changes sync from notify view POST")
-            tasks.iana_changes_update_task.delay()
+            # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
+            transaction.on_commit(
+                lambda: tasks.iana_changes_update_task.delay()
+            )
         elif notification == "protocols":
             log("Queuing IANA protocols sync from notify view POST")
-            tasks.iana_protocols_update_task.delay()
+            # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
+            transaction.on_commit(
+                lambda: tasks.iana_protocols_update_task.delay()
+            )
 
         return HttpResponse("OK", content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET)
 

From 5348aefef927ec452a9df4e0d9a119126a0ab94d Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Wed, 6 Nov 2024 08:52:04 +0000
Subject: [PATCH 153/656] feat: show existing recordings on materials page
 (#8102)

* feat: show existing recordings on materials page

* chore: notes and recordings tests WIP

* chore: test session recordings

* feat: label all session recording urls as meetecho
---
 ietf/meeting/tests_views.py                   | 34 ++++++++
 .../meeting/session_details_panel.html        | 87 +++++++++----------
 2 files changed, 77 insertions(+), 44 deletions(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 4f83cf0cd7..0ab7afbdfa 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -407,6 +407,40 @@ def test_meeting_agenda(self):
         r = self.client.get(urlreverse('floor-plan', kwargs=dict(num=meeting.number)))
         self.assertEqual(r.status_code, 200)
 
+    def test_session_recordings_via_factories(self):
+        session = SessionFactory(meeting__type_id="ietf")
+        self.assertEqual(session.meetecho_recording_name, "")
+        self.assertEqual(len(session.recordings()), 0)
+        url = urlreverse("ietf.meeting.views.session_details", kwargs=dict(num=session.meeting.number, acronym=session.group.acronym))
+        r = self.client.get(url)
+        q = PyQuery(r.content)
+        # debug.show("q('#notes_and_recordings_1')")
+        self.assertEqual(len(q("#notes_and_recordings_1 tr")), 1)
+        link = q("#notes_and_recordings_1 tr a")
+        self.assertEqual(len(link), 1)
+        self.assertEqual(link[0].attrib['href'], str(session.session_recording_url()))
+
+        session.meetecho_recording_name = 'my_test_session_name'
+        session.save()
+        r = self.client.get(url)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q("#notes_and_recordings_1 tr")), 1)
+        links = q("#notes_and_recordings_1 tr a")
+        self.assertEqual(len(links), 1)
+        self.assertEqual(links[0].attrib['href'], session.session_recording_url())
+
+        new_recording_url = "https://www.youtube.com/watch?v=jNQXAC9IVRw"
+        new_recording_title = "Me at the zoo"
+        create_recording(session, new_recording_url, new_recording_title)
+        r = self.client.get(url)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q("#notes_and_recordings_1 tr")), 2)
+        links = q("#notes_and_recordings_1 tr a")
+        self.assertEqual(len(links), 2)
+        self.assertEqual(links[0].attrib['href'], new_recording_url)
+        self.assertIn(new_recording_title, links[0].text_content())
+        #debug.show("q('#notes_and_recordings_1')")
+
     def test_agenda_ical_next_meeting_type(self):
         # start with no upcoming IETF meetings, just an interim
         MeetingFactory(
diff --git a/ietf/templates/meeting/session_details_panel.html b/ietf/templates/meeting/session_details_panel.html
index 52aeaaa8c1..a0f5884b9d 100644
--- a/ietf/templates/meeting/session_details_panel.html
+++ b/ietf/templates/meeting/session_details_panel.html
@@ -320,51 +320,50 @@ <h3 class="mt-4">Notes and recordings</h3>
                 </tr>
             {% endif %}
             {# Recordings #}
-            {% if session.has_recordings %}
-                {% with session.recordings as recordings %}
-                    {% if recordings %}
-                        {# There's no guaranteed order, so this is a bit messy: #}
-                        {# First, the audio recordings, if any #}
-                        {% for r in recordings %}
-                            {% if r.get_href and 'audio' in r.get_href %}
-                                <tr>
-                                    <td>
-                                        <a href="{{ r.get_href }}"><i class="bi bi-file-play"></i> {{ r.title }}</a>
-                                    </td>
-                                </tr>
-                            {% endif %}
-                        {% endfor %}
-                        {# Then the youtube recordings #}
-                        {% for r in recordings %}
-                            {% if r.get_href and 'youtu' in r.get_href %}
-                                <tr>
-                                    <td>
-                                        <a href="{{ r.get_href }}"><i class="bi bi-file-slides"></i> {{ r.title }}</a>
-                                    </td>
-                                </tr>
-                            {% endif %}
-                        {% endfor %}
-                        {# Finally, any other recordings #}
-                        {% for r in recordings %}
-                            {% if r.get_href and not 'audio' in r.get_href and not 'youtu' in r.get_href %}
-                                <tr>
-                                    <td>
-                                        <a href="{{ r.get_href }}"><i class="bi bi-file-play"></i> {{ r.title }}</a>
-                                    </td>
-                                </tr>
-                            {% endif %}
-                        {% endfor %}
-                    {% endif %}
-                {% endwith %}
-                {% if session.video_stream_url %}
-                    <tr>
-                        <td>
-                            <a href="{{ session.session_recording_url }}">
-                                <i class="bi bi-file-slides"></i> Session recording
-                            </a>
-                        </td>
-                    </tr>
+            {% with session.recordings as recordings %}
+                {% if recordings %}
+                    {# There's no guaranteed order, so this is a bit messy: #}
+                    {# First, the audio recordings, if any #}
+                    {% for r in recordings %}
+                        {% if r.get_href and 'audio' in r.get_href %}
+                            <tr>
+                                <td>
+                                    <a href="{{ r.get_href }}"><i class="bi bi-file-play"></i> {{ r.title }}</a>
+                                </td>
+                            </tr>
+                        {% endif %}
+                    {% endfor %}
+                    {# Then the youtube recordings #}
+                    {% for r in recordings %}
+                        {% if r.get_href and 'youtu' in r.get_href %}
+                            <tr>
+                                <td>
+                                    <a href="{{ r.get_href }}"><i class="bi bi-file-slides"></i> {{ r.title }}</a>
+                                </td>
+                            </tr>
+                        {% endif %}
+                    {% endfor %}
+                    {# Finally, any other recordings #}
+                    {% for r in recordings %}
+                        {% if r.get_href and not 'audio' in r.get_href and not 'youtu' in r.get_href %}
+                            <tr>
+                                <td>
+                                    <a href="{{ r.get_href }}"><i class="bi bi-file-play"></i> {{ r.title }}</a>
+                                </td>
+                            </tr>
+                        {% endif %}
+                    {% endfor %}
                 {% endif %}
+            {% endwith %}
+            {% if session.session_recording_url %}
+                <tr>
+                    <td>
+                        <a href="{{ session.session_recording_url }}">
+                            <i class="bi bi-file-slides"></i>
+                            Meetecho session recording
+                        </a>
+                    </td>
+                </tr>
             {% endif %}
             </tbody>
         </table>

From 701e8a50164cfd824707c29c6fd1dac2a85e0b9d Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 6 Nov 2024 07:42:44 -0500
Subject: [PATCH 154/656] ci: fix build.yml clone branch

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index bf340c5233..405a793266 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -287,7 +287,7 @@ jobs:
           sudo mkdir -p /workspace
           sudo chown azureuser /workspace
           cd /workspace
-          gh repo clone ietf-tools/datatracker -- --depth=1 --no-tags
+          gh repo clone ietf-tools/datatracker -- --depth=1 --no-tags --branch ${{ github.ref_name }}
           cd datatracker
 
           if [ "$SKIP_TESTS" = "false" ] || [ "$GITHUB_REF_NAME" = "release" ] ; then

From 277f1aafeb169068295d69c974bc312971a8fa7e Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 6 Nov 2024 16:38:37 +0000
Subject: [PATCH 155/656] fix: search for correct ids in test (#8168)

---
 ietf/meeting/tests_views.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 0ab7afbdfa..afc72da9f7 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -414,9 +414,9 @@ def test_session_recordings_via_factories(self):
         url = urlreverse("ietf.meeting.views.session_details", kwargs=dict(num=session.meeting.number, acronym=session.group.acronym))
         r = self.client.get(url)
         q = PyQuery(r.content)
-        # debug.show("q('#notes_and_recordings_1')")
-        self.assertEqual(len(q("#notes_and_recordings_1 tr")), 1)
-        link = q("#notes_and_recordings_1 tr a")
+        # debug.show("q(f'#notes_and_recordings_{session.pk}')")
+        self.assertEqual(len(q(f"#notes_and_recordings_{session.pk} tr")), 1)
+        link = q(f"#notes_and_recordings_{session.pk} tr a")
         self.assertEqual(len(link), 1)
         self.assertEqual(link[0].attrib['href'], str(session.session_recording_url()))
 
@@ -424,8 +424,8 @@ def test_session_recordings_via_factories(self):
         session.save()
         r = self.client.get(url)
         q = PyQuery(r.content)
-        self.assertEqual(len(q("#notes_and_recordings_1 tr")), 1)
-        links = q("#notes_and_recordings_1 tr a")
+        self.assertEqual(len(q(f"#notes_and_recordings_{session.pk} tr")), 1)
+        links = q(f"#notes_and_recordings_{session.pk} tr a")
         self.assertEqual(len(links), 1)
         self.assertEqual(links[0].attrib['href'], session.session_recording_url())
 
@@ -434,12 +434,12 @@ def test_session_recordings_via_factories(self):
         create_recording(session, new_recording_url, new_recording_title)
         r = self.client.get(url)
         q = PyQuery(r.content)
-        self.assertEqual(len(q("#notes_and_recordings_1 tr")), 2)
-        links = q("#notes_and_recordings_1 tr a")
+        self.assertEqual(len(q(f"#notes_and_recordings_{session.pk} tr")), 2)
+        links = q(f"#notes_and_recordings_{session.pk} tr a")
         self.assertEqual(len(links), 2)
         self.assertEqual(links[0].attrib['href'], new_recording_url)
         self.assertIn(new_recording_title, links[0].text_content())
-        #debug.show("q('#notes_and_recordings_1')")
+        #debug.show("q(f'#notes_and_recordings_{session_pk}')")
 
     def test_agenda_ical_next_meeting_type(self):
         # start with no upcoming IETF meetings, just an interim

From 8c787b54f9246663ddd1bfc7b4a2faee2fb7d459 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 7 Nov 2024 17:41:36 +0000
Subject: [PATCH 156/656] fix: control test meeting date to be in the past
 (#8177)

---
 ietf/meeting/tests_views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index afc72da9f7..0647da52ab 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -408,7 +408,7 @@ def test_meeting_agenda(self):
         self.assertEqual(r.status_code, 200)
 
     def test_session_recordings_via_factories(self):
-        session = SessionFactory(meeting__type_id="ietf")
+        session = SessionFactory(meeting__type_id="ietf", meeting__date=date_today()-datetime.timedelta(days=180))
         self.assertEqual(session.meetecho_recording_name, "")
         self.assertEqual(len(session.recordings()), 0)
         url = urlreverse("ietf.meeting.views.session_details", kwargs=dict(num=session.meeting.number, acronym=session.group.acronym))

From a35aa67c703f00a5ea5b6bca68f4bb2129872b93 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 8 Nov 2024 11:55:47 +0000
Subject: [PATCH 157/656] chore: clarify transaction-related comments (#8180)

---
 ietf/community/models.py |  2 +-
 ietf/submit/tests.py     |  4 ++--
 ietf/submit/views.py     |  6 ++++--
 ietf/sync/views.py       | 12 ++++++++----
 4 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/ietf/community/models.py b/ietf/community/models.py
index 834f596355..0407730107 100644
--- a/ietf/community/models.py
+++ b/ietf/community/models.py
@@ -117,7 +117,7 @@ def notify_events(sender, instance, **kwargs):
     # start a Celery task during tests. To prevent this, don't queue a celery task if we're running
     # tests.
     if settings.SERVER_MODE != "test":
-        # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
+        # Wrap in on_commit in case a transaction is open
         transaction.on_commit(
             lambda: notify_event_to_subscribers_task.delay(event_id=instance.pk)
         )
diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index 21ed6672df..6a56839177 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -2334,8 +2334,8 @@ def test_cancel_preapproval(self):
         self.assertEqual(len(Preapproval.objects.filter(name=preapproval.name)), 0)
 
 
-# Transaction.on_commit() requires use of TransactionTestCase, but that has a performance penalty. Replace it
-# with a no-op for testing purposes.
+# Transaction.on_commit() interacts badly with TestCase's transaction behavior. Replace it
+# with a pass-through for testing purposes.
 @mock.patch.object(transaction, 'on_commit', lambda x: x())
 @override_settings(IDTRACKER_BASE_URL='https://datatracker.example.com')
 class ApiSubmissionTests(BaseSubmitTestCase):
diff --git a/ietf/submit/views.py b/ietf/submit/views.py
index f710bdeb0c..2b9b55c00e 100644
--- a/ietf/submit/views.py
+++ b/ietf/submit/views.py
@@ -90,7 +90,8 @@ def upload_submission(request):
             clear_existing_files(form)
             save_files(form)
             create_submission_event(request, submission, desc="Uploaded submission")
-            # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
+            # Wrap in on_commit in case a transaction is open
+            # (As of 2024-11-08, this only runs in a transaction during tests)
             transaction.on_commit(
                 lambda: process_uploaded_submission_task.delay(submission.pk)
             )
@@ -166,7 +167,8 @@ def err(code, error, messages=None):
                 save_files(form)
                 create_submission_event(request, submission, desc="Uploaded submission through API")
 
-                # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
+                # Wrap in on_commit in case a transaction is open
+                # (As of 2024-11-08, this only runs in a transaction during tests)
                 transaction.on_commit(
                     lambda: process_and_accept_uploaded_submission_task.delay(submission.pk)
                 )
diff --git a/ietf/sync/views.py b/ietf/sync/views.py
index 1b7ff2a36c..62306c6fc1 100644
--- a/ietf/sync/views.py
+++ b/ietf/sync/views.py
@@ -80,25 +80,29 @@ def notify(request, org, notification):
     if request.method == "POST":
         if notification == "index":
             log("Queuing RFC Editor index sync from notify view POST")
-            # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
+            # Wrap in on_commit in case a transaction is open
+            # (As of 2024-11-08, this only runs in a transaction during tests)
             transaction.on_commit(
                 lambda: tasks.rfc_editor_index_update_task.delay()
             )
         elif notification == "queue":
             log("Queuing RFC Editor queue sync from notify view POST")
-            # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
+            # Wrap in on_commit in case a transaction is open
+            # (As of 2024-11-08, this only runs in a transaction during tests)
             transaction.on_commit(
                 lambda: tasks.rfc_editor_queue_updates_task.delay()
             )
         elif notification == "changes":
             log("Queuing IANA changes sync from notify view POST")
-            # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
+            # Wrap in on_commit in case a transaction is open
+            # (As of 2024-11-08, this only runs in a transaction during tests)
             transaction.on_commit(
                 lambda: tasks.iana_changes_update_task.delay()
             )
         elif notification == "protocols":
             log("Queuing IANA protocols sync from notify view POST")
-            # Wrap in on_commit so the delayed task cannot start until the view is done with the DB
+            # Wrap in on_commit in case a transaction is open
+            # (As of 2024-11-08, this only runs in a transaction during tests)
             transaction.on_commit(
                 lambda: tasks.iana_protocols_update_task.delay()
             )

From b4fd3bfbb7e8f56ab20297d0cbb3437d91f739a4 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 13 Nov 2024 09:51:43 -0600
Subject: [PATCH 158/656] chore: remove dead code (#8207)

* chore: remove dead SYNC_BIN_PATH

* chore: remove unused Status class name attribute

* chore: remove unused import
---
 ietf/status/models.py | 1 -
 ietf/sync/views.py    | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/ietf/status/models.py b/ietf/status/models.py
index b3f97d989e..c551920275 100644
--- a/ietf/status/models.py
+++ b/ietf/status/models.py
@@ -8,7 +8,6 @@
 import debug                            # pyflakes:ignore
 
 class Status(models.Model):
-    name = 'Status'
     
     date = models.DateTimeField(default=timezone.now)
     slug = models.SlugField(blank=False, null=False, unique=True)
diff --git a/ietf/sync/views.py b/ietf/sync/views.py
index 62306c6fc1..bd539d4e18 100644
--- a/ietf/sync/views.py
+++ b/ietf/sync/views.py
@@ -2,7 +2,6 @@
 # -*- coding: utf-8 -*-
 
 import datetime
-import os
 import json
 
 from django.conf import settings
@@ -23,8 +22,6 @@
 from ietf.utils.response import permission_denied
 
 
-SYNC_BIN_PATH = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../bin"))
-
 #@role_required('Secretariat', 'IANA', 'RFC Editor')
 def discrepancies(request):
     sections = find_discrepancies()

From a70171dfd9354be46a28d6bc8d632df5bf07287f Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 13 Nov 2024 12:51:04 -0400
Subject: [PATCH 159/656] chore: remove more dead code (#8208)

* chore: remove unused settings

* chore: remove unused ietf.utils.aliases / config
---
 ietf/settings.py      |  9 +----
 ietf/utils/aliases.py | 82 -------------------------------------------
 2 files changed, 1 insertion(+), 90 deletions(-)
 delete mode 100644 ietf/utils/aliases.py

diff --git a/ietf/settings.py b/ietf/settings.py
index 0c57d87d11..30f4b367c6 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -744,8 +744,6 @@ def skip_unreadable_post(record):
 IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes"
 IANA_SYNC_PROTOCOLS_URL = "https://www.iana.org/protocols/"
 
-RFC_TEXT_RSYNC_SOURCE="ftp.rfc-editor.org::rfcs-text-only"
-
 RFC_EDITOR_SYNC_PASSWORD="secret"
 RFC_EDITOR_SYNC_NOTIFICATION_URL = "https://www.rfc-editor.org/parser/parser.php"
 RFC_EDITOR_GROUP_NOTIFICATION_EMAIL = "webmaster@rfc-editor.org"
@@ -972,7 +970,6 @@ def skip_unreadable_post(record):
 # ==============================================================================
 
 
-RSYNC_BINARY = '/usr/bin/rsync'
 YANGLINT_BINARY = '/usr/bin/yanglint'
 DE_GFM_BINARY = '/usr/bin/de-gfm.ruby2.5'
 
@@ -1014,7 +1011,6 @@ def skip_unreadable_post(record):
 # CHAT_ARCHIVE_URL_PATTERN = 'https://www.ietf.org/jabber/logs/{chat_room_name}?C=M;O=D'
 
 PYFLAKES_DEFAULT_ARGS= ["ietf", ]
-VULTURE_DEFAULT_ARGS= ["ietf", ]
 
 # Automatic Scheduling
 #
@@ -1062,8 +1058,6 @@ def skip_unreadable_post(record):
 TEST_DATA_DIR = os.path.abspath(BASE_DIR + "/../test/data")
 
 
-POSTCONFIRM_PATH   = "/a/postconfirm/wrapper"
-
 USER_PREFERENCE_DEFAULTS = {
     "expires_soon"  : "14",
     "new_enough"    : "14",
@@ -1078,6 +1072,7 @@ def skip_unreadable_post(record):
     "@ietf.org$",
 ]
 
+# Configuration for django-markup
 MARKUP_SETTINGS = {
     'restructuredtext': {
         'settings_overrides': {
@@ -1091,8 +1086,6 @@ def skip_unreadable_post(record):
     }
 }
 
-MAILMAN_LIB_DIR = '/usr/lib/mailman'
-
 # This is the number of seconds required between subscribing to an ietf
 # mailing list and datatracker account creation being accepted
 LIST_ACCOUNT_DELAY = 60*60*25           # 25 hours
diff --git a/ietf/utils/aliases.py b/ietf/utils/aliases.py
deleted file mode 100644
index 9f9aebc0b9..0000000000
--- a/ietf/utils/aliases.py
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/usr/bin/env python
-# Copyright The IETF Trust 2013-2020, All Rights Reserved
-# -*- coding: utf-8 -*-
-# -*- Python -*-
-#
-# $Id: aliasutil.py $
-#
-# Author: Markus Stenberg <mstenber@cisco.com>
-#
-
-
-"""
-
-Mailing list alias dumping utilities
-
-"""
-
-
-from django.conf import settings
-from ietf.utils.log import log
-
-import debug                            # pyflakes:ignore
-
-def rewrite_email_address(email):
-    """ Prettify the email address (and if it's empty, skip it by
-    returning None). """
-    if not email:
-        return
-    email = email.strip()
-    if not email:
-        return
-    if email[0]=='<' and email[-1] == '>':
-        email = email[1:-1]
-    # If it doesn't look like email, skip
-    if '@' not in email and '?' not in email:
-        return
-    return email
-
-def rewrite_address_list(l):
-    """ This utility function makes sure there is exactly one instance
-    of an address within the result list, and preserves order
-    (although it may not be relevant to start with) """
-    h = {}
-    for address in l:
-        #address = address.strip()
-        if address in h: continue
-        h[address] = True
-        yield address
-
-def dump_sublist(afile, vfile, alias, adomains, vdomain, emails):
-    if not emails:
-        return emails
-    # Nones in the list should be skipped
-    emails = [_f for _f in emails if _f]
-
-    # Make sure emails are sane and eliminate the Nones again for
-    # non-sane ones
-    emails = [rewrite_email_address(e) for e in emails]
-    emails = [_f for _f in emails if _f]
-
-    # And we'll eliminate the duplicates too but preserve order
-    emails = list(rewrite_address_list(emails))
-    if not emails:
-        return emails
-    try:
-        filtername  = 'xfilter-%s' % (alias, )  # in aliases, --> | expandname
-        expandname  = 'expand-%s' % (alias, )   # in virtual, --> email list
-
-        for domain in adomains:
-            aliasaddr   = '%s@%s' % (alias, domain) # in virtual, --> filtername
-            vfile.write('%-64s  %s\n' % (aliasaddr, filtername))
-        afile.write('%-64s  "|%s filter %s %s"\n' % (filtername+':', settings.POSTCONFIRM_PATH, expandname, vdomain))
-        vfile.write('%-64s  %s\n' % ("%s@%s"%(expandname, vdomain), ', '.join(emails)))
-
-    except UnicodeEncodeError:
-        # If there's unicode in email address, something is badly
-        # wrong and we just silently punt
-        # XXX - is there better approach?
-        log('Error encoding email address for an %s alias: %s' % (alias, repr(emails)))
-        return []
-    return emails
-

From 7905b8db611dacb1389a13f33146fd019c723907 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 14 Nov 2024 01:53:55 -0500
Subject: [PATCH 160/656] ci: Update build.yml to use GHA larger runner

---
 .github/workflows/build.yml | 289 ++++++++++++------------------------
 1 file changed, 91 insertions(+), 198 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 405a793266..67f24ba76f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -149,7 +149,8 @@ jobs:
     name: Make Release
     if: ${{ !failure() && !cancelled() }}
     needs: [tests, prepare]
-    runs-on: ubuntu-latest
+    runs-on:
+      group: hperf-8c32r
     permissions:
       contents: write
       packages: write
@@ -166,215 +167,107 @@ jobs:
         fetch-depth: 1
         fetch-tags: false
 
-    - name: Launch build VM
-      id: azlaunch
-      timeout-minutes: 10
+    - name: Setup Node.js environment
+      uses: actions/setup-node@v4
+      with:
+        node-version: 18.x
+
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.x"
+
+    - name: Download a Coverage Results
+      if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
+      uses: actions/download-artifact@v4.1.8
+      with:
+        name: coverage
+
+    - name: Make Release Build
+      env:
+        DEBIAN_FRONTEND: noninteractive
+        BROWSERSLIST_IGNORE_OLD_DATA: 1
       run: |
-        echo "Authenticating to Azure..."
-        az login --service-principal -u ${{ secrets.AZ_BUILD_APP_ID }} -p ${{ secrets.AZ_BUILD_PWD }} --tenant ${{ secrets.AZ_BUILD_TENANT_ID }}
-        
-        echo "Creating VM..."
-        vminfo=$(az vm create \
-          --resource-group ghaDatatracker \
-          --name tmpGhaBuildVM-${{ github.run_number }} \
-          --image Ubuntu2204 \
-          --admin-username azureuser \
-          --generate-ssh-keys \
-          --priority Spot \
-          --size Standard_D8ads_v5 \
-          --max-price -1 \
-          --ephemeral-os-disk \
-          --os-disk-size-gb 100 \
-          --eviction-policy Delete \
-          --nic-delete-option Delete \
-          --os-disk-delete-option Delete \
-          --output tsv \
-          --query "publicIpAddress")
-        echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT"
-        echo "VM Public IP: $vminfo"
-        cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key
+        echo "PKG_VERSION: $PKG_VERSION"
+        echo "GITHUB_SHA: $GITHUB_SHA"
+        echo "GITHUB_REF_NAME: $GITHUB_REF_NAME"
+        echo "Running frontend build script..."
+        echo "Compiling native node packages..."
+        yarn rebuild
+        echo "Packaging static assets..."
+        yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/
+        yarn legacy:build
+        echo "Setting version $PKG_VERSION..."
+        sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py
+        sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py
+        sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py
+
+    - name: Set Production Flags
+      if: ${{ env.SHOULD_DEPLOY == 'true' }}
+      run: |
+        echo "Setting production flags in settings.py..."
+        sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py
         
-        echo "Fetching SSH host public keys..."
-        until ssh-keyscan -t rsa $vminfo 2> /dev/null
-        do
-          echo "Will try again in 5 seconds..."
-          sleep 5
-        done
-        ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts
+    - name: Make Release Tarball
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        echo "Build release tarball..."
+        mkdir -p /home/runner/work/release
+        tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt .
 
-    - name: Remote SSH into Build VM
-      uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1
+    - name: Collect + Push Statics
       env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        GITHUB_ACTOR: ${{ github.actor }}
-        GITHUB_SHA: ${{ github.sha }}
-        GITHUB_REF_NAME: ${{ github.ref_name }}
-        GITHUB_RUN_ID: ${{ github.run_id }}
+        DEBIAN_FRONTEND: noninteractive
         AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }}
         AWS_DEFAULT_REGION: auto
         AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }}
-        PKG_VERSION: ${{ env.PKG_VERSION }}
-        SHOULD_DEPLOY: ${{ env.SHOULD_DEPLOY }}
-        SKIP_TESTS: ${{ github.event.inputs.skiptests }}
+      run: |
+        echo "Collecting statics..."
+        echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}"
+        docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh
+        echo "Pushing statics..."
+        cd static
+        aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
+
+    - name: Augment dockerignore for docker image build
+      env:
         DEBIAN_FRONTEND: noninteractive
-        BROWSERSLIST_IGNORE_OLD_DATA: 1
-        TARGET_BASE: ${{ env.TARGET_BASE }}
+      run: |
+        cat >> .dockerignore <<EOL
+        .devcontainer
+        .github
+        .vscode
+        k8s
+        playwright
+        svn-history
+        docker-compose.yml
+        EOL
+
+    - name: Setup Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v3
       with:
-        host: ${{ steps.azlaunch.outputs.ipaddr }}
-        port: 22
-        username: azureuser
-        command_timeout: 60m
-        key_path: ${{ github.workspace }}/prvkey.key
-        envs: GITHUB_TOKEN,GITHUB_ACTOR,GITHUB_SHA,GITHUB_REF_NAME,GITHUB_RUN_ID,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_DEFAULT_REGION,AWS_ENDPOINT_URL,PKG_VERSION,SHOULD_DEPLOY,SKIP_TESTS,DEBIAN_FRONTEND,BROWSERSLIST_IGNORE_OLD_DATA
-        script_stop: true
-        script: |
-          export DEBIAN_FRONTEND=noninteractive
-          lsb_release -a
-          sudo apt-get update
-          sudo apt-get upgrade -y
-          sudo apt-get install wget unzip curl -y
-
-          echo "=========================================================================="
-          echo "Installing Docker..."
-          echo "=========================================================================="
-          curl -fsSL https://get.docker.com -o get-docker.sh
-          sudo sh get-docker.sh
-          sudo docker buildx create \
-            --name container-builder \
-            --driver docker-container \
-            --bootstrap --use
-
-          echo "=========================================================================="
-          echo "Login to ghcr.io..."
-          echo "=========================================================================="
-          echo $GITHUB_TOKEN | sudo docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
-
-          echo "=========================================================================="
-          echo "Installing GH CLI..."
-          echo "=========================================================================="
-          sudo mkdir -p -m 755 /etc/apt/keyrings \
-          	&& wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
-          	&& sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
-          	&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
-          	&& sudo apt update \
-          	&& sudo apt install gh -y
-
-          echo "=========================================================================="
-          echo "Installing AWS CLI..."
-          echo "=========================================================================="
-          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-          unzip awscliv2.zip
-          sudo ./aws/install
-
-          echo "=========================================================================="
-          echo "Install Node.js..."
-          echo "=========================================================================="
-          curl -fsSL https://deb.nodesource.com/setup_18.x -o nodesource_setup.sh
-          sudo bash nodesource_setup.sh
-          sudo apt-get install -y nodejs
-          sudo corepack enable
-
-          echo "=========================================================================="
-          echo "Install Python 3.x..."
-          echo "=========================================================================="
-          sudo apt-get install python3 python3-dev -y
-          python3 --version
-
-          echo "=========================================================================="
-          echo "Clone project..."
-          echo "=========================================================================="
-          sudo mkdir -p /workspace
-          sudo chown azureuser /workspace
-          cd /workspace
-          gh repo clone ietf-tools/datatracker -- --depth=1 --no-tags --branch ${{ github.ref_name }}
-          cd datatracker
-
-          if [ "$SKIP_TESTS" = "false" ] || [ "$GITHUB_REF_NAME" = "release" ] ; then
-            echo "=========================================================================="
-            echo "Downloading coverage..."
-            echo "=========================================================================="
-            gh run download $GITHUB_RUN_ID -n coverage
-          fi
-
-          echo "=========================================================================="
-          echo "Building project..."
-          echo "=========================================================================="
-          echo "PKG_VERSION: $PKG_VERSION"
-          echo "GITHUB_SHA: $GITHUB_SHA"
-          echo "GITHUB_REF_NAME: $GITHUB_REF_NAME"
-          echo "Running frontend build script..."
-          echo "Compiling native node packages..."
-          yarn rebuild
-          echo "Packaging static assets..."
-          yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/
-          yarn legacy:build
-          echo "Setting version $PKG_VERSION..."
-          sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py
-          sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py
-          sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
 
-          if [ "$SHOULD_DEPLOY" = "true" ] ; then
-            echo "=========================================================================="
-            echo "Setting production flags in settings.py..."
-            echo "=========================================================================="
-            sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py
-          fi
-
-          echo "=========================================================================="
-          echo "Build release tarball..."
-          echo "=========================================================================="
-          mkdir -p /workspace/release
-          tar -czf /workspace/release.tar.gz -X dev/build/exclude-patterns.txt .
-
-          echo "=========================================================================="
-          echo "Collecting statics..."
-          echo "=========================================================================="
-          echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}"
-          sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh
-          echo "Pushing statics..."
-          cd static
-          aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
-          cd ..
-
-          echo "=========================================================================="
-          echo "Augment dockerignore for docker image build..."
-          echo "=========================================================================="
-          cat >> .dockerignore <<EOL
-          .devcontainer
-          .github
-          .vscode
-          k8s
-          playwright
-          svn-history
-          docker-compose.yml
-          EOL
-
-          echo "=========================================================================="
-          echo "Building Images..."
-          echo "=========================================================================="
-          sudo docker buildx build --file dev/build/Dockerfile --platform linux/amd64,linux/arm64 --tag ghcr.io/ietf-tools/datatracker:$PKG_VERSION --push .
-
-    - name: Fetch release tarball
-      run: |
-        mkdir -p /home/runner/work/release
-        chmod 0600 ${{ github.workspace }}/prvkey.key
-        scp -i ${{ github.workspace }}/prvkey.key azureuser@${{ steps.azlaunch.outputs.ipaddr }}:/workspace/release.tar.gz /home/runner/work/release/release.tar.gz
+    - name: Build Images
+      uses: docker/build-push-action@v6
+      env:
+        DOCKER_BUILD_SUMMARY: false
+      with:
+        context: .
+        file: dev/build/Dockerfile
+        platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }}
+        push: true
+        tags: ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
 
-    - name: Destroy Build VM + resources
-      if: always()
-      run: |
-        echo "Terminate VM..."        
-        az vm delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }} --yes --force-deletion true
-        echo "Delete Public IP..."
-        az resource delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }}PublicIP --resource-type "Microsoft.Network/publicIPAddresses"
-        echo "Delete Network Security Group..."
-        az resource delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }}NSG --resource-type "Microsoft.Network/networkSecurityGroups"
-        echo "Delete Virtual Network..."
-        az resource delete -g ghaDatatracker -n tmpGhaBuildVM-${{ github.run_number }}VNET --resource-type "Microsoft.Network/virtualNetworks"
-        echo "Logout from Azure..."
-        az logout
-        
     - name: Update CHANGELOG
       id: changelog
       uses: Requarks/changelog-action@v1

From 84a5aa3805eb8933ebb6a4441a9a10ba5a7dce04 Mon Sep 17 00:00:00 2001
From: NGPixel <NGPixel@users.noreply.github.com>
Date: Thu, 14 Nov 2024 20:05:03 +0000
Subject: [PATCH 161/656] ci: update base image target version to 20241114T1954

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 728b37156c..c9f91163ce 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20241029T1632
+FROM ghcr.io/ietf-tools/datatracker-app-base:20241114T1954
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 2f8aac20f9..7bbb32355d 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20241029T1632
+20241114T1954

From 901fdd8d44cfe071d5184bcace4d7dbf1e093d37 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 14 Nov 2024 17:21:48 -0400
Subject: [PATCH 162/656] chore: remove newrevisiondocevent stats view (#8210)

* refactor: update node, eslint, neostandard + fix esm (#8083)

* chore: update dependencies

* fix: eslint + neostandard

* fix: add corepack prompt env var to init script

* docs: Update README.md

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>

* chore: remove newrevisiondocevent stats view

Not functional anyway

* chore: fix lint

* chore: remove unused template

* Revert "refactor: update node, eslint, neostandard + fix esm (#8083)"

This reverts commit 649879efd745470f6e0cc6768d889f45640e1505.

---------

Co-authored-by: Nicolas Giard <github@ngpixel.com>
Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/doc/tests.py                       | 54 -------------------------
 ietf/doc/urls.py                        |  4 --
 ietf/doc/views_stats.py                 | 50 +----------------------
 ietf/templates/doc/stats/highstock.html | 29 -------------
 4 files changed, 1 insertion(+), 136 deletions(-)
 delete mode 100644 ietf/templates/doc/stats/highstock.html

diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index e3534ba725..a1f9f8da2b 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -2739,60 +2739,6 @@ def test_get_related_meeting(self):
                 self.assertIsNone(doc.get_related_meeting(), f'{doc.type.slug} should not be related to meeting')
 
 class ChartTests(ResourceTestCaseMixin, TestCase):
-    def test_search_chart_conf(self):
-        doc = IndividualDraftFactory()
-
-        conf_url = urlreverse('ietf.doc.views_stats.chart_conf_newrevisiondocevent')
-
-        # No qurey arguments; expect an empty json object
-        r = self.client.get(conf_url)
-        self.assertValidJSONResponse(r)
-        self.assertEqual(unicontent(r), '{}')
-
-        # No match
-        r = self.client.get(conf_url + '?activedrafts=on&name=thisisnotadocumentname')
-        self.assertValidJSONResponse(r)
-        d = r.json()
-        self.assertEqual(d['chart']['type'], settings.CHART_TYPE_COLUMN_OPTIONS['chart']['type'])
-
-        r = self.client.get(conf_url + '?activedrafts=on&name=%s'%doc.name[6:12])
-        self.assertValidJSONResponse(r)
-        d = r.json()
-        self.assertEqual(d['chart']['type'], settings.CHART_TYPE_COLUMN_OPTIONS['chart']['type'])
-        self.assertEqual(len(d['series'][0]['data']), 0)
-
-    def test_search_chart_data(self):
-        doc = IndividualDraftFactory()
-
-        data_url = urlreverse('ietf.doc.views_stats.chart_data_newrevisiondocevent')
-
-        # No qurey arguments; expect an empty json list
-        r = self.client.get(data_url)
-        self.assertValidJSONResponse(r)
-        self.assertEqual(unicontent(r), '[]')
-
-        # No match
-        r = self.client.get(data_url + '?activedrafts=on&name=thisisnotadocumentname')
-        self.assertValidJSONResponse(r)
-        d = r.json()
-        self.assertEqual(unicontent(r), '[]')
-
-        r = self.client.get(data_url + '?activedrafts=on&name=%s'%doc.name[6:12])
-        self.assertValidJSONResponse(r)
-        d = r.json()
-        self.assertEqual(len(d), 1)
-        self.assertEqual(len(d[0]), 2)
-
-    def test_search_chart(self):
-        doc = IndividualDraftFactory()
-
-        chart_url = urlreverse('ietf.doc.views_stats.chart_newrevisiondocevent')
-        r = self.client.get(chart_url)
-        self.assertEqual(r.status_code, 200)
-
-        r = self.client.get(chart_url + '?activedrafts=on&name=%s'%doc.name[6:12])
-        self.assertEqual(r.status_code, 200)
-        
     def test_personal_chart(self):
         person = PersonFactory.create()
         IndividualDraftFactory.create(
diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py
index 7d930f7f40..6e2f03a3dc 100644
--- a/ietf/doc/urls.py
+++ b/ietf/doc/urls.py
@@ -68,10 +68,6 @@
     ),
     url(r'^investigate/?$', views_doc.investigate),
 
-
-    url(r'^stats/newrevisiondocevent/?$', views_stats.chart_newrevisiondocevent),
-    url(r'^stats/newrevisiondocevent/conf/?$', views_stats.chart_conf_newrevisiondocevent),
-    url(r'^stats/newrevisiondocevent/data/?$', views_stats.chart_data_newrevisiondocevent),
     url(r'^stats/person/(?P<id>[0-9]+)/drafts/conf/?$', views_stats.chart_conf_person_drafts),
     url(r'^stats/person/(?P<id>[0-9]+)/drafts/data/?$', views_stats.chart_data_person_drafts),
 
diff --git a/ietf/doc/views_stats.py b/ietf/doc/views_stats.py
index cefc7e152b..0bbf0b91c5 100644
--- a/ietf/doc/views_stats.py
+++ b/ietf/doc/views_stats.py
@@ -4,20 +4,15 @@
 import datetime
 
 from django.conf import settings
-from django.core.cache import cache
-from django.urls import reverse as urlreverse
 from django.db.models.aggregates import Count
 from django.db.models.functions import TruncDate
-from django.http import JsonResponse, HttpResponseBadRequest
-from django.shortcuts import render
+from django.http import JsonResponse
 from django.views.decorators.cache import cache_page
 
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import DocEvent
 from ietf.doc.templatetags.ietf_filters import comma_separated_list
-from ietf.doc.utils import get_search_cache_key
-from ietf.doc.views_search import SearchForm, retrieve_search_results
 from ietf.name.models import DocTypeName
 from ietf.person.models import Person
 from ietf.utils.timezone import date_today
@@ -113,49 +108,6 @@ def make_title(queryargs):
         title += ' with name matching "%s"' % name
     return title
 
-def chart_newrevisiondocevent(request):
-    return render(request, "doc/stats/highstock.html", {
-            "title": "Document Statistics",
-            "confurl": urlreverse("ietf.doc.views_stats.chart_conf_newrevisiondocevent"),
-            "dataurl": urlreverse("ietf.doc.views_stats.chart_data_newrevisiondocevent"),
-            "queryargs": request.GET.urlencode(),
-            }
-        )
-
-#@cache_page(60*15)
-def chart_data_newrevisiondocevent(request):
-    queryargs = request.GET
-    if queryargs:
-        cache_key = get_search_cache_key(queryargs)
-        results = cache.get(cache_key)
-        if not results:
-            form = SearchForm(queryargs)
-            if not form.is_valid():
-                return HttpResponseBadRequest("form not valid: %s" % form.errors)
-            results = retrieve_search_results(form)
-            if results.exists():
-                cache.set(cache_key, results)
-        if results.exists():
-            data = model_to_timeline_data(DocEvent, doc__in=results, type='new_revision')
-        else:
-            data = []
-    else:
-        data = []
-    return JsonResponse(data, safe=False)
-
-
-@cache_page(60*15)
-def chart_conf_newrevisiondocevent(request):
-    queryargs = request.GET
-    if queryargs:
-        conf = copy.deepcopy(settings.CHART_TYPE_COLUMN_OPTIONS)
-        conf['title']['text'] = make_title(queryargs)
-        conf['series'][0]['name'] = "Submitted %s" % get_doctypes(queryargs, pluralize=True).lower(),
-    else:
-        conf = {}
-    return JsonResponse(conf)
-
-
 @cache_page(60*15)
 def chart_conf_person_drafts(request, id):
     person = Person.objects.filter(id=id).first()
diff --git a/ietf/templates/doc/stats/highstock.html b/ietf/templates/doc/stats/highstock.html
deleted file mode 100644
index 6719a3b9eb..0000000000
--- a/ietf/templates/doc/stats/highstock.html
+++ /dev/null
@@ -1,29 +0,0 @@
-{% extends "base.html" %}
-{# Copyright The IETF Trust 2015, All Rights Reserved #}
-{% load origin %}
-{% load static %}
-{% load ietf_filters %}
-{% block pagehead %}
-    <link rel="stylesheet" href="{% static "ietf/css/highcharts.css" %}">
-{% endblock %}
-{% block js %}
-    <script src="{% static 'ietf/js/highstock.js' %}"></script>
-    <script>
-        $(function () {
-            var chart;
-            $.getJSON('{{ confurl }}?{{ queryargs }}', function (conf) {
-                chart = Highcharts.stockChart('chart', conf);
-                chart.showLoading();
-                $.getJSON('{{ dataurl }}?{{ queryargs }}', function (data) {
-                    chart.series[0].setData(data);
-                    chart.hideLoading();
-                });
-            });
-        });
-    </script>
-{% endblock %}
-{% block title %}Document Statistics{% endblock %}
-{% block content %}
-    {% origin %}
-    <div id="chart"></div>
-{% endblock %}
\ No newline at end of file

From b65a37b6e8f6cfe105cd89c61c7046fff9d21523 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 14 Nov 2024 18:05:38 -0400
Subject: [PATCH 163/656] feat: POST for document search requests (#8206)

* refactor: doc search via POST (WIP)

Changes the search view to use a POST instead of a GET. Refactors cache key computation to use cleaned data.

Still todo:
 * refactor frontpage view to match
 * refactor menubar search (?)
 * refactor stats view that uses SearchForm
 * revive or drop the "backwards compatibility" branch

* feat: convert GET search to POST

Still todo:
 * refactor frontpage view to match
 * refactor menubar search (?)
 * refactor stats view that uses SearchForm

* fix: revert frontpage changes, search works

Still todo:
 * refactor stats view that uses SearchForm

* fix: define vars in all branches

* refactor: update stats use of SearchForm

* chore: improve message

* fix: remove lint

* chore: comments re: QuerySetAny

* test: test query string search params

* style: Black

* test: refactor test_search()

* test: refactor test_search_became_rfc()

* test: use scroll_and_click helper
---
 ietf/api/serializer.py                     |   1 +
 ietf/doc/tests.py                          | 155 +++++++++++++++------
 ietf/doc/tests_js.py                       |   4 +-
 ietf/doc/utils.py                          |  11 +-
 ietf/doc/views_search.py                   | 109 ++++++++++-----
 ietf/liaisons/forms.py                     |   1 +
 ietf/liaisons/widgets.py                   |   1 +
 ietf/templates/doc/search/search_form.html |   2 +
 8 files changed, 197 insertions(+), 87 deletions(-)

diff --git a/ietf/api/serializer.py b/ietf/api/serializer.py
index 27f194c5b5..ca34ea649e 100644
--- a/ietf/api/serializer.py
+++ b/ietf/api/serializer.py
@@ -146,6 +146,7 @@ def end_object(self, obj):
                                 field_value = None
                         else:
                             field_value = field
+                        # Need QuerySetAny instead of QuerySet until django-stubs 5.0.1
                         if isinstance(field_value, QuerySetAny) or isinstance(field_value, list):
                             self._current[name] = dict([ (rel.pk, self.expand_related(rel, name)) for rel in field_value ])
                         else:
diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index a1f9f8da2b..fa655cb884 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -71,96 +71,163 @@
 
 
 class SearchTests(TestCase):
-    def test_search(self):
+    def test_search_handles_querystring_parameters(self):
+        """Search parameters via querystring should not actually search"""
+        url = urlreverse("ietf.doc.views_search.search")
+        r = self.client.get(url + "?name=some-document-name&oldDrafts=on")
+        # Check that we got a valid response and that the warning about query string parameters is shown.
+        self.assertContains(
+            r,
+            "Searching via the URL query string is no longer supported.",
+            status_code=200,
+        )
+        # Check that the form was filled in correctly (not an exhaustive check, but different from the
+        # form defaults)
+        pq = PyQuery(r.content)
+        self.assertEqual(
+            pq("form#search_form input#id_name").attr("value"),
+            "some-document-name",
+            "The name field should be set in the SearchForm",
+        )
+        self.assertEqual(
+            pq("form#search_form input#id_olddrafts").attr("checked"),
+            "checked",
+            "The old drafts checkbox should be selected in the SearchForm",
+        )
+        self.assertIsNone(
+            pq("form#search_form input#id_rfcs").attr("checked"),
+            "The RFCs checkbox should not be selected in the SearchForm",
+        )
+        self.assertIsNone(
+            pq("form#search_form input#id_activedrafts").attr("checked"),
+            "The active drafts checkbox should not be selected in the SearchForm",
+        )
 
-        draft = WgDraftFactory(name='draft-ietf-mars-test',group=GroupFactory(acronym='mars',parent=Group.objects.get(acronym='farfut')),authors=[PersonFactory()],ad=PersonFactory())
+    def test_search(self):
+        draft = WgDraftFactory(
+            name="draft-ietf-mars-test",
+            group=GroupFactory(acronym="mars", parent=Group.objects.get(acronym="farfut")),
+            authors=[PersonFactory()],
+            ad=PersonFactory(),
+        )
         rfc = WgRfcFactory()
         draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="pub-req"))
-        old_draft = IndividualDraftFactory(name='draft-foo-mars-test',authors=[PersonFactory()],title="Optimizing Martian Network Topologies")
+        old_draft = IndividualDraftFactory(
+            name="draft-foo-mars-test",
+            authors=[PersonFactory()],
+            title="Optimizing Martian Network Topologies",
+        )
         old_draft.set_state(State.objects.get(used=True, type="draft", slug="expired"))
-
-        base_url = urlreverse('ietf.doc.views_search.search')
-
+    
+        url = urlreverse("ietf.doc.views_search.search")
+    
         # only show form, no search yet
-        r = self.client.get(base_url)
+        r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-
+    
         # no match
-        r = self.client.get(base_url + "?activedrafts=on&name=thisisnotadocumentname")
+        r = self.client.post(url, {"activedrafts": "on", "name": "thisisnotadocumentname"})
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "No documents match")
-
-        r = self.client.get(base_url + "?rfcs=on&name=xyzzy")
+    
+        r = self.client.post(url, {"rfcs": "on", "name": "xyzzy"})
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "No documents match")
-
-        r = self.client.get(base_url + "?olddrafts=on&name=bar")
+    
+        r = self.client.post(url, {"olddrafts": "on", "name": "bar"})
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "No documents match")
-
-        r = self.client.get(base_url + "?olddrafts=on&name=foo")
+    
+        r = self.client.post(url, {"olddrafts": "on", "name": "foo"})
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "draft-foo-mars-test")
-
-        r = self.client.get(base_url + "?olddrafts=on&name=FoO")  # mixed case
+    
+        r = self.client.post(url, {"olddrafts": "on", "name": "FoO"})  # mixed case
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "draft-foo-mars-test")
-
+    
         # find by RFC
-        r = self.client.get(base_url + "?rfcs=on&name=%s" % rfc.name)
+        r = self.client.post(url, {"rfcs": "on", "name": rfc.name})
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, rfc.title)
-
+    
         # find by active/inactive
-
+    
         draft.set_state(State.objects.get(type="draft", slug="active"))
-        r = self.client.get(base_url + "?activedrafts=on&name=%s" % draft.name)
+        r = self.client.post(url, {"activedrafts": "on", "name": draft.name})
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-
+    
         draft.set_state(State.objects.get(type="draft", slug="expired"))
-        r = self.client.get(base_url + "?olddrafts=on&name=%s" % draft.name)
+        r = self.client.post(url, {"olddrafts": "on", "name": draft.name})
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-        
+    
         draft.set_state(State.objects.get(type="draft", slug="active"))
-
+    
         # find by title
-        r = self.client.get(base_url + "?activedrafts=on&name=%s" % draft.title.split()[0])
+        r = self.client.post(url, {"activedrafts": "on", "name": draft.title.split()[0]})
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-
+    
         # find by author
-        r = self.client.get(base_url + "?activedrafts=on&by=author&author=%s" % draft.documentauthor_set.first().person.name_parts()[1])
+        r = self.client.post(
+            url,
+            {
+                "activedrafts": "on",
+                "by": "author",
+                "author": draft.documentauthor_set.first().person.name_parts()[1],
+            },
+        )
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-
+    
         # find by group
-        r = self.client.get(base_url + "?activedrafts=on&by=group&group=%s" % draft.group.acronym)
+        r = self.client.post(
+            url,
+            {"activedrafts": "on", "by": "group", "group": draft.group.acronym},
+        )
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-
-        r = self.client.get(base_url + "?activedrafts=on&by=group&group=%s" % draft.group.acronym.swapcase())
+    
+        r = self.client.post(
+            url,
+            {"activedrafts": "on", "by": "group", "group": draft.group.acronym.swapcase()},
+        )
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-
+    
         # find by area
-        r = self.client.get(base_url + "?activedrafts=on&by=area&area=%s" % draft.group.parent_id)
+        r = self.client.post(
+            url,
+            {"activedrafts": "on", "by": "area", "area": draft.group.parent_id},
+        )
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-
+    
         # find by area
-        r = self.client.get(base_url + "?activedrafts=on&by=area&area=%s" % draft.group.parent_id)
+        r = self.client.post(
+            url,
+            {"activedrafts": "on", "by": "area", "area": draft.group.parent_id},
+        )
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-
+    
         # find by AD
-        r = self.client.get(base_url + "?activedrafts=on&by=ad&ad=%s" % draft.ad_id)
+        r = self.client.post(url, {"activedrafts": "on", "by": "ad", "ad": draft.ad_id})
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-
+    
         # find by IESG state
-        r = self.client.get(base_url + "?activedrafts=on&by=state&state=%s&substate=" % draft.get_state("draft-iesg").pk)
+        r = self.client.post(
+            url,
+            {
+                "activedrafts": "on",
+                "by": "state",
+                "state": draft.get_state("draft-iesg").pk,
+                "substate": "",
+            },
+        )
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
 
@@ -169,15 +236,15 @@ def test_search_became_rfc(self):
         rfc = WgRfcFactory()
         draft.set_state(State.objects.get(type="draft", slug="rfc"))
         draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc)
-        base_url = urlreverse('ietf.doc.views_search.search')
+        url = urlreverse("ietf.doc.views_search.search")
 
         # find by RFC
-        r = self.client.get(base_url + f"?rfcs=on&name={rfc.name}")
+        r = self.client.post(url, {"rfcs": "on", "name": rfc.name})
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, rfc.title)
 
         # find by draft
-        r = self.client.get(base_url + f"?activedrafts=on&rfcs=on&name={draft.name}")
+        r = self.client.post(url, {"activedrafts": "on", "rfcs": "on", "name": draft.name})
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, rfc.title)
 
diff --git a/ietf/doc/tests_js.py b/ietf/doc/tests_js.py
index acd74c4a0b..9a5aad13b9 100644
--- a/ietf/doc/tests_js.py
+++ b/ietf/doc/tests_js.py
@@ -92,10 +92,8 @@ def _read_author_form(form_elt):
         self.assertEqual(len(author_forms), 1)
 
         # get the "add author" button so we can add blank author forms
-        add_author_button = self.driver.find_element(By.ID, 'add-author-button')
         for index, auth in enumerate(authors):
-            self.scroll_to_element(add_author_button)  # Can only click if it's in view!
-            add_author_button.click()  # Create a new form. Automatically scrolls to it.
+            self.scroll_and_click((By.ID, 'add-author-button'))  # Create new form. Automatically scrolls to it.
             author_forms = authors_list.find_elements(By.CLASS_NAME, 'author-panel')
             authors_added = index + 1
             self.assertEqual(len(author_forms), authors_added + 1)  # Started with 1 author, hence +1
diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index 97243a20d6..9b2570d8ba 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -3,9 +3,7 @@
 
 
 import datetime
-import hashlib
 import io
-import json
 import math
 import os
 import re
@@ -348,6 +346,7 @@ def augment_events_with_revision(doc, events):
     """Take a set of events for doc and add a .rev attribute with the
     revision they refer to by checking NewRevisionDocEvents."""
 
+    # Need QuerySetAny instead of QuerySet until django-stubs 5.0.1
     if isinstance(events, QuerySetAny):
         qs = events.filter(newrevisiondocevent__isnull=False)
     else:
@@ -1047,12 +1046,8 @@ def get_replaces_tree(doc):
     return sorted(history, key=lambda x: x['published'])
 
 
-def get_search_cache_key(params):
-    from ietf.doc.views_search import SearchForm
-    fields = set(SearchForm.base_fields) - set(['sort',])
-    kwargs = dict([ (k,v) for (k,v) in list(params.items()) if k in fields ])
-    key = "doc:document:search:" + hashlib.sha512(json.dumps(kwargs, sort_keys=True).encode('utf-8')).hexdigest()
-    return key
+def get_search_cache_key(key_fragment):
+    return f"doc:document:search:{key_fragment}"
 
 
 def build_file_urls(doc: Union[Document, DocHistory]):
diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py
index 4fa3b2560c..7b71dd77bd 100644
--- a/ietf/doc/views_search.py
+++ b/ietf/doc/views_search.py
@@ -37,6 +37,8 @@
 import re
 import datetime
 import copy
+import hashlib
+import json
 import operator
 
 from collections import defaultdict
@@ -44,16 +46,17 @@
 
 from django import forms
 from django.conf import settings
+from django.contrib import messages
 from django.core.cache import cache, caches
 from django.urls import reverse as urlreverse
 from django.db.models import Q
-from django.http import Http404, HttpResponseBadRequest, HttpResponse, HttpResponseRedirect, QueryDict
+from django.http import Http404, HttpResponseBadRequest, HttpResponse, HttpResponseRedirect
 from django.shortcuts import render
 from django.utils import timezone
 from django.utils.html import strip_tags
 from django.utils.cache import _generate_cache_key # type: ignore
 from django.utils.text import slugify
-
+from django_stubs_ext import QuerySetAny
 
 import debug                            # pyflakes:ignore
 
@@ -145,6 +148,29 @@ def clean(self):
             q['irtfstate'] = None
         return q
 
+    def cache_key_fragment(self):
+        """Hash a bound form to get a value for use in a cache key
+        
+        Raises a ValueError if the form is not valid.
+        """
+        def _serialize_value(val):
+            # Need QuerySetAny instead of QuerySet until django-stubs 5.0.1
+            if isinstance(val, QuerySetAny):
+                return [item.pk for item in val]
+            else:
+                return getattr(val, "pk", val)  # use pk if present, else value
+    
+        if not self.is_valid():
+            raise ValueError(f"SearchForm invalid: {self.errors}")
+        contents = {
+            field_name: _serialize_value(field_value)
+            for field_name, field_value in self.cleaned_data.items() 
+            if field_name != "sort" and field_value is not None
+        }
+        contents_json = json.dumps(contents, sort_keys=True)
+        return hashlib.sha512(contents_json.encode("utf-8")).hexdigest()
+
+
 def retrieve_search_results(form, all_types=False):
     """Takes a validated SearchForm and return the results."""
 
@@ -256,45 +282,64 @@ def retrieve_search_results(form, all_types=False):
 
     return docs
 
+
 def search(request):
-    if request.GET:
-        # backwards compatibility
-        get_params = request.GET.copy()
-        if 'activeDrafts' in request.GET:
-            get_params['activedrafts'] = request.GET['activeDrafts']
-        if 'oldDrafts' in request.GET:
-            get_params['olddrafts'] = request.GET['oldDrafts']
-        if 'subState' in request.GET:
-            get_params['substate'] = request.GET['subState']
-
-        form = SearchForm(get_params)
-        if not form.is_valid():
-            return HttpResponseBadRequest("form not valid: %s" % form.errors)
-
-        cache_key = get_search_cache_key(get_params)
-        cached_val = cache.get(cache_key)
-        if cached_val:
-            [results, meta] = cached_val
-        else:
-            results = retrieve_search_results(form)
-            results, meta = prepare_document_table(request, results, get_params)
-            cache.set(cache_key, [results, meta]) # for settings.CACHE_MIDDLEWARE_SECONDS
-            log(f"Search results computed for {get_params}")
-        meta['searching'] = True
+    """Search for a draft"""
+    # defaults for results / meta
+    results = []
+    meta = {"by": None, "searching": False}
+
+    if request.method == "POST":
+        form = SearchForm(data=request.POST)
+        if form.is_valid():
+            cache_key = get_search_cache_key(form.cache_key_fragment())
+            cached_val = cache.get(cache_key)
+            if cached_val:
+                [results, meta] = cached_val
+            else:
+                results = retrieve_search_results(form)
+                results, meta = prepare_document_table(
+                    request, results, form.cleaned_data
+                )
+                cache.set(
+                    cache_key, [results, meta]
+                )  # for settings.CACHE_MIDDLEWARE_SECONDS
+                log(f"Search results computed for {form.cleaned_data}")
+            meta["searching"] = True
     else:
-        form = SearchForm()
-        results = []
-        meta = { 'by': None, 'searching': False }
-        get_params = QueryDict('')
+        if request.GET:
+            # backwards compatibility - fill in the form
+            get_params = request.GET.copy()
+            if "activeDrafts" in request.GET:
+                get_params["activedrafts"] = request.GET["activeDrafts"]
+            if "oldDrafts" in request.GET:
+                get_params["olddrafts"] = request.GET["oldDrafts"]
+            if "subState" in request.GET:
+                get_params["substate"] = request.GET["subState"]
+            form = SearchForm(data=get_params)
+            messages.error(
+                request,
+                (
+                    "Searching via the URL query string is no longer supported. "
+                    "The form below has been filled in with the parameters from your request. "
+                    'To execute your search, please click "Search".'
+                ),
+            )
+        else:
+            form = SearchForm()
 
-    return render(request, 'doc/search/search.html', {
-        'form':form, 'docs':results, 'meta':meta, 'queryargs':get_params.urlencode() },
+    return render(
+        request,
+        "doc/search/search.html",
+        context={"form": form, "docs": results, "meta": meta},
     )
 
+
 def frontpage(request):
     form = SearchForm()
     return render(request, 'doc/frontpage.html', {'form':form})
 
+
 def search_for_name(request, name):
     def find_unique(n):
         exact = Document.objects.filter(name__iexact=n).first()
diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py
index 1d91041b25..a75028bf79 100644
--- a/ietf/liaisons/forms.py
+++ b/ietf/liaisons/forms.py
@@ -203,6 +203,7 @@ def get_results(self):
 class CustomModelMultipleChoiceField(ModelMultipleChoiceField):
     '''If value is a QuerySet, return it as is (for use in widget.render)'''
     def prepare_value(self, value):
+        # Need QuerySetAny instead of QuerySet until django-stubs 5.0.1
         if isinstance(value, QuerySetAny):
             return value
         if (hasattr(value, '__iter__') and
diff --git a/ietf/liaisons/widgets.py b/ietf/liaisons/widgets.py
index 74368e83f2..3d4f2d13a5 100644
--- a/ietf/liaisons/widgets.py
+++ b/ietf/liaisons/widgets.py
@@ -35,6 +35,7 @@ def render(self, name, value, **kwargs):
         html = '<div id="id_%s">' % name
         html += '<span class="d-none showAttachmentsEmpty form-control widget">No files attached</span>'
         html += '<div class="attachedFiles form-control widget">'
+        # Need QuerySetAny instead of QuerySet until django-stubs 5.0.1
         if value and isinstance(value, QuerySetAny):
             for attachment in value:
                 html += '<a class="initialAttach" href="%s">%s</a>&nbsp' % (conditional_escape(attachment.document.get_href()), conditional_escape(attachment.document.title))
diff --git a/ietf/templates/doc/search/search_form.html b/ietf/templates/doc/search/search_form.html
index d4f463ec66..6c91894c8c 100644
--- a/ietf/templates/doc/search/search_form.html
+++ b/ietf/templates/doc/search/search_form.html
@@ -4,8 +4,10 @@
 {% load widget_tweaks %}
 {% load ietf_filters %}
 <form id="search_form"
+      method="post"
       class="form-horizontal"
       action="{% url 'ietf.doc.views_search.search' %}">
+    {% csrf_token %}
     <!-- [html-validate-disable-block input-missing-label -- labelled via aria-label] -->
     <div class="input-group search_field">
         {{ form.name|add_class:"form-control"|attr:"placeholder:Document name/title/RFC number"|attr:"aria-label:Document name/title/RFC number" }}

From c89646ef8d00d23cd9914f457ee5114900307a3e Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Thu, 14 Nov 2024 22:59:05 +0000
Subject: [PATCH 164/656] feat: Agenda proposed badge (#8164)

---
 client/agenda/AgendaScheduleList.vue | 28 ++++++++++++++++++++++++++--
 ietf/meeting/views.py                |  1 +
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index 2869992579..0cac7e39ee 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -83,6 +83,14 @@
                 template(#trigger)
                   span.badge.is-bof BoF
                 span #[a(href='https://www.ietf.org/how/bofs/', target='_blank') Birds of a Feather] sessions (BoFs) are initial discussions about a particular topic of interest to the IETF community.
+              n-popover(
+                v-if='item.isProposed'
+                trigger='hover'
+                :width='250'
+                )
+                template(#trigger)
+                  span.badge.is-proposed Proposed
+                span #[a(href='https://www.ietf.org/process/wgs/', target='_blank') Proposed WGs] are groups in the process of being chartered. If the charter is not approved by the IESG before the IETF meeting, the session may be canceled.
               .agenda-table-note(v-if='item.note')
                 i.bi.bi-arrow-return-right.me-1
                 span {{item.note}}
@@ -468,6 +476,7 @@ const meetingEvents = computed(() => {
       // groupParentName: item.groupParent?.name,
       icon,
       isBoF: item.isBoF,
+      isProposed: item.isProposed,
       isSessionEvent: item.type === 'regular',
       links,
       location: item.location,
@@ -1012,10 +1021,25 @@ onBeforeUnmount(() => {
         word-wrap: break-word;
       }
 
-      .badge.is-bof {
-        background-color: $teal-500;
+      .badge {
         margin: 0 8px;
 
+        &.is-bof {
+          background-color: $teal-500;
+
+          @at-root .theme-dark & {
+            background-color: $teal-700;
+          }
+        }
+
+        &.is-proposed {
+          background-color: $gray-500;
+
+          @at-root .theme-dark & {
+            background-color: $gray-700;
+          }
+        }
+
         @media screen and (max-width: $bs5-break-md) {
           width: 30px;
           display: block;
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 628125776f..a195e74ce3 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -1786,6 +1786,7 @@ def agenda_extract_schedule (item):
         "type": item.session.type.slug,
         "purpose": item.session.purpose.slug,
         "isBoF": item.session.group_at_the_time().state_id == "bof",
+        "isProposed": item.session.group_at_the_time().state_id == "proposed",
         "filterKeywords": item.filter_keywords,
         "groupAcronym": item.session.group_at_the_time().acronym,
         "groupName": item.session.group_at_the_time().name,

From 8bc51e3cc2db645033974b2805e268a18b098ab8 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Fri, 15 Nov 2024 18:16:34 +0000
Subject: [PATCH 165/656] feat: important dates page: autoid-n to IETFn (#8175)

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/templates/meeting/important-dates.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/templates/meeting/important-dates.html b/ietf/templates/meeting/important-dates.html
index 1d41b4a7f0..568276f566 100644
--- a/ietf/templates/meeting/important-dates.html
+++ b/ietf/templates/meeting/important-dates.html
@@ -19,7 +19,7 @@ <h1>Important Dates</h1>
     </a>
     {% for meeting in meetings %}
         {% if meeting.show_important_dates %}
-            <h2 class="mt-5">
+            <h2 class="mt-5" id="IETF{{ meeting.number }}">
                 IETF {{ meeting.number }}
                 <br>
                 <small class="text-body-secondary">{{ meeting.date }}, {{ meeting.city }}, {{ meeting.country }}</small>

From b07d4dbebc34e6b9abc589899a9290b02791e8aa Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Fri, 15 Nov 2024 18:18:09 +0000
Subject: [PATCH 166/656] feat: add group leadership list (#8135)

* feat: add Group Leadership list

* fix: only offer export to staff

* fix: fix export button conditional

* fix: improve tests. black format

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/group/tests.py                        | 47 ++++++++++++++++++++++
 ietf/group/urls.py                         |  4 +-
 ietf/group/views.py                        | 43 ++++++++++++++++++++
 ietf/templates/group/group_leadership.html | 34 ++++++++++++++++
 4 files changed, 127 insertions(+), 1 deletion(-)
 create mode 100644 ietf/templates/group/group_leadership.html

diff --git a/ietf/group/tests.py b/ietf/group/tests.py
index 130c68b3fc..31f8cc45b5 100644
--- a/ietf/group/tests.py
+++ b/ietf/group/tests.py
@@ -65,6 +65,53 @@ def test_stream_edit(self):
         self.assertTrue(Role.objects.filter(name="delegate", group__acronym=stream_acronym, email__address="ad2@ietf.org"))
 
 
+class GroupLeadershipTests(TestCase):
+    def test_leadership_wg(self):
+        # setup various group states
+        bof_role = RoleFactory(
+            group__type_id="wg", group__state_id="bof", name_id="chair"
+        )
+        proposed_role = RoleFactory(
+            group__type_id="wg", group__state_id="proposed", name_id="chair"
+        )
+        active_role = RoleFactory(
+            group__type_id="wg", group__state_id="active", name_id="chair"
+        )
+        conclude_role = RoleFactory(
+            group__type_id="wg", group__state_id="conclude", name_id="chair"
+        )
+        url = urlreverse(
+            "ietf.group.views.group_leadership", kwargs={"group_type": "wg"}
+        )
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, "Group Leadership")
+        self.assertContains(r, bof_role.person.last_name())
+        self.assertContains(r, proposed_role.person.last_name())
+        self.assertContains(r, active_role.person.last_name())
+        self.assertNotContains(r, conclude_role.person.last_name())
+
+    def test_leadership_wg_csv(self):
+        url = urlreverse(
+            "ietf.group.views.group_leadership_csv", kwargs={"group_type": "wg"}
+        )
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r["Content-Type"], "text/csv")
+        self.assertContains(r, "Chairman, Sops")
+
+    def test_leadership_rg(self):
+        role = RoleFactory(group__type_id="rg", name_id="chair")
+        url = urlreverse(
+            "ietf.group.views.group_leadership", kwargs={"group_type": "rg"}
+        )
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, "Group Leadership")
+        self.assertContains(r, role.person.last_name())
+        self.assertNotContains(r, "Chairman, Sops")
+
+
 class GroupStatsTests(TestCase):
     def setUp(self):
         super().setUp()
diff --git a/ietf/group/urls.py b/ietf/group/urls.py
index b2af8d9e2b..1824564c4d 100644
--- a/ietf/group/urls.py
+++ b/ietf/group/urls.py
@@ -57,7 +57,9 @@
 
 
 group_urls = [
-    url(r'^$', views.active_groups), 
+    url(r'^$', views.active_groups),
+    url(r'^leadership/(?P<group_type>(wg|rg))/$', views.group_leadership),
+    url(r'^leadership/(?P<group_type>(wg|rg))/csv/$', views.group_leadership_csv),
     url(r'^groupstats.json', views.group_stats_data, None, 'ietf.group.views.group_stats_data'),
     url(r'^groupmenu.json', views.group_menu_data, None, 'ietf.group.views.group_menu_data'),
     url(r'^chartering/$', views.chartering_groups),
diff --git a/ietf/group/views.py b/ietf/group/views.py
index 71986384e0..f30569d230 100644
--- a/ietf/group/views.py
+++ b/ietf/group/views.py
@@ -35,6 +35,7 @@
 
 
 import copy
+import csv
 import datetime
 import itertools
 import math
@@ -437,6 +438,48 @@ def prepare_group_documents(request, group, clist):
 
     return docs, meta, docs_related, meta_related
 
+
+def get_leadership(group_type):
+    people = Person.objects.filter(
+        role__name__slug="chair",
+        role__group__type=group_type,
+        role__group__state__slug__in=("active", "bof", "proposed"),
+    ).distinct()
+    leaders = []
+    for person in people:
+        parts = person.name_parts()
+        groups = [
+            r.group.acronym
+            for r in person.role_set.filter(
+                name__slug="chair",
+                group__type=group_type,
+                group__state__slug__in=("active", "bof", "proposed"),
+            )
+        ]
+        entry = {"name": "%s, %s" % (parts[3], parts[1]), "groups": ", ".join(groups)}
+        leaders.append(entry)
+    return sorted(leaders, key=lambda a: a["name"])
+
+
+def group_leadership(request, group_type=None):
+    context = {}
+    context["leaders"] = get_leadership(group_type)
+    context["group_type"] = group_type
+    return render(request, "group/group_leadership.html", context)
+
+
+def group_leadership_csv(request, group_type=None):
+    leaders = get_leadership(group_type)
+    response = HttpResponse(content_type="text/csv")
+    response["Content-Disposition"] = (
+        f'attachment; filename="group_leadership_{group_type}.csv"'
+    )
+    writer = csv.writer(response, dialect=csv.excel, delimiter=str(","))
+    writer.writerow(["Name", "Groups"])
+    for leader in leaders:
+        writer.writerow([leader["name"], leader["groups"]])
+    return response
+
 def group_home(request, acronym, group_type=None):
     group = get_group_or_404(acronym, group_type)
     kwargs = dict(acronym=group.acronym)
diff --git a/ietf/templates/group/group_leadership.html b/ietf/templates/group/group_leadership.html
new file mode 100644
index 0000000000..644be3e150
--- /dev/null
+++ b/ietf/templates/group/group_leadership.html
@@ -0,0 +1,34 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2024, All Rights Reserved #}
+{% load origin static person_filters ietf_filters %}
+{% block pagehead %}
+    <link rel="stylesheet" href="{% static 'ietf/css/list.css' %}">
+{% endblock %}
+{% block title %}Group Leadership{% endblock %}
+{% block content %}
+    {% origin %}
+    <h1>Group Leadership ({{ group_type }})</h1>
+    {% if user|has_role:"Secretariat" %}
+        <div class="text-end">
+            <a class="btn btn-primary" href="{% url 'ietf.group.views.group_leadership_csv' group_type=group_type %}">
+            <i class="bi bi-file-ruled"></i> Export as CSV
+            </a>
+        </div>
+    {% endif %}
+    <table class="table table-sm table-striped">
+        <thead>
+            <tr>
+                <th scope="col">Leader</th>
+                <th scope="col">Groups</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for leader in leaders %}
+                <tr>
+                    <td>{{ leader.name }}</td>
+                    <td>{{ leader.groups }}</td>
+                </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+{% endblock %}

From 6608c9d530b62d10c88d637f949a00fb5fea4526 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 18 Nov 2024 10:53:50 -0400
Subject: [PATCH 167/656] refactor: eliminate single-use helper (#8226)

---
 ietf/doc/utils.py        | 4 ----
 ietf/doc/views_search.py | 4 ++--
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index 9b2570d8ba..a30430829a 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -1046,10 +1046,6 @@ def get_replaces_tree(doc):
     return sorted(history, key=lambda x: x['published'])
 
 
-def get_search_cache_key(key_fragment):
-    return f"doc:document:search:{key_fragment}"
-
-
 def build_file_urls(doc: Union[Document, DocHistory]):
     if doc.type_id == "rfc":
         base_path = os.path.join(settings.RFC_PATH, doc.name + ".")
diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py
index 7b71dd77bd..0f1937efb3 100644
--- a/ietf/doc/views_search.py
+++ b/ietf/doc/views_search.py
@@ -65,7 +65,7 @@
     IESG_BALLOT_ACTIVE_STATES, IESG_STATCHG_CONFLREV_ACTIVE_STATES,
     IESG_CHARTER_ACTIVE_STATES )
 from ietf.doc.fields import select2_id_doc_name_json
-from ietf.doc.utils import get_search_cache_key, augment_events_with_revision, needed_ballot_positions
+from ietf.doc.utils import augment_events_with_revision, needed_ballot_positions
 from ietf.group.models import Group
 from ietf.idindex.index import active_drafts_index_by_group
 from ietf.name.models import DocTagName, DocTypeName, StreamName
@@ -292,7 +292,7 @@ def search(request):
     if request.method == "POST":
         form = SearchForm(data=request.POST)
         if form.is_valid():
-            cache_key = get_search_cache_key(form.cache_key_fragment())
+            cache_key = f"doc:document:search:{form.cache_key_fragment()}"
             cached_val = cache.get(cache_key)
             if cached_val:
                 [results, meta] = cached_val

From 48f339194c69d8d62736fbb3d1f54316a4b2f9ff Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 18 Nov 2024 10:41:40 -0600
Subject: [PATCH 168/656] chore: remove unused merge-person-records (#8227)

---
 ietf/bin/merge-person-records | 65 -----------------------------------
 1 file changed, 65 deletions(-)
 delete mode 100755 ietf/bin/merge-person-records

diff --git a/ietf/bin/merge-person-records b/ietf/bin/merge-person-records
deleted file mode 100755
index 155e5755f6..0000000000
--- a/ietf/bin/merge-person-records
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# -*- Python -*-
-#
-'''
-This script merges two Person records into one.  It determines which record is the target
-based on most current User record (last_login) unless -f (force) option is used to
-force SOURCE TARGET as specified on the command line.  The order of operations is
-important.  We must complete all source.save() operations before moving the aliases to
-the target, this is to avoid extra "Possible duplicate Person" emails going out, if the
-Person is saved without an alias the Person.save() creates another one, which then
-conflicts with the moved one.
-'''
-
-# Set PYTHONPATH and load environment variables for standalone script -----------------
-import os, sys
-basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
-sys.path = [ basedir ] + sys.path
-os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
-
-virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py")
-if os.path.exists(virtualenv_activation):
-    execfile(virtualenv_activation, dict(__file__=virtualenv_activation))
-
-import django
-django.setup()
-# -------------------------------------------------------------------------------------
-
-import argparse
-from django.contrib import admin
-from ietf.person.models import Person
-from ietf.person.utils import (merge_persons, send_merge_notification, handle_users,
-    determine_merge_order)
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("source_id",type=int)
-    parser.add_argument("target_id",type=int)
-    parser.add_argument('-f','--force', help='force merge order',action='store_true')
-    parser.add_argument('-v','--verbose', help='verbose output',action='store_true')
-    args = parser.parse_args()
-
-    source = Person.objects.get(pk=args.source_id)
-    target = Person.objects.get(pk=args.target_id)
-
-    # set merge order
-    if not args.force:
-        source,target = determine_merge_order(source,target)
-
-    # confirm
-    print "Merging person {}({}) to {}({})".format(source.ascii,source.pk,target.ascii,target.pk)
-    print handle_users(source,target,check_only=True)
-    response = raw_input('Ok to continue y/n? ')
-    if response.lower() != 'y':
-        sys.exit()
-
-    # perform merge
-    success, changes = merge_persons(source, target, verbose=args.verbose)
-
-    # send email notification
-    send_merge_notification(target,changes)
-
-if __name__ == "__main__":
-    main()

From 1d7be8c98ce6f6fe420440d01bbe70a843253e6d Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 18 Nov 2024 12:48:47 -0600
Subject: [PATCH 169/656] chore: remove dead activate_this branches (#8228)

---
 ietf/bin/expire-ids         | 4 ----
 ietf/bin/iana-review-email  | 4 ----
 ietf/bin/notify-expirations | 4 ----
 3 files changed, 12 deletions(-)

diff --git a/ietf/bin/expire-ids b/ietf/bin/expire-ids
index 98ee8d75fe..bb0b94ee61 100755
--- a/ietf/bin/expire-ids
+++ b/ietf/bin/expire-ids
@@ -13,10 +13,6 @@ basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
 sys.path = [ basedir ] + sys.path
 os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
 
-virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py")
-if os.path.exists(virtualenv_activation):
-    execfile(virtualenv_activation, dict(__file__=virtualenv_activation))
-
 syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER)
 
 import django
diff --git a/ietf/bin/iana-review-email b/ietf/bin/iana-review-email
index 5c7a7183b9..27aee4015e 100755
--- a/ietf/bin/iana-review-email
+++ b/ietf/bin/iana-review-email
@@ -8,10 +8,6 @@ basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
 sys.path = [ basedir ] + sys.path
 os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
 
-virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py")
-if os.path.exists(virtualenv_activation):
-    execfile(virtualenv_activation, dict(__file__=virtualenv_activation))
-
 syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER)
 
 import django
diff --git a/ietf/bin/notify-expirations b/ietf/bin/notify-expirations
index 0270c13765..fc2fd86a31 100755
--- a/ietf/bin/notify-expirations
+++ b/ietf/bin/notify-expirations
@@ -7,10 +7,6 @@ basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
 sys.path = [ basedir ] + sys.path
 os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
 
-virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py")
-if os.path.exists(virtualenv_activation):
-    execfile(virtualenv_activation, dict(__file__=virtualenv_activation))
-
 import django
 django.setup()
 

From fd816c4f415accd6ebb693b7e0033a2d8cf99487 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 18 Nov 2024 13:57:33 -0600
Subject: [PATCH 170/656] chore: remove unused activate_this code branch
 (#8230)

---
 ietf/bin/expire-submissions | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/ietf/bin/expire-submissions b/ietf/bin/expire-submissions
index 22db38322d..113a53ddfa 100755
--- a/ietf/bin/expire-submissions
+++ b/ietf/bin/expire-submissions
@@ -8,10 +8,6 @@ basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
 sys.path = [ basedir ] + sys.path
 os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
 
-virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py")
-if os.path.exists(virtualenv_activation):
-    execfile(virtualenv_activation, dict(__file__=virtualenv_activation))
-
 syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER)
 
 import django

From 952bc90ee0e3a6a4f773dd1a6125dd7da9f97979 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 19 Nov 2024 08:07:49 -0600
Subject: [PATCH 171/656] fix: follow replaces when building list for diff
 control (#8234)

---
 ietf/doc/tests.py     | 14 ++++++++++++++
 ietf/doc/views_doc.py |  8 ++++----
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index fa655cb884..abac10a5e9 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -5,6 +5,7 @@
 import os
 import datetime
 import io
+from django.http import HttpRequest
 import lxml
 import bibtexparser
 import mock
@@ -52,6 +53,7 @@
     generate_idnits2_rfcs_obsoleted,
     get_doc_email_aliases,
 )
+from ietf.doc.views_doc import get_diff_revisions
 from ietf.group.models import Group, Role
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.ipr.factories import HolderIprDisclosureFactory
@@ -1954,6 +1956,18 @@ def test_writeup(self):
         self.assertContains(r, notes.text)
         self.assertContains(r, rfced_note.text)
 
+    def test_diff_revisions(self):
+        ind_doc = IndividualDraftFactory(create_revisions=range(2))
+        wg_doc = WgDraftFactory(
+            relations=[("replaces", ind_doc)], create_revisions=range(2)
+        )
+        diff_revisions = get_diff_revisions(HttpRequest(), wg_doc.name, wg_doc)
+        self.assertEqual(len(diff_revisions), 4)
+        self.assertEqual(
+            [t[3] for t in diff_revisions],
+            [f"{n}-{v:02d}" for n in [wg_doc.name, ind_doc.name] for v in [1, 0]],
+        )
+
     def test_history(self):
         doc = IndividualDraftFactory()
 
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 50c60aefc5..9f7cf12bcb 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -1133,10 +1133,10 @@ def get_diff_revisions(request, name, doc):
 
     diff_documents = [doc]
     diff_documents.extend(
-        Document.objects.filter(
-            relateddocument__source=doc,
-            relateddocument__relationship="replaces",
-        )
+        [
+            r.target
+            for r in RelatedDocument.objects.filter(source=doc, relationship="replaces")
+        ]
     )
     if doc.came_from_draft():
         diff_documents.append(doc.came_from_draft())

From 71f52bc964b2bd1e19116f1ed601871d4666a280 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 19 Nov 2024 15:01:20 -0500
Subject: [PATCH 172/656] fix: move future meetings button in group meetings
 page to the left (#8239)

---
 ietf/templates/group/meetings.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/templates/group/meetings.html b/ietf/templates/group/meetings.html
index 8acc688cc1..deaea1e675 100644
--- a/ietf/templates/group/meetings.html
+++ b/ietf/templates/group/meetings.html
@@ -40,7 +40,7 @@ <h2 class="mt-5" id="inprogressmeets">Meetings in progress</h2>
     {% if future %}
         <h2 class="mt-5" id="futuremeets">
             Future Meetings
-            <a class="float-end"
+            <a class="ms-2"
                aria-label="icalendar entry for all scheduled future {{ group.acronym }} meetings"
                title="icalendar entry for all scheduled future {{ group.acronym }} meetings"
                href="{% url 'ietf.meeting.views.upcoming_ical' %}?show={{ group.acronym }}">
@@ -174,4 +174,4 @@ <h2 class="mt-5" id="farpastmeets">Meetings more than four years ago</h2>
             ietf_timezone.initialize('local');
     });
     </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

From bdf503a73ba42ea1fe514ecfd8c00ad12638b120 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 19 Nov 2024 15:06:56 -0600
Subject: [PATCH 173/656] fix: show meetecho recordings for groups with more
 than one session (#8238)

---
 ietf/meeting/views.py                         |  1 +
 ietf/templates/meeting/group_proceedings.html | 13 +++++++------
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index a195e74ce3..f386f8932f 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -4105,6 +4105,7 @@ def _format_materials(items):
                 'minutes': _format_materials((s, s.minutes()) for s in ss),
                 'bluesheets': _format_materials((s, s.bluesheets()) for s in ss),
                 'recordings': _format_materials((s, s.recordings()) for s in ss),
+                'meetecho_recordings': _format_materials((s, [s.session_recording_url()]) for s in ss),
                 'chatlogs': _format_materials((s, s.chatlogs()) for s in ss),
                 'slides': _format_materials((s, s.slides()) for s in ss),
                 'drafts': _format_materials((s, s.drafts()) for s in ss),
diff --git a/ietf/templates/meeting/group_proceedings.html b/ietf/templates/meeting/group_proceedings.html
index 95d6dc5da9..496fa92263 100644
--- a/ietf/templates/meeting/group_proceedings.html
+++ b/ietf/templates/meeting/group_proceedings.html
@@ -88,12 +88,13 @@ <h1>{{ entry.group }}</h1>
               </a>
               <br>
           {% endfor %}
-            {% if entry.session.video_stream_url %}
-                <a href="{{ entry.session.session_recording_url }}">
-                    Session recording
-                </a>
-                <br>
-            {% endif %}
+          {% for rec in entry.meetecho_recordings %}
+              <a href="{{ rec.material }}">
+                  Session recording 
+                  {% if rec.time %}{{ rec.time|date:"D G:i"}}{% endif %}
+              </a>
+              <br>
+          {% endfor%}
         </td>
         {# slides #}
         <td>

From 70b8c856774c6916550455dab20ddd8162b42b18 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 19 Nov 2024 16:13:54 -0500
Subject: [PATCH 174/656] ci: Update tests.yml workflow

---
 .github/workflows/tests.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index bb767513c9..4708119875 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -73,9 +73,10 @@ jobs:
         path: geckodriver.log
 
     - name: Upload Coverage Results to Codecov
-      uses: codecov/codecov-action@v4.6.0
+      uses: codecov/codecov-action@v5
       with:
         files: coverage.xml
+        token: ${{ secrets.CODECOV_TOKEN }}
         
     - name: Convert Coverage Results
       if: ${{ always() }}

From 0027db42ebf4a86487a9a4b61c8be7f5dfa8faa8 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 19 Nov 2024 16:15:38 -0500
Subject: [PATCH 175/656] ci: Update tests.yml workflow

---
 .github/workflows/tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 4708119875..8eb6adc953 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -75,6 +75,7 @@ jobs:
     - name: Upload Coverage Results to Codecov
       uses: codecov/codecov-action@v5
       with:
+        disable_search: true
         files: coverage.xml
         token: ${{ secrets.CODECOV_TOKEN }}
         

From 15b95da9a11abc52ea9c51fa8e5c5f4767b10b1d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 19 Nov 2024 16:24:03 -0500
Subject: [PATCH 176/656] chore(deps): bump appleboy/ssh-action from 1.1.0 to
 1.2.0 (#8221)

Bumps [appleboy/ssh-action](https://github.com/appleboy/ssh-action) from 1.1.0 to 1.2.0.
- [Release notes](https://github.com/appleboy/ssh-action/releases)
- [Changelog](https://github.com/appleboy/ssh-action/blob/master/.goreleaser.yaml)
- [Commits](https://github.com/appleboy/ssh-action/compare/25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1...7eaf76671a0d7eec5d98ee897acda4f968735a17)

---
updated-dependencies:
- dependency-name: appleboy/ssh-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/tests-az.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml
index 3f828430a9..6d53a121aa 100644
--- a/.github/workflows/tests-az.yml
+++ b/.github/workflows/tests-az.yml
@@ -38,7 +38,7 @@ jobs:
         ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts
 
     - name: Remote SSH into VM
-      uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1
+      uses: appleboy/ssh-action@7eaf76671a0d7eec5d98ee897acda4f968735a17
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:

From c3e0d28cad7dc8d7974665880f47bcb320f3a18c Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 19 Nov 2024 20:31:02 -0500
Subject: [PATCH 177/656] ci: Update build.yml workflow

---
 .github/workflows/build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 67f24ba76f..0dd6058607 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -137,6 +137,7 @@ jobs:
     uses: ./.github/workflows/tests.yml
     if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
     needs: [prepare]
+    secrets: inherit
     with:
       ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }}
       skipSelenium: true

From 51a76c603d193f4b09b76e6d19b5bc2b64125690 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 20 Nov 2024 11:40:23 -0600
Subject: [PATCH 178/656] fix: removew assumption about what pks tests
 milestones get (#8243)

---
 ietf/group/tests_info.py | 61 +++++++++++++++++++++-------------------
 1 file changed, 32 insertions(+), 29 deletions(-)

diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py
index 95fe9aa2e5..35c8b2b0b7 100644
--- a/ietf/group/tests_info.py
+++ b/ietf/group/tests_info.py
@@ -1431,7 +1431,7 @@ def create_test_milestones(self):
         RoleFactory(group=group,name_id='chair',person=PersonFactory(user__username='marschairman'))
         draft = WgDraftFactory(group=group)
 
-        m1 = GroupMilestone.objects.create(id=1,
+        m1 = GroupMilestone.objects.create(
                                            group=group,
                                            desc="Test 1",
                                            due=date_today(DEADLINE_TZINFO),
@@ -1439,7 +1439,7 @@ def create_test_milestones(self):
                                            state_id="active")
         m1.docs.set([draft])
 
-        m2 = GroupMilestone.objects.create(id=2,
+        m2 = GroupMilestone.objects.create(
                                            group=group,
                                            desc="Test 2",
                                            due=date_today(DEADLINE_TZINFO),
@@ -1580,13 +1580,14 @@ def test_accept_milestone(self):
         events_before = group.groupevent_set.count()
 
         # add
-        r = self.client.post(url, { 'prefix': "m1",
-                                    'm1-id': m1.id,
-                                    'm1-desc': m1.desc,
-                                    'm1-due': m1.due.strftime("%B %Y"),
-                                    'm1-resolved': m1.resolved,
-                                    'm1-docs': pklist(m1.docs),
-                                    'm1-review': "accept",
+        mstr = f"m{m1.id}"
+        r = self.client.post(url, { 'prefix': mstr,
+                                    f'{mstr}-id': m1.id,
+                                    f'{mstr}-desc': m1.desc,
+                                    f'{mstr}-due': m1.due.strftime("%B %Y"),
+                                    f'{mstr}-resolved': m1.resolved,
+                                    f'{mstr}-docs': pklist(m1.docs),
+                                    f'{mstr}-review': "accept",
                                     'action': "save",
                                     })
         self.assertEqual(r.status_code, 302)
@@ -1606,13 +1607,14 @@ def test_delete_milestone(self):
         events_before = group.groupevent_set.count()
 
         # delete
-        r = self.client.post(url, { 'prefix': "m1",
-                                    'm1-id': m1.id,
-                                    'm1-desc': m1.desc,
-                                    'm1-due': m1.due.strftime("%B %Y"),
-                                    'm1-resolved': "",
-                                    'm1-docs': pklist(m1.docs),
-                                    'm1-delete': "checked",
+        mstr = f"m{m1.id}"
+        r = self.client.post(url, { 'prefix': mstr,
+                                    f'{mstr}-id': m1.id,
+                                    f'{mstr}-desc': m1.desc,
+                                    f'{mstr}-due': m1.due.strftime("%B %Y"),
+                                    f'{mstr}-resolved': "",
+                                    f'{mstr}-docs': pklist(m1.docs),
+                                    f'{mstr}-delete': "checked",
                                     'action': "save",
                                     })
         self.assertEqual(r.status_code, 302)
@@ -1635,13 +1637,14 @@ def test_edit_milestone(self):
 
         due = self.last_day_of_month(date_today(DEADLINE_TZINFO) + datetime.timedelta(days=365))
 
+        mstr = f"m{m1.id}"
         # faulty post
-        r = self.client.post(url, { 'prefix': "m1",
-                                    'm1-id': m1.id,
-                                    'm1-desc': "", # no description
-                                    'm1-due': due.strftime("%B %Y"),
-                                    'm1-resolved': "",
-                                    'm1-docs': doc_pks,
+        r = self.client.post(url, { 'prefix': mstr,
+                                    f'{mstr}-id': m1.id,
+                                    f'{mstr}-desc': "", # no description
+                                    f'{mstr}-due': due.strftime("%B %Y"),
+                                    f'{mstr}-resolved': "",
+                                    f'{mstr}-docs': doc_pks,
                                     'action': "save",
                                     })
         self.assertEqual(r.status_code, 200)
@@ -1653,13 +1656,13 @@ def test_edit_milestone(self):
 
         # edit
         mailbox_before = len(outbox)
-        r = self.client.post(url, { 'prefix': "m1",
-                                    'm1-id': m1.id,
-                                    'm1-desc': "Test 2 - changed",
-                                    'm1-due': due.strftime("%B %Y"),
-                                    'm1-resolved': "Done",
-                                    'm1-resolved_checkbox': "checked",
-                                    'm1-docs': doc_pks,
+        r = self.client.post(url, { 'prefix': mstr,
+                                    f'{mstr}-id': m1.id,
+                                    f'{mstr}-desc': "Test 2 - changed",
+                                    f'{mstr}-due': due.strftime("%B %Y"),
+                                    f'{mstr}-resolved': "Done",
+                                    f'{mstr}-resolved_checkbox': "checked",
+                                    f'{mstr}-docs': doc_pks,
                                     'action': "save",
                                     })
         self.assertEqual(r.status_code, 302)

From 7c025c9f2bbff6c229ec7b7befcdd2534bb84900 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 20 Nov 2024 12:59:39 -0500
Subject: [PATCH 179/656] chore(deps): bump nanoid in /dev/deploy-to-container
 in the npm group (#8098)

Bumps the npm group in /dev/deploy-to-container with 1 update: [nanoid](https://github.com/ai/nanoid).


Updates `nanoid` from 5.0.7 to 5.0.8
- [Release notes](https://github.com/ai/nanoid/releases)
- [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md)
- [Commits](https://github.com/ai/nanoid/compare/5.0.7...5.0.8)

---
updated-dependencies:
- dependency-name: nanoid
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/deploy-to-container/package-lock.json | 14 +++++++-------
 dev/deploy-to-container/package.json      |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json
index f636a45f05..d787f6aa9e 100644
--- a/dev/deploy-to-container/package-lock.json
+++ b/dev/deploy-to-container/package-lock.json
@@ -8,7 +8,7 @@
       "dependencies": {
         "dockerode": "^4.0.2",
         "fs-extra": "^11.2.0",
-        "nanoid": "5.0.7",
+        "nanoid": "5.0.8",
         "nanoid-dictionary": "5.0.0-beta.1",
         "slugify": "1.6.6",
         "tar": "^7.4.3",
@@ -546,9 +546,9 @@
       "optional": true
     },
     "node_modules/nanoid": {
-      "version": "5.0.7",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.7.tgz",
-      "integrity": "sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ==",
+      "version": "5.0.8",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.8.tgz",
+      "integrity": "sha512-TcJPw+9RV9dibz1hHUzlLVy8N4X9TnwirAjrU08Juo6BNKggzVfP2ZJ/3ZUSq15Xl5i85i+Z89XBO90pB2PghQ==",
       "funding": [
         {
           "type": "github",
@@ -1346,9 +1346,9 @@
       "optional": true
     },
     "nanoid": {
-      "version": "5.0.7",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.7.tgz",
-      "integrity": "sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ=="
+      "version": "5.0.8",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.8.tgz",
+      "integrity": "sha512-TcJPw+9RV9dibz1hHUzlLVy8N4X9TnwirAjrU08Juo6BNKggzVfP2ZJ/3ZUSq15Xl5i85i+Z89XBO90pB2PghQ=="
     },
     "nanoid-dictionary": {
       "version": "5.0.0-beta.1",
diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json
index be77fa5cce..530d1f3b8b 100644
--- a/dev/deploy-to-container/package.json
+++ b/dev/deploy-to-container/package.json
@@ -4,7 +4,7 @@
   "dependencies": {
     "dockerode": "^4.0.2",
     "fs-extra": "^11.2.0",
-    "nanoid": "5.0.7",
+    "nanoid": "5.0.8",
     "nanoid-dictionary": "5.0.0-beta.1",
     "slugify": "1.6.6",
     "tar": "^7.4.3",

From 3c42d98c5d759045fbb0535fd187969600eb4b22 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 20 Nov 2024 13:09:28 -0500
Subject: [PATCH 180/656] ci: Update build.yml workflow

---
 .github/workflows/build.yml | 60 ++++++++++++++-----------------------
 1 file changed, 22 insertions(+), 38 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0dd6058607..f1044223dc 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -350,50 +350,34 @@ jobs:
     steps:      
     - name: Notify on Slack (Success)
       if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
-      uses: slackapi/slack-github-action@v1.27.0
+      uses: slackapi/slack-github-action@v2
       with:
-        channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
+        token: ${{ secrets.SLACK_GH_BOT }}
+        method: chat.postMessage
         payload: |
-          {
-            "text": "Datatracker Build <https://github.com/ietf-tools/datatracker/actions/runs/${{ github.run_id }}|${{ env.PKG_VERSION }}> by ${{ github.triggering_actor }} - <@${{ secrets.SLACK_UID_RJSPARKS }}>",
-            "attachments": [
-              {
-                "color": "28a745",
-                "fields": [
-                  {
-                    "title": "Status",
-                    "short": true,
-                    "value": "Completed"
-                  }
-                ]
-              }
-            ]
-          }
-      env:
-        SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}
+          channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
+          text: "Datatracker Build <https://github.com/ietf-tools/datatracker/actions/runs/${{ github.run_id }}|${{ env.PKG_VERSION }}> by ${{ github.triggering_actor }}"
+          attachments:
+            - color: "28a745"
+              fields:
+                - title: "Status"
+                  short: true
+                  value: "Completed"
     - name: Notify on Slack (Failure)
       if: ${{ contains(join(needs.*.result, ','), 'failure') }}
-      uses: slackapi/slack-github-action@v1.27.0
+      uses: slackapi/slack-github-action@v2
       with:
-        channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
+        token: ${{ secrets.SLACK_GH_BOT }}
+        method: chat.postMessage
         payload: |
-          {
-            "text": "Datatracker Build <https://github.com/ietf-tools/datatracker/actions/runs/${{ github.run_id }}|${{ env.PKG_VERSION }}> by ${{ github.triggering_actor }} - <@${{ secrets.SLACK_UID_RJSPARKS }}>",
-            "attachments": [
-              {
-                "color": "a82929",
-                "fields": [
-                  {
-                    "title": "Status",
-                    "short": true,
-                    "value": "Failed"
-                  }
-                ]
-              }
-            ]
-          }
-      env:
-        SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}
+          channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
+          text: "Datatracker Build <https://github.com/ietf-tools/datatracker/actions/runs/${{ github.run_id }}|${{ env.PKG_VERSION }}> by ${{ github.triggering_actor }}"
+          attachments:
+            - color: "a82929"
+              fields:
+                - title: "Status"
+                  short: true
+                  value: "Failed"
         
   # -----------------------------------------------------------------
   # SANDBOX

From bdf753e05250129475c3310fa60cc31240028932 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 20 Nov 2024 17:48:00 -0500
Subject: [PATCH 181/656] chore: set gitattributes for normalizing line endings
 (#8245)

* chore: add proper .gitattributes

* chore: normalize file endings

* chore: fix .gitattributes

* chore: normalize file endings (2)
---
 .gitattributes                                | 282 ++++++++++++-
 dev/build/settings_local_collectstatics.py    |  16 +-
 dev/tests/docker-compose.debug.yml            |  64 +--
 docker/configs/pgadmin-servers.json           |  44 +-
 ietf/dbtemplate/fixtures/nomcom_templates.xml | 380 +++++++++---------
 ietf/secr/static/js/dynamic_inlines.js        | 142 +++----
 k8s/README.md                                 |   8 +-
 k8s/kustomization.yaml                        |  32 +-
 k8s/memcached.yaml                            | 160 ++++----
 k8s/secrets.yaml                              | 164 ++++----
 10 files changed, 785 insertions(+), 507 deletions(-)

diff --git a/.gitattributes b/.gitattributes
index 937c0eb379..62f4aae432 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,280 @@
-/.yarn/releases/** binary
-/.yarn/plugins/** binary
+# Auto detect text files and perform LF normalization
+* text=auto
+
+# ---------------------------------------------------
+# Python Projects
+# ---------------------------------------------------
+
+# Source files
+*.pxd    text diff=python
+*.py     text diff=python
+*.py3    text diff=python
+*.pyw    text diff=python
+*.pyx    text diff=python
+*.pyz    text diff=python
+*.pyi    text diff=python
+
+# Binary files
+*.db     binary
+*.p      binary
+*.pkl    binary
+*.pickle binary
+*.pyc    binary export-ignore
+*.pyo    binary export-ignore
+*.pyd    binary
+
+# Jupyter notebook
+*.ipynb  text eol=lf
+
+# ---------------------------------------------------
+# Web Projects
+# ---------------------------------------------------
+
+# Source code
+*.bash            text eol=lf
+*.bat             text eol=crlf
+*.cmd             text eol=crlf
+*.coffee          text
+*.css             text diff=css
+*.htm             text diff=html
+*.html            text diff=html
+*.inc             text
+*.ini             text
+*.js              text
+*.mjs             text
+*.cjs             text
+*.json            text
+*.jsx             text
+*.less            text
+*.ls              text
+*.map             text -diff
+*.od              text
+*.onlydata        text
+*.php             text diff=php
+*.pl              text
+*.ps1             text eol=crlf
+*.py              text diff=python
+*.rb              text diff=ruby
+*.sass            text
+*.scm             text
+*.scss            text diff=css
+*.sh              text eol=lf
+.husky/*          text eol=lf
+*.sql             text
+*.styl            text
+*.tag             text
+*.ts              text
+*.tsx             text
+*.xml             text
+*.xhtml           text diff=html
+
+# Docker
+Dockerfile        text
+
+# Documentation
+*.ipynb           text eol=lf
+*.markdown        text diff=markdown
+*.md              text diff=markdown
+*.mdwn            text diff=markdown
+*.mdown           text diff=markdown
+*.mkd             text diff=markdown
+*.mkdn            text diff=markdown
+*.mdtxt           text
+*.mdtext          text
+*.txt             text
+AUTHORS           text
+CHANGELOG         text
+CHANGES           text
+CONTRIBUTING      text
+COPYING           text
+copyright         text
+*COPYRIGHT*       text
+INSTALL           text
+license           text
+LICENSE           text
+NEWS              text
+readme            text
+*README*          text
+TODO              text
+
+# Templates
+*.dot             text
+*.ejs             text
+*.erb             text
+*.haml            text
+*.handlebars      text
+*.hbs             text
+*.hbt             text
+*.jade            text
+*.latte           text
+*.mustache        text
+*.njk             text
+*.phtml           text
+*.pug             text
+*.svelte          text
+*.tmpl            text
+*.tpl             text
+*.twig            text
+*.vue             text
+
+# Configs
+*.cnf             text
+*.conf            text
+*.config          text
+.editorconfig     text
+.env              text
+.gitattributes    text
+.gitconfig        text
+.htaccess         text
+*.lock            text -diff
+package.json      text eol=lf
+package-lock.json text eol=lf -diff
+pnpm-lock.yaml    text eol=lf -diff
+.prettierrc       text
+yarn.lock         text -diff
+*.toml            text
+*.yaml            text
+*.yml             text
+browserslist      text
+Makefile          text
+makefile          text
+# Fixes syntax highlighting on GitHub to allow comments
+tsconfig.json     linguist-language=JSON-with-Comments
+
+# Heroku
+Procfile          text
+
+# Graphics
+*.ai              binary
+*.bmp             binary
+*.eps             binary
+*.gif             binary
+*.gifv            binary
+*.ico             binary
+*.jng             binary
+*.jp2             binary
+*.jpg             binary
+*.jpeg            binary
+*.jpx             binary
+*.jxr             binary
+*.pdf             binary
+*.png             binary
+*.psb             binary
+*.psd             binary
+*.svg             text
+*.svgz            binary
+*.tif             binary
+*.tiff            binary
+*.wbmp            binary
+*.webp            binary
+
+# Audio
+*.kar             binary
+*.m4a             binary
+*.mid             binary
+*.midi            binary
+*.mp3             binary
+*.ogg             binary
+*.ra              binary
+
+# Video
+*.3gpp            binary
+*.3gp             binary
+*.as              binary
+*.asf             binary
+*.asx             binary
+*.avi             binary
+*.fla             binary
+*.flv             binary
+*.m4v             binary
+*.mng             binary
+*.mov             binary
+*.mp4             binary
+*.mpeg            binary
+*.mpg             binary
+*.ogv             binary
+*.swc             binary
+*.swf             binary
+*.webm            binary
+
+# Archives
+*.7z              binary
+*.gz              binary
+*.jar             binary
+*.rar             binary
+*.tar             binary
+*.zip             binary
+
+# Fonts
+*.ttf             binary
+*.eot             binary
+*.otf             binary
+*.woff            binary
+*.woff2           binary
+
+# Executables
+*.exe             binary
+*.pyc             binary
+# Prevents massive diffs caused by vendored, minified files
+**/.yarn/releases/**   binary
+**/.yarn/plugins/**    binary
+
+# RC files (like .babelrc or .eslintrc)
+*.*rc             text
+
+# Ignore files (like .npmignore or .gitignore)
+*.*ignore         text
+
+# Prevents massive diffs from built files
+dist/*            binary
+
+# ---------------------------------------------------
+# Common
+# ---------------------------------------------------
+
+# Documents
+*.bibtex   text diff=bibtex
+*.doc      diff=astextplain
+*.DOC      diff=astextplain
+*.docx     diff=astextplain
+*.DOCX     diff=astextplain
+*.dot      diff=astextplain
+*.DOT      diff=astextplain
+*.pdf      diff=astextplain
+*.PDF      diff=astextplain
+*.rtf      diff=astextplain
+*.RTF      diff=astextplain
+*.md       text diff=markdown
+*.mdx      text diff=markdown
+*.tex      text diff=tex
+*.adoc     text
+*.textile  text
+*.mustache text
+*.csv      text eol=crlf
+*.tab      text
+*.tsv      text
+*.txt      text
+*.sql      text
+*.epub     diff=astextplain
+
+# Text files where line endings should be preserved
+*.patch    -text
+
+# ---------------------------------------------------
+# Vzic specific
+# ---------------------------------------------------
+
+*.pl text diff=perl
+*.pm text diff=perl
+
+# C/C++
+*.c     text diff=cpp
+*.cc    text diff=cpp
+*.cxx   text diff=cpp
+*.cpp   text diff=cpp
+*.cpi   text diff=cpp
+*.c++   text diff=cpp
+*.hpp   text diff=cpp
+*.h     text diff=cpp
+*.h++   text diff=cpp
+*.hh    text diff=cpp
\ No newline at end of file
diff --git a/dev/build/settings_local_collectstatics.py b/dev/build/settings_local_collectstatics.py
index 016b3f934a..ccb4b33979 100644
--- a/dev/build/settings_local_collectstatics.py
+++ b/dev/build/settings_local_collectstatics.py
@@ -1,8 +1,8 @@
-# Copyright The IETF Trust 2007-2019, All Rights Reserved
-# -*- coding: utf-8 -*-
-
-from ietf import __version__
-from ietf.settings import *                                          # pyflakes:ignore
-
-STATIC_URL = "https://static.ietf.org/dt/%s/"%__version__
-STATIC_ROOT = os.path.abspath(BASE_DIR + "/../static/")
+# Copyright The IETF Trust 2007-2019, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from ietf import __version__
+from ietf.settings import *                                          # pyflakes:ignore
+
+STATIC_URL = "https://static.ietf.org/dt/%s/"%__version__
+STATIC_ROOT = os.path.abspath(BASE_DIR + "/../static/")
diff --git a/dev/tests/docker-compose.debug.yml b/dev/tests/docker-compose.debug.yml
index 6362ef072e..8d939e0ea2 100644
--- a/dev/tests/docker-compose.debug.yml
+++ b/dev/tests/docker-compose.debug.yml
@@ -1,32 +1,32 @@
-# This docker-compose replicates the test workflow happening on GitHub during a PR / build check.
-# To be used from the debug.sh script.
-
-version: '3.8'
-
-services:
-    app:
-        image: ghcr.io/ietf-tools/datatracker-app-base:latest
-        command: -f /dev/null
-        working_dir: /__w/datatracker/datatracker
-        entrypoint: tail
-        hostname: app
-        volumes:
-            - /var/run/docker.sock:/var/run/docker.sock
-        environment:
-            CI: 'true'
-            GITHUB_ACTIONS: 'true'
-            HOME: /github/home
-        deploy:    
-            resources:
-                limits:
-                    cpus: '2'
-                    memory: '7GB'
-
-    db:
-        image: ghcr.io/ietf-tools/datatracker-db:latest
-        restart: unless-stopped
-        volumes:
-            - postgresdb-data:/var/lib/postgresql/data
-
-volumes:
-    postgresdb-data:
+# This docker-compose replicates the test workflow happening on GitHub during a PR / build check.
+# To be used from the debug.sh script.
+
+version: '3.8'
+
+services:
+    app:
+        image: ghcr.io/ietf-tools/datatracker-app-base:latest
+        command: -f /dev/null
+        working_dir: /__w/datatracker/datatracker
+        entrypoint: tail
+        hostname: app
+        volumes:
+            - /var/run/docker.sock:/var/run/docker.sock
+        environment:
+            CI: 'true'
+            GITHUB_ACTIONS: 'true'
+            HOME: /github/home
+        deploy:    
+            resources:
+                limits:
+                    cpus: '2'
+                    memory: '7GB'
+
+    db:
+        image: ghcr.io/ietf-tools/datatracker-db:latest
+        restart: unless-stopped
+        volumes:
+            - postgresdb-data:/var/lib/postgresql/data
+
+volumes:
+    postgresdb-data:
diff --git a/docker/configs/pgadmin-servers.json b/docker/configs/pgadmin-servers.json
index 8b1c181d13..b4458af923 100644
--- a/docker/configs/pgadmin-servers.json
+++ b/docker/configs/pgadmin-servers.json
@@ -1,22 +1,22 @@
-{
-  "Servers": {
-      "1": {
-          "Name": "Local Dev",
-          "Group": "Servers",
-          "Host": "db",
-          "Port": 5432,
-          "MaintenanceDB": "postgres",
-          "Username": "django",
-          "UseSSHTunnel": 0,
-          "TunnelPort": "22",
-          "TunnelAuthentication": 0,
-          "KerberosAuthentication": false,
-          "ConnectionParameters": {
-              "sslmode": "prefer",
-              "connect_timeout": 10,
-              "sslcert": "<STORAGE_DIR>/.postgresql/postgresql.crt",
-              "sslkey": "<STORAGE_DIR>/.postgresql/postgresql.key"
-          }
-      }
-  }
-}
+{
+  "Servers": {
+      "1": {
+          "Name": "Local Dev",
+          "Group": "Servers",
+          "Host": "db",
+          "Port": 5432,
+          "MaintenanceDB": "postgres",
+          "Username": "django",
+          "UseSSHTunnel": 0,
+          "TunnelPort": "22",
+          "TunnelAuthentication": 0,
+          "KerberosAuthentication": false,
+          "ConnectionParameters": {
+              "sslmode": "prefer",
+              "connect_timeout": 10,
+              "sslcert": "<STORAGE_DIR>/.postgresql/postgresql.crt",
+              "sslkey": "<STORAGE_DIR>/.postgresql/postgresql.key"
+          }
+      }
+  }
+}
diff --git a/ietf/dbtemplate/fixtures/nomcom_templates.xml b/ietf/dbtemplate/fixtures/nomcom_templates.xml
index abf0cb58f6..e7065b84cd 100644
--- a/ietf/dbtemplate/fixtures/nomcom_templates.xml
+++ b/ietf/dbtemplate/fixtures/nomcom_templates.xml
@@ -1,190 +1,190 @@
-<?xml version="1.0" encoding="utf-8"?>
-<django-objects version="1.0">
-    <object pk="1" model="dbtemplate.dbtemplate">
-        <field type="CharField" name="path">/nomcom/defaults/home.rst</field>
-        <field type="CharField" name="title">Home page of group</field>
-        <field type="TextField" name="variables"></field>
-        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">rst</field>
-        <field type="TextField" name="content">Home page
-=========
-
-This is the home page of the nomcom group.</field>
-        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
-    </object>
-    <object pk="2" model="dbtemplate.dbtemplate">
-        <field type="CharField" name="path">/nomcom/defaults/email/inexistent_person.txt</field>
-        <field type="CharField" name="title">Email sent to chair of nomcom and secretariat when Email and Person are created if some of them don't exist</field>
-        <field type="TextField" name="variables">$email: Newly created email
-$fullname: Fullname of the new person
-$person_id: Id of the new Person object
-$group: Name of the group</field>
-        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
-        <field type="TextField" name="content">Hello,
-
-A new person with name $fullname and email $email has been created. The new Person object has the following id: '$person_id'.
-
-Please, check if there is some more action nedeed.</field>
-        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
-    </object>
-    <object pk="3" model="dbtemplate.dbtemplate">
-        <field type="CharField" name="path">/nomcom/defaults/email/new_nominee.txt</field>
-        <field type="CharField" name="title">Email sent to nominees when they are nominated</field>
-        <field type="TextField" name="variables">$nominee: Full name of the nominee
-$position: Name of the position
-$domain: Server domain
-$accept_url: Url hash to accept nominations
-$decline_url: Url hash to decline nominations</field>
-        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
-        <field type="TextField" name="content">Hi,
-
-You have been nominated for the position of $position.
-
-The NomCom would appreciate receiving an indication of whether or not you accept this nomination to stand for consideration as a candidate for this position.
-
-You can accept the nomination via web going to the following link https://$domain$accept_url or decline the nomination going the following link https://$domain$decline_url
-
-If you accept, you will need to fill out a questionnaire.  You will receive the questionnaire by email.
-
-Best regards,
-</field>
-        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
-    </object>
-    <object pk="4" model="dbtemplate.dbtemplate">
-        <field type="CharField" name="path">/nomcom/defaults/email/new_nomination.txt</field>
-        <field type="CharField" name="title">Email sent to nominators and secretariat when the nominators make the nominations</field>
-        <field type="TextField" name="variables">$nominator: Full name of the nominator
-$nominator_email: Email of the nominator
-$nominee: Full name of the nominee
-$nominee_email: Email of the nominee
-$position: Nomination position</field>
-        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
-        <field type="TextField" name="content">A new nomination have been received.
-
-Nominator: $nominator ($nominator_email)
-Nominee: $nominee ($nominee_email)
-Position: $position</field>
-        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
-    </object>
-    <object pk="5" model="dbtemplate.dbtemplate">
-        <field type="CharField" name="path">/nomcom/defaults/position/questionnaire.txt</field>
-        <field type="CharField" name="title">Questionnaire sent to the nomine</field>
-        <field type="TextField" name="variables">$position: Position</field>
-        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
-        <field type="TextField" name="content">Enter here the questionnaire for the position $position:
-
-Questionnaire</field>
-        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
-    </object>
-    <object pk="6" model="dbtemplate.dbtemplate">
-        <field type="CharField" name="path">/nomcom/defaults/position/requirements</field>
-        <field type="CharField" name="title">Position requirements</field>
-        <field type="TextField" name="variables">$position: Position</field>
-        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">rst</field>
-        <field type="TextField" name="content">These are the requirements for the position $position:
-
-Requirements.</field>
-        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
-    </object>
-    <object pk="7" model="dbtemplate.dbtemplate">
-        <field type="CharField" name="path">/nomcom/defaults/position/header_questionnaire.txt</field>
-        <field type="CharField" name="title">Header of the email that contains the questionnaire sent to the nomine</field>
-        <field type="TextField" name="variables">$nominee: Full name of the nomine
-$position: Position</field>
-        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
-        <field type="TextField" name="content">Hi $nominee, this is the questionnaire for the position $position:
-
-</field>
-        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
-    </object>
-    <object pk="8" model="dbtemplate.dbtemplate">
-        <field type="CharField" name="path">/nomcom/defaults/email/nomination_accept_reminder.txt</field>
-        <field type="CharField" name="title">Email sent to nominees asking them to accept (or decline) the nominations.</field>
-        <field type="TextField" name="variables">$positions: Nomination positions</field>
-        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
-        <field type="TextField" name="content">Hi,
-
-You have been nominated for the position of $position.
-
-The NomCom would appreciate receiving an indication of whether or not you accept this nomination to stand for consideration as a candidate for this position.
-
-You can accept the nomination via web going to the following link https://$domain$accept_url or decline the nomination going the following link https://$domain$decline_url
-
-If you accept, you will need to fill out a questionnaire.
-
-Best regards,</field>
-        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
-    </object>
-    <object pk="9" model="dbtemplate.dbtemplate">
-        <field type="CharField" name="path">/nomcom/defaults/email/nomination_receipt.txt</field>
-        <field type="CharField" name="title">Email sent to nominator to get a confirmation mail containing feedback in cleartext</field>
-        <field type="TextField" name="variables">$nominee: Full name of the nominee
-$position: Name of the position
-$domain: Server domain
-$accept_url: Url hash to accept nominations
-$decline_url: Url hash to decline nominations</field>
-        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
-        <field type="TextField" name="content">Hi,
-
-Your nomination of $nominee for the position of
-$position has been received and registered.
-
-The following comments have also been registered:
-
---------------------------------------------------------------------------
-$comments
---------------------------------------------------------------------------
-
-Thank you,</field>
-        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
-    </object>
-    <object pk="10" model="dbtemplate.dbtemplate">
-        <field type="CharField" name="path">/nomcom/defaults/email/feedback_receipt.txt</field>
-        <field type="CharField" name="title">Email sent to feedback author to get a confirmation mail containing feedback in cleartext</field>
-        <field type="TextField" name="variables">$nominee: Full name of the nominee
-$position: Nomination position
-$comments: Comments on this candidate</field>
-        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
-        <field type="TextField" name="content">Hi,
-
-Your input regarding $about has been received and registered.
-
-The following comments have been registered:
-
---------------------------------------------------------------------------
-$comments
---------------------------------------------------------------------------
-
-Thank you,</field>
-        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
-    </object>
-    <object pk="11" model="dbtemplate.dbtemplate">
-        <field type="CharField" name="path">/nomcom/defaults/email/questionnaire_reminder.txt</field>
-        <field type="CharField" name="title">Email sent to nominees reminding them to complete a questionnaire</field>
-        <field type="TextField" name="variables">$positions: Nomination positions</field>
-        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
-        <field type="TextField" name="content">
-Thank you for accepting your nomination for the position of $position.
-
-Please remember to complete and return the questionnaire for this position at your earliest opportunity.
-The questionnaire is repeated below for your convenience.
-
---------</field>
-        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
-    </object>
-    <object pk="12" model="dbtemplate.dbtemplate">
-      <field type="CharField" name="path">/nomcom/defaults/topic/description</field>
-      <field type="CharField" name="title">Description of Topic</field>
-      <field type="TextField" name="variables">$topic: Topic'</field>
-      <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">rst</field>
-      <field type="TextField" name="content">This is a description of the topic "$topic"
-
-Describe the topic and add any information/instructions for the responder here.
-</field>
-    </object>
-    <object pk="13" model="dbtemplate.dbtemplate">
-        <field type="CharField" name="path">/nomcom/defaults/iesg_requirements</field>
-        <field type="CharField" name="title">Generic IESG Requirements</field>
-        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">rst</field>
-        <field type="TextField" name="content">Generic IESG Requirements Yo!</field>
-    </object>
-</django-objects>
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+    <object pk="1" model="dbtemplate.dbtemplate">
+        <field type="CharField" name="path">/nomcom/defaults/home.rst</field>
+        <field type="CharField" name="title">Home page of group</field>
+        <field type="TextField" name="variables"></field>
+        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">rst</field>
+        <field type="TextField" name="content">Home page
+=========
+
+This is the home page of the nomcom group.</field>
+        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
+    </object>
+    <object pk="2" model="dbtemplate.dbtemplate">
+        <field type="CharField" name="path">/nomcom/defaults/email/inexistent_person.txt</field>
+        <field type="CharField" name="title">Email sent to chair of nomcom and secretariat when Email and Person are created if some of them don't exist</field>
+        <field type="TextField" name="variables">$email: Newly created email
+$fullname: Fullname of the new person
+$person_id: Id of the new Person object
+$group: Name of the group</field>
+        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
+        <field type="TextField" name="content">Hello,
+
+A new person with name $fullname and email $email has been created. The new Person object has the following id: '$person_id'.
+
+Please, check if there is some more action nedeed.</field>
+        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
+    </object>
+    <object pk="3" model="dbtemplate.dbtemplate">
+        <field type="CharField" name="path">/nomcom/defaults/email/new_nominee.txt</field>
+        <field type="CharField" name="title">Email sent to nominees when they are nominated</field>
+        <field type="TextField" name="variables">$nominee: Full name of the nominee
+$position: Name of the position
+$domain: Server domain
+$accept_url: Url hash to accept nominations
+$decline_url: Url hash to decline nominations</field>
+        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
+        <field type="TextField" name="content">Hi,
+
+You have been nominated for the position of $position.
+
+The NomCom would appreciate receiving an indication of whether or not you accept this nomination to stand for consideration as a candidate for this position.
+
+You can accept the nomination via web going to the following link https://$domain$accept_url or decline the nomination going the following link https://$domain$decline_url
+
+If you accept, you will need to fill out a questionnaire.  You will receive the questionnaire by email.
+
+Best regards,
+</field>
+        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
+    </object>
+    <object pk="4" model="dbtemplate.dbtemplate">
+        <field type="CharField" name="path">/nomcom/defaults/email/new_nomination.txt</field>
+        <field type="CharField" name="title">Email sent to nominators and secretariat when the nominators make the nominations</field>
+        <field type="TextField" name="variables">$nominator: Full name of the nominator
+$nominator_email: Email of the nominator
+$nominee: Full name of the nominee
+$nominee_email: Email of the nominee
+$position: Nomination position</field>
+        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
+        <field type="TextField" name="content">A new nomination have been received.
+
+Nominator: $nominator ($nominator_email)
+Nominee: $nominee ($nominee_email)
+Position: $position</field>
+        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
+    </object>
+    <object pk="5" model="dbtemplate.dbtemplate">
+        <field type="CharField" name="path">/nomcom/defaults/position/questionnaire.txt</field>
+        <field type="CharField" name="title">Questionnaire sent to the nomine</field>
+        <field type="TextField" name="variables">$position: Position</field>
+        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
+        <field type="TextField" name="content">Enter here the questionnaire for the position $position:
+
+Questionnaire</field>
+        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
+    </object>
+    <object pk="6" model="dbtemplate.dbtemplate">
+        <field type="CharField" name="path">/nomcom/defaults/position/requirements</field>
+        <field type="CharField" name="title">Position requirements</field>
+        <field type="TextField" name="variables">$position: Position</field>
+        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">rst</field>
+        <field type="TextField" name="content">These are the requirements for the position $position:
+
+Requirements.</field>
+        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
+    </object>
+    <object pk="7" model="dbtemplate.dbtemplate">
+        <field type="CharField" name="path">/nomcom/defaults/position/header_questionnaire.txt</field>
+        <field type="CharField" name="title">Header of the email that contains the questionnaire sent to the nomine</field>
+        <field type="TextField" name="variables">$nominee: Full name of the nomine
+$position: Position</field>
+        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
+        <field type="TextField" name="content">Hi $nominee, this is the questionnaire for the position $position:
+
+</field>
+        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
+    </object>
+    <object pk="8" model="dbtemplate.dbtemplate">
+        <field type="CharField" name="path">/nomcom/defaults/email/nomination_accept_reminder.txt</field>
+        <field type="CharField" name="title">Email sent to nominees asking them to accept (or decline) the nominations.</field>
+        <field type="TextField" name="variables">$positions: Nomination positions</field>
+        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
+        <field type="TextField" name="content">Hi,
+
+You have been nominated for the position of $position.
+
+The NomCom would appreciate receiving an indication of whether or not you accept this nomination to stand for consideration as a candidate for this position.
+
+You can accept the nomination via web going to the following link https://$domain$accept_url or decline the nomination going the following link https://$domain$decline_url
+
+If you accept, you will need to fill out a questionnaire.
+
+Best regards,</field>
+        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
+    </object>
+    <object pk="9" model="dbtemplate.dbtemplate">
+        <field type="CharField" name="path">/nomcom/defaults/email/nomination_receipt.txt</field>
+        <field type="CharField" name="title">Email sent to nominator to get a confirmation mail containing feedback in cleartext</field>
+        <field type="TextField" name="variables">$nominee: Full name of the nominee
+$position: Name of the position
+$domain: Server domain
+$accept_url: Url hash to accept nominations
+$decline_url: Url hash to decline nominations</field>
+        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
+        <field type="TextField" name="content">Hi,
+
+Your nomination of $nominee for the position of
+$position has been received and registered.
+
+The following comments have also been registered:
+
+--------------------------------------------------------------------------
+$comments
+--------------------------------------------------------------------------
+
+Thank you,</field>
+        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
+    </object>
+    <object pk="10" model="dbtemplate.dbtemplate">
+        <field type="CharField" name="path">/nomcom/defaults/email/feedback_receipt.txt</field>
+        <field type="CharField" name="title">Email sent to feedback author to get a confirmation mail containing feedback in cleartext</field>
+        <field type="TextField" name="variables">$nominee: Full name of the nominee
+$position: Nomination position
+$comments: Comments on this candidate</field>
+        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
+        <field type="TextField" name="content">Hi,
+
+Your input regarding $about has been received and registered.
+
+The following comments have been registered:
+
+--------------------------------------------------------------------------
+$comments
+--------------------------------------------------------------------------
+
+Thank you,</field>
+        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
+    </object>
+    <object pk="11" model="dbtemplate.dbtemplate">
+        <field type="CharField" name="path">/nomcom/defaults/email/questionnaire_reminder.txt</field>
+        <field type="CharField" name="title">Email sent to nominees reminding them to complete a questionnaire</field>
+        <field type="TextField" name="variables">$positions: Nomination positions</field>
+        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">plain</field>
+        <field type="TextField" name="content">
+Thank you for accepting your nomination for the position of $position.
+
+Please remember to complete and return the questionnaire for this position at your earliest opportunity.
+The questionnaire is repeated below for your convenience.
+
+--------</field>
+        <field to="group.group" name="group" rel="ManyToOneRel"><None></None></field>
+    </object>
+    <object pk="12" model="dbtemplate.dbtemplate">
+      <field type="CharField" name="path">/nomcom/defaults/topic/description</field>
+      <field type="CharField" name="title">Description of Topic</field>
+      <field type="TextField" name="variables">$topic: Topic'</field>
+      <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">rst</field>
+      <field type="TextField" name="content">This is a description of the topic "$topic"
+
+Describe the topic and add any information/instructions for the responder here.
+</field>
+    </object>
+    <object pk="13" model="dbtemplate.dbtemplate">
+        <field type="CharField" name="path">/nomcom/defaults/iesg_requirements</field>
+        <field type="CharField" name="title">Generic IESG Requirements</field>
+        <field to="name.dbtemplatetypename" name="type" rel="ManyToOneRel">rst</field>
+        <field type="TextField" name="content">Generic IESG Requirements Yo!</field>
+    </object>
+</django-objects>
diff --git a/ietf/secr/static/js/dynamic_inlines.js b/ietf/secr/static/js/dynamic_inlines.js
index a1abc2d0e5..d0753a3a63 100644
--- a/ietf/secr/static/js/dynamic_inlines.js
+++ b/ietf/secr/static/js/dynamic_inlines.js
@@ -1,71 +1,71 @@
-/* Following functions based off code written by Arne Brodowski
-http://www.arnebrodowski.de/blog/507-Add-and-remove-Django-Admin-Inlines-with-JavaScript.html
-
-2012-02-01 customized for new Rolodex.  Email formset doesn't have an id field, rather a "address"
-field as primary key.  Also for some reason the "active" boolean field doesn't get saved properly
-if the checkbox input has an empty "value" argument.
-*/
-import $ from 'jquery';
-
-function increment_form_ids(el, to, name) {
-    var from = to-1
-    $(':input', $(el)).each(function(i,e){
-        var old_name = $(e).attr('name')
-        var old_id = $(e).attr('id')
-        $(e).attr('name', old_name.replace(from, to))
-        $(e).attr('id', old_id.replace(from, to))
-        if ($(e).attr('type') != 'checkbox') {
-           $(e).val('')
-        }
-    })
-}
-
-function add_inline_form(name) {
-    if (name=="email") {
-        var first = $('#id_'+name+'-0-address').parents('.inline-related')
-    }
-    else {
-        var first = $('#id_'+name+'-0-id').parents('.inline-related')
-    }
-    // check to see if this is a stacked or tabular inline
-    if (first.hasClass("tabular")) {
-        var field_table = first.parent().find('table > tbody')
-        const children = field_table.children('tr.dynamic-inline')
-        var count = children.length
-        const last = $(children[count-1])
-        var copy = last.clone(true)
-        copy.removeClass("row1 row2")
-        copy.find("input[name$='address']").attr("readonly", false)
-        copy.addClass("row"+((count % 2) ? 2 : 1))
-        copy.insertAfter(last)
-        increment_form_ids($(copy), count, name)
-    }
-    else {
-        var last = $(first).parent().children('.last-related')
-        var copy = $(last).clone(true)
-        var count = $(first).parent().children('.inline-related').length
-        $(last).removeClass('last-related')
-        var header = $('h3', copy)
-        header.html(header.html().replace("#"+count, "#"+(count+1)))
-        $(last).after(copy)
-        increment_form_ids($(first).parents('.inline-group').children('.last-related'), count, name)
-    }
-    $('input#id_'+name+'-TOTAL_FORMS').val(count+1)
-    return false;
-}
-
-// Add all the "Add Another" links to the bottom of each inline group
-$(function() {
-    var html_template = '<ul class="tools">'+
-        '<li>'+
-            '<a id="addlink-{{prefix}}" class="addlink" href="#">'+
-            'Add another</a>'+
-        '</li>'+
-    '</ul>'
-    $('.inline-group').each(function(i) {
-        //prefix is in the name of the input fields before the "-"
-        var prefix = $("input[type='hidden'][name!='csrfmiddlewaretoken']", this).attr("name").split("-")[0];
-        $(this).append(html_template.replace("{{prefix}}", prefix));
-        $('#addlink-' + prefix).on('click', () => add_inline_form(prefix));
-    })
-})
+/* Following functions based off code written by Arne Brodowski
+http://www.arnebrodowski.de/blog/507-Add-and-remove-Django-Admin-Inlines-with-JavaScript.html
+
+2012-02-01 customized for new Rolodex.  Email formset doesn't have an id field, rather a "address"
+field as primary key.  Also for some reason the "active" boolean field doesn't get saved properly
+if the checkbox input has an empty "value" argument.
+*/
+import $ from 'jquery';
+
+function increment_form_ids(el, to, name) {
+    var from = to-1
+    $(':input', $(el)).each(function(i,e){
+        var old_name = $(e).attr('name')
+        var old_id = $(e).attr('id')
+        $(e).attr('name', old_name.replace(from, to))
+        $(e).attr('id', old_id.replace(from, to))
+        if ($(e).attr('type') != 'checkbox') {
+           $(e).val('')
+        }
+    })
+}
+
+function add_inline_form(name) {
+    if (name=="email") {
+        var first = $('#id_'+name+'-0-address').parents('.inline-related')
+    }
+    else {
+        var first = $('#id_'+name+'-0-id').parents('.inline-related')
+    }
+    // check to see if this is a stacked or tabular inline
+    if (first.hasClass("tabular")) {
+        var field_table = first.parent().find('table > tbody')
+        const children = field_table.children('tr.dynamic-inline')
+        var count = children.length
+        const last = $(children[count-1])
+        var copy = last.clone(true)
+        copy.removeClass("row1 row2")
+        copy.find("input[name$='address']").attr("readonly", false)
+        copy.addClass("row"+((count % 2) ? 2 : 1))
+        copy.insertAfter(last)
+        increment_form_ids($(copy), count, name)
+    }
+    else {
+        var last = $(first).parent().children('.last-related')
+        var copy = $(last).clone(true)
+        var count = $(first).parent().children('.inline-related').length
+        $(last).removeClass('last-related')
+        var header = $('h3', copy)
+        header.html(header.html().replace("#"+count, "#"+(count+1)))
+        $(last).after(copy)
+        increment_form_ids($(first).parents('.inline-group').children('.last-related'), count, name)
+    }
+    $('input#id_'+name+'-TOTAL_FORMS').val(count+1)
+    return false;
+}
+
+// Add all the "Add Another" links to the bottom of each inline group
+$(function() {
+    var html_template = '<ul class="tools">'+
+        '<li>'+
+            '<a id="addlink-{{prefix}}" class="addlink" href="#">'+
+            'Add another</a>'+
+        '</li>'+
+    '</ul>'
+    $('.inline-group').each(function(i) {
+        //prefix is in the name of the input fields before the "-"
+        var prefix = $("input[type='hidden'][name!='csrfmiddlewaretoken']", this).attr("name").split("-")[0];
+        $(this).append(html_template.replace("{{prefix}}", prefix));
+        $('#addlink-' + prefix).on('click', () => add_inline_form(prefix));
+    })
+})
diff --git a/k8s/README.md b/k8s/README.md
index 73b5978675..3966101ab8 100644
--- a/k8s/README.md
+++ b/k8s/README.md
@@ -1,5 +1,5 @@
-# Kustomize deployment
-
-## Run locally
-
+# Kustomize deployment
+
+## Run locally
+
 The `secrets.yaml` file is provided as a reference only and must be referenced manually in the `kustomization.yaml` file.
\ No newline at end of file
diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml
index 4b79f00753..2b623da2bd 100644
--- a/k8s/kustomization.yaml
+++ b/k8s/kustomization.yaml
@@ -1,16 +1,16 @@
-namespace: datatracker
-namePrefix: dt-
-configMapGenerator:
-  - name: files-cfgmap
-    files:
-      - nginx-logging.conf
-      - nginx-auth.conf
-      - nginx-datatracker.conf
-      - settings_local.py
-resources:
-  - auth.yaml
-  - beat.yaml
-  - celery.yaml
-  - datatracker.yaml
-  - memcached.yaml
-  - rabbitmq.yaml
+namespace: datatracker
+namePrefix: dt-
+configMapGenerator:
+  - name: files-cfgmap
+    files:
+      - nginx-logging.conf
+      - nginx-auth.conf
+      - nginx-datatracker.conf
+      - settings_local.py
+resources:
+  - auth.yaml
+  - beat.yaml
+  - celery.yaml
+  - datatracker.yaml
+  - memcached.yaml
+  - rabbitmq.yaml
diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml
index 4b362c88c1..5a4c9f0aed 100644
--- a/k8s/memcached.yaml
+++ b/k8s/memcached.yaml
@@ -1,80 +1,80 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: memcached
-spec:
-  replicas: 1
-  revisionHistoryLimit: 2
-  selector:
-    matchLabels:
-      app: memcached
-  template:
-    metadata:
-      labels:
-        app: memcached
-    spec:
-      securityContext:
-        runAsNonRoot: true
-      containers:
-        # -----------------------------------------------------
-        # Memcached
-        # -----------------------------------------------------
-        - image: "memcached:1.6-alpine"
-          imagePullPolicy: IfNotPresent
-          args: ["-m", "1024"]
-          name: memcached
-          ports:
-            - name: memcached
-              containerPort: 11211
-              protocol: TCP
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - ALL
-            readOnlyRootFilesystem: true
-            # memcached image sets up uid/gid 11211
-            runAsUser: 11211
-            runAsGroup: 11211
-        # -----------------------------------------------------
-        # Memcached Exporter for Prometheus
-        # -----------------------------------------------------
-        - image: "quay.io/prometheus/memcached-exporter:v0.14.3"
-          imagePullPolicy: IfNotPresent
-          name: memcached-exporter
-          ports:
-            - name: metrics
-              containerPort: 9150
-              protocol: TCP
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop:
-                - ALL
-            readOnlyRootFilesystem: true
-            runAsUser: 65534  # nobody
-            runAsGroup: 65534  # nobody
-      dnsPolicy: ClusterFirst
-      restartPolicy: Always
-      terminationGracePeriodSeconds: 30
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: memcached
-  annotations:
-    k8s.grafana.com/scrape: "true"  # this is not a bool
-    k8s.grafana.com/metrics.portName: "metrics"
-spec:
-  type: ClusterIP
-  ports:
-    - port: 11211
-      targetPort: memcached
-      protocol: TCP
-      name: memcached
-    - port: 9150
-      targetPort: metrics
-      protocol: TCP
-      name: metrics
-  selector:
-    app: memcached
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: memcached
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: memcached
+  template:
+    metadata:
+      labels:
+        app: memcached
+    spec:
+      securityContext:
+        runAsNonRoot: true
+      containers:
+        # -----------------------------------------------------
+        # Memcached
+        # -----------------------------------------------------
+        - image: "memcached:1.6-alpine"
+          imagePullPolicy: IfNotPresent
+          args: ["-m", "1024"]
+          name: memcached
+          ports:
+            - name: memcached
+              containerPort: 11211
+              protocol: TCP
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: true
+            # memcached image sets up uid/gid 11211
+            runAsUser: 11211
+            runAsGroup: 11211
+        # -----------------------------------------------------
+        # Memcached Exporter for Prometheus
+        # -----------------------------------------------------
+        - image: "quay.io/prometheus/memcached-exporter:v0.14.3"
+          imagePullPolicy: IfNotPresent
+          name: memcached-exporter
+          ports:
+            - name: metrics
+              containerPort: 9150
+              protocol: TCP
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: true
+            runAsUser: 65534  # nobody
+            runAsGroup: 65534  # nobody
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 30
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: memcached
+  annotations:
+    k8s.grafana.com/scrape: "true"  # this is not a bool
+    k8s.grafana.com/metrics.portName: "metrics"
+spec:
+  type: ClusterIP
+  ports:
+    - port: 11211
+      targetPort: memcached
+      protocol: TCP
+      name: memcached
+    - port: 9150
+      targetPort: metrics
+      protocol: TCP
+      name: metrics
+  selector:
+    app: memcached
diff --git a/k8s/secrets.yaml b/k8s/secrets.yaml
index 4e76a86a53..ba90af9c2a 100644
--- a/k8s/secrets.yaml
+++ b/k8s/secrets.yaml
@@ -1,83 +1,83 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: secrets-env
-type: Opaque
-stringData:
-  DATATRACKER_SERVER_MODE: "development"  # development for staging, production for production
-  DATATRACKER_ADMINS: |-
-    Robert Sparks <rjsparks@nostrum.com>
-    Ryan Cross <rcross@amsl.com>
-    Kesara Rathnayake <kesara@staff.ietf.org>
-    Jennifer Richards <jennifer@staff.ietf.org>
-    Nicolas Giard <nick@staff.ietf.org>
-  DATATRACKER_ALLOWED_HOSTS: ".ietf.org"  # newline-separated list also allowed
-  # DATATRACKER_DATATRACKER_DEBUG: "false"
-  
-  # DB access details - needs to be filled in
-  # DATATRACKER_DB_HOST: "db"
-  # DATATRACKER_DB_PORT: "5432"
-  # DATATRACKER_DB_NAME: "datatracker"
-  # DATATRACKER_DB_USER: "django"  # secret
-  # DATATRACKER_DB_PASS: "RkTkDPFnKpko"  # secret
-  # DATATRACKER_DB_CONN_MAX_AGE: "0"  # connection per request if not set, no limit if set to "None"
-  # DATATRACKER_DB_CONN_HEALTH_CHECKS: "false"
-  
-  DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk"  # secret
-
-  # Set this to point testing / staging at the production statics server until we
-  # sort that out
-  # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/"
-  
-  # DATATRACKER_EMAIL_DEBUG: "true"
-  
-  # Outgoing email details
-  # DATATRACKER_EMAIL_HOST: "localhost"  # defaults to localhost
-  # DATATRACKER_EMAIL_PORT: "2025"  # defaults to 2025
-
-  # The value here is the default from settings.py (i.e., not actually secret)
-  DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ="  # secret
-
-  DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password"  # secret
-  DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password"  # secret
-  DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key"  # secret
-  DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key"  # secret
-
-  # API key configuration
-  DATATRACKER_API_KEY_TYPE: "ES265"
-  # secret - value here is the default from settings.py (i.e., not actually secret)
-  DATATRACKER_API_PUBLIC_KEY_PEM_B64: |-
-    Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS
-    29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel
-    ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM
-    weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo=
-  # secret - value here is the default from settings.py (i.e., not actually secret)
-  DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- 
-    Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O
-    UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1
-    F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY
-    yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4
-    N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo=
-
-  #DATATRACKER_REGISTRATION_API_KEY: "some-key"  # secret"
-
-  # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/"
-  DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id"  # secret
-  DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret"  # secret
-
-  # DATATRACKER_MATOMO_SITE_ID: "7"  # must be present to enable Matomo
-  # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org"
-
-  CELERY_PASSWORD: "this-is-a-secret"  # secret
-
-  # Only one of these may be set
-  # DATATRACKER_APP_API_TOKENS_JSON_B64: "e30K"  # secret
-  # DATATRACKER_APP_API_TOKENS_JSON: "{}"  # secret 
-
-  # use this to override default - one entry per line
-  # DATATRACKER_CSRF_TRUSTED_ORIGINS: |-
-  #   https://datatracker.staging.ietf.org
-
-  # Scout configuration
-  DATATRACKER_SCOUT_KEY: "this-is-the-scout-key"
+apiVersion: v1
+kind: Secret
+metadata:
+  name: secrets-env
+type: Opaque
+stringData:
+  DATATRACKER_SERVER_MODE: "development"  # development for staging, production for production
+  DATATRACKER_ADMINS: |-
+    Robert Sparks <rjsparks@nostrum.com>
+    Ryan Cross <rcross@amsl.com>
+    Kesara Rathnayake <kesara@staff.ietf.org>
+    Jennifer Richards <jennifer@staff.ietf.org>
+    Nicolas Giard <nick@staff.ietf.org>
+  DATATRACKER_ALLOWED_HOSTS: ".ietf.org"  # newline-separated list also allowed
+  # DATATRACKER_DATATRACKER_DEBUG: "false"
+  
+  # DB access details - needs to be filled in
+  # DATATRACKER_DB_HOST: "db"
+  # DATATRACKER_DB_PORT: "5432"
+  # DATATRACKER_DB_NAME: "datatracker"
+  # DATATRACKER_DB_USER: "django"  # secret
+  # DATATRACKER_DB_PASS: "RkTkDPFnKpko"  # secret
+  # DATATRACKER_DB_CONN_MAX_AGE: "0"  # connection per request if not set, no limit if set to "None"
+  # DATATRACKER_DB_CONN_HEALTH_CHECKS: "false"
+  
+  DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk"  # secret
+
+  # Set this to point testing / staging at the production statics server until we
+  # sort that out
+  # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/"
+  
+  # DATATRACKER_EMAIL_DEBUG: "true"
+  
+  # Outgoing email details
+  # DATATRACKER_EMAIL_HOST: "localhost"  # defaults to localhost
+  # DATATRACKER_EMAIL_PORT: "2025"  # defaults to 2025
+
+  # The value here is the default from settings.py (i.e., not actually secret)
+  DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ="  # secret
+
+  DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password"  # secret
+  DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password"  # secret
+  DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key"  # secret
+  DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key"  # secret
+
+  # API key configuration
+  DATATRACKER_API_KEY_TYPE: "ES265"
+  # secret - value here is the default from settings.py (i.e., not actually secret)
+  DATATRACKER_API_PUBLIC_KEY_PEM_B64: |-
+    Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS
+    29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel
+    ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM
+    weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo=
+  # secret - value here is the default from settings.py (i.e., not actually secret)
+  DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- 
+    Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O
+    UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1
+    F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY
+    yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4
+    N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo=
+
+  #DATATRACKER_REGISTRATION_API_KEY: "some-key"  # secret"
+
+  # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/"
+  DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id"  # secret
+  DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret"  # secret
+
+  # DATATRACKER_MATOMO_SITE_ID: "7"  # must be present to enable Matomo
+  # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org"
+
+  CELERY_PASSWORD: "this-is-a-secret"  # secret
+
+  # Only one of these may be set
+  # DATATRACKER_APP_API_TOKENS_JSON_B64: "e30K"  # secret
+  # DATATRACKER_APP_API_TOKENS_JSON: "{}"  # secret 
+
+  # use this to override default - one entry per line
+  # DATATRACKER_CSRF_TRUSTED_ORIGINS: |-
+  #   https://datatracker.staging.ietf.org
+
+  # Scout configuration
+  DATATRACKER_SCOUT_KEY: "this-is-the-scout-key"
   DATATRACKER_SCOUT_NAME: "StagingDatatracker"
\ No newline at end of file

From d09d17c338656c7a880c203af4f721830ffbb77f Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 21 Nov 2024 12:06:09 -0400
Subject: [PATCH 182/656] chore(deps): pin django-tastypie version (#8252)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 2e6e2714d7..5c80d5301e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -21,7 +21,7 @@ django-oidc-provider>=0.8.1    # 0.8 dropped Django 2 support
 django-referrer-policy>=1.0
 django-simple-history>=3.0.0
 django-stubs>=4.2.7,<5    # The django-stubs version used determines the the mypy version indicated below
-django-tastypie>=0.14.5    # Version must be locked in sync with version of Django
+django-tastypie>=0.14.7,<0.15.0    # Version must be locked in sync with version of Django
 django-vite>=2.0.2,<3
 django-widget-tweaks>=1.4.12
 djlint>=1.0.0    # To auto-indent templates via "djlint --profile django --reformat"

From a9e50573753fcc7fcd86c5baba86588a1de86155 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 21 Nov 2024 18:57:58 -0400
Subject: [PATCH 183/656] chore: crlf -> lf (#8255)

---
 .github/workflows/build-base-app.yml | 132 +++++-----
 .github/workflows/tests.yml          | 374 +++++++++++++--------------
 2 files changed, 253 insertions(+), 253 deletions(-)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index 609b641ef9..c8f66a22b7 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -1,66 +1,66 @@
-name: Build Base App Docker Image
-
-on:
-  push:
-    branches:
-      - 'main'
-    paths:
-      - 'docker/base.Dockerfile'
-      - 'requirements.txt'
-
-  workflow_dispatch:
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      packages: write
-
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        token: ${{ secrets.GH_COMMON_TOKEN }}
-
-    - name: Set Version
-      run: |
-        printf -v CURDATE '%(%Y%m%dT%H%M)T' -1
-        echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-    
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
-
-    - name: Login to GitHub Container Registry
-      uses: docker/login-action@v3
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Docker Build & Push
-      uses: docker/build-push-action@v6
-      env:
-        DOCKER_BUILD_NO_SUMMARY: true
-      with:
-        context: .
-        file: docker/base.Dockerfile
-        platforms: linux/amd64,linux/arm64
-        push: true
-        tags: |
-          ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }}
-          ghcr.io/ietf-tools/datatracker-app-base:latest
-
-    - name: Update version references
-      run: |
-        sed -i "1s/.*/FROM ghcr.io\/ietf-tools\/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile
-        echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE
-
-    - name: Commit CHANGELOG.md
-      uses: stefanzweifel/git-auto-commit-action@v5
-      with:
-        branch: main
-        commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}'
-        file_pattern: dev/build/Dockerfile dev/build/TARGET_BASE
+name: Build Base App Docker Image
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'docker/base.Dockerfile'
+      - 'requirements.txt'
+
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      packages: write
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GH_COMMON_TOKEN }}
+
+    - name: Set Version
+      run: |
+        printf -v CURDATE '%(%Y%m%dT%H%M)T' -1
+        echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+    
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Docker Build & Push
+      uses: docker/build-push-action@v6
+      env:
+        DOCKER_BUILD_NO_SUMMARY: true
+      with:
+        context: .
+        file: docker/base.Dockerfile
+        platforms: linux/amd64,linux/arm64
+        push: true
+        tags: |
+          ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }}
+          ghcr.io/ietf-tools/datatracker-app-base:latest
+
+    - name: Update version references
+      run: |
+        sed -i "1s/.*/FROM ghcr.io\/ietf-tools\/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile
+        echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE
+
+    - name: Commit CHANGELOG.md
+      uses: stefanzweifel/git-auto-commit-action@v5
+      with:
+        branch: main
+        commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}'
+        file_pattern: dev/build/Dockerfile dev/build/TARGET_BASE
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 8eb6adc953..5457415f59 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -1,187 +1,187 @@
-name: Reusable Tests Workflow
-
-on:
-  workflow_call:
-    inputs:
-      ignoreLowerCoverage:
-        description: 'Ignore Lower Coverage'
-        default: false
-        required: true
-        type: boolean
-      skipSelenium:
-        description: 'Skip Selenium Tests'
-        default: false
-        required: false
-        type: boolean
-      targetBaseVersion:
-        description: 'Target Base Image Version'
-        default: latest
-        required: false
-        type: string
-
-jobs:
-  tests-python:
-    name: Python Tests
-    runs-on: ubuntu-latest
-    container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }}
-    
-    services:
-      db:
-        image: ghcr.io/ietf-tools/datatracker-db:latest
-    
-    steps:
-    - uses: actions/checkout@v4
-    
-    - name: Prepare for tests
-      run: |
-        chmod +x ./dev/tests/prepare.sh
-        sh ./dev/tests/prepare.sh
-        
-    - name: Ensure DB is ready
-      run: |
-        /usr/local/bin/wait-for db:5432 -- echo "DB ready"
-        
-    - name: Run all tests
-      shell: bash
-      run: |
-        echo "Running checks..."
-        ./ietf/manage.py check
-        ./ietf/manage.py migrate --fake-initial
-        echo "Validating migrations..."
-        if ! ( ietf/manage.py makemigrations --dry-run --check --verbosity 3 ) ; then
-          echo "Model changes without migrations found."
-          exit 1
-        fi
-        if [[ "x${{ inputs.skipSelenium }}" == "xtrue" ]]; then
-          echo "Disable selenium tests..."
-          rm /usr/bin/geckodriver
-        fi
-        echo "Running tests..."
-        if [[ "x${{ inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then
-          echo "Lower coverage failures will be ignored."
-          HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage
-        else
-          HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test
-        fi
-        coverage xml
-
-    - name: Upload geckodriver.log
-      uses: actions/upload-artifact@v4
-      if: ${{ failure() }}
-      with:
-        name: geckodriverlog
-        path: geckodriver.log
-
-    - name: Upload Coverage Results to Codecov
-      uses: codecov/codecov-action@v5
-      with:
-        disable_search: true
-        files: coverage.xml
-        token: ${{ secrets.CODECOV_TOKEN }}
-        
-    - name: Convert Coverage Results
-      if: ${{ always() }}
-      run: |
-        mv latest-coverage.json coverage.json
-        
-    - name: Upload Coverage Results as Build Artifact
-      uses: actions/upload-artifact@v4
-      if: ${{ always() }}
-      with:
-        name: coverage
-        path: coverage.json
-
-  tests-playwright:
-    name: Playwright Tests
-    runs-on: macos-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        project: [chromium, firefox]
-    
-    steps:
-    - uses: actions/checkout@v4
-
-    - uses: actions/setup-node@v4
-      with:
-        node-version: '18'
-
-    - name: Run all tests
-      run: |
-        echo "Installing dependencies..."
-        yarn
-        echo "Installing Playwright..."
-        cd playwright
-        mkdir test-results
-        npm ci
-        npx playwright install --with-deps ${{ matrix.project }}
-        echo "Running tests..."
-        npx playwright test --project=${{ matrix.project }}
-        
-    - name: Upload Report
-      uses: actions/upload-artifact@v4
-      if: ${{ always() }}
-      continue-on-error: true
-      with:
-        name: playwright-results-${{ matrix.project }}
-        path: playwright/test-results/
-        if-no-files-found: ignore
-
-  tests-playwright-legacy:
-    name: Playwright Legacy Tests
-    runs-on: ubuntu-latest
-    container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }}
-    strategy:
-      fail-fast: false
-      matrix:
-        project: [chromium, firefox]
-    
-    services:
-      db:
-        image: ghcr.io/ietf-tools/datatracker-db:latest
-    
-    steps:
-    - uses: actions/checkout@v4
-    
-    - name: Prepare for tests
-      run: |
-        chmod +x ./dev/tests/prepare.sh
-        sh ./dev/tests/prepare.sh
-        
-    - name: Ensure DB is ready
-      run: |
-        /usr/local/bin/wait-for db:5432 -- echo "DB ready"
-        
-    - name: Start Datatracker
-      run: |
-        echo "Running checks..."
-        ./ietf/manage.py check
-        ./ietf/manage.py migrate --fake-initial
-        echo "Starting datatracker..."
-        ./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local &
-        echo "Waiting for datatracker to be ready..."
-        /usr/local/bin/wait-for localhost:8000 -- echo "Datatracker ready"
-
-    - name: Run all tests
-      env:
-        # Required to get firefox to run as root:
-        HOME: ""
-      run: |
-        echo "Installing dependencies..."
-        yarn
-        echo "Installing Playwright..."
-        cd playwright
-        mkdir test-results
-        npm ci
-        npx playwright install --with-deps ${{ matrix.project }}
-        echo "Running tests..."
-        npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js
-        
-    - name: Upload Report
-      uses: actions/upload-artifact@v4
-      if: ${{ always() }}
-      continue-on-error: true
-      with:
-        name: playwright-legacy-results-${{ matrix.project }}
-        path: playwright/test-results/
-        if-no-files-found: ignore
+name: Reusable Tests Workflow
+
+on:
+  workflow_call:
+    inputs:
+      ignoreLowerCoverage:
+        description: 'Ignore Lower Coverage'
+        default: false
+        required: true
+        type: boolean
+      skipSelenium:
+        description: 'Skip Selenium Tests'
+        default: false
+        required: false
+        type: boolean
+      targetBaseVersion:
+        description: 'Target Base Image Version'
+        default: latest
+        required: false
+        type: string
+
+jobs:
+  tests-python:
+    name: Python Tests
+    runs-on: ubuntu-latest
+    container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }}
+    
+    services:
+      db:
+        image: ghcr.io/ietf-tools/datatracker-db:latest
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Prepare for tests
+      run: |
+        chmod +x ./dev/tests/prepare.sh
+        sh ./dev/tests/prepare.sh
+        
+    - name: Ensure DB is ready
+      run: |
+        /usr/local/bin/wait-for db:5432 -- echo "DB ready"
+        
+    - name: Run all tests
+      shell: bash
+      run: |
+        echo "Running checks..."
+        ./ietf/manage.py check
+        ./ietf/manage.py migrate --fake-initial
+        echo "Validating migrations..."
+        if ! ( ietf/manage.py makemigrations --dry-run --check --verbosity 3 ) ; then
+          echo "Model changes without migrations found."
+          exit 1
+        fi
+        if [[ "x${{ inputs.skipSelenium }}" == "xtrue" ]]; then
+          echo "Disable selenium tests..."
+          rm /usr/bin/geckodriver
+        fi
+        echo "Running tests..."
+        if [[ "x${{ inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then
+          echo "Lower coverage failures will be ignored."
+          HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage
+        else
+          HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test
+        fi
+        coverage xml
+
+    - name: Upload geckodriver.log
+      uses: actions/upload-artifact@v4
+      if: ${{ failure() }}
+      with:
+        name: geckodriverlog
+        path: geckodriver.log
+
+    - name: Upload Coverage Results to Codecov
+      uses: codecov/codecov-action@v5
+      with:
+        disable_search: true
+        files: coverage.xml
+        token: ${{ secrets.CODECOV_TOKEN }}
+        
+    - name: Convert Coverage Results
+      if: ${{ always() }}
+      run: |
+        mv latest-coverage.json coverage.json
+        
+    - name: Upload Coverage Results as Build Artifact
+      uses: actions/upload-artifact@v4
+      if: ${{ always() }}
+      with:
+        name: coverage
+        path: coverage.json
+
+  tests-playwright:
+    name: Playwright Tests
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        project: [chromium, firefox]
+    
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '18'
+
+    - name: Run all tests
+      run: |
+        echo "Installing dependencies..."
+        yarn
+        echo "Installing Playwright..."
+        cd playwright
+        mkdir test-results
+        npm ci
+        npx playwright install --with-deps ${{ matrix.project }}
+        echo "Running tests..."
+        npx playwright test --project=${{ matrix.project }}
+        
+    - name: Upload Report
+      uses: actions/upload-artifact@v4
+      if: ${{ always() }}
+      continue-on-error: true
+      with:
+        name: playwright-results-${{ matrix.project }}
+        path: playwright/test-results/
+        if-no-files-found: ignore
+
+  tests-playwright-legacy:
+    name: Playwright Legacy Tests
+    runs-on: ubuntu-latest
+    container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }}
+    strategy:
+      fail-fast: false
+      matrix:
+        project: [chromium, firefox]
+    
+    services:
+      db:
+        image: ghcr.io/ietf-tools/datatracker-db:latest
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Prepare for tests
+      run: |
+        chmod +x ./dev/tests/prepare.sh
+        sh ./dev/tests/prepare.sh
+        
+    - name: Ensure DB is ready
+      run: |
+        /usr/local/bin/wait-for db:5432 -- echo "DB ready"
+        
+    - name: Start Datatracker
+      run: |
+        echo "Running checks..."
+        ./ietf/manage.py check
+        ./ietf/manage.py migrate --fake-initial
+        echo "Starting datatracker..."
+        ./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local &
+        echo "Waiting for datatracker to be ready..."
+        /usr/local/bin/wait-for localhost:8000 -- echo "Datatracker ready"
+
+    - name: Run all tests
+      env:
+        # Required to get firefox to run as root:
+        HOME: ""
+      run: |
+        echo "Installing dependencies..."
+        yarn
+        echo "Installing Playwright..."
+        cd playwright
+        mkdir test-results
+        npm ci
+        npx playwright install --with-deps ${{ matrix.project }}
+        echo "Running tests..."
+        npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js
+        
+    - name: Upload Report
+      uses: actions/upload-artifact@v4
+      if: ${{ always() }}
+      continue-on-error: true
+      with:
+        name: playwright-legacy-results-${{ matrix.project }}
+        path: playwright/test-results/
+        if-no-files-found: ignore

From 3074be8ccf634af7690615a684dad3cc4501759c Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 21 Nov 2024 18:06:37 -0400
Subject: [PATCH 184/656] ci: tag feature branch release images

---
 .github/workflows/build.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f1044223dc..46cb150e05 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -256,6 +256,10 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
+    - name: Add feature-latest tag
+      if: ${{ startsWith(github.ref_name, 'feat/') }}
+      run: echo "FEATURE_LATEST_TAG=$(echo $GITHUB_REF_NAME | tr / -)" >> $GITHUB_ENV
+
     - name: Build Images
       uses: docker/build-push-action@v6
       env:
@@ -265,7 +269,9 @@ jobs:
         file: dev/build/Dockerfile
         platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }}
         push: true
-        tags: ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }}
+        tags: |
+          ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }}
+          ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }}
         cache-from: type=gha
         cache-to: type=gha,mode=max
 

From 394e32e022aa99e07c93823f7ad53679815a6124 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 22 Nov 2024 10:13:53 -0400
Subject: [PATCH 185/656] fix: lock session requests for new meetings (#8251)

* fix: lock session requests for new meetings

* fix: lint
---
 ietf/secr/meetings/tests.py | 4 ++++
 ietf/secr/meetings/views.py | 7 +++----
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/ietf/secr/meetings/tests.py b/ietf/secr/meetings/tests.py
index a241d2b5eb..08c792ce1e 100644
--- a/ietf/secr/meetings/tests.py
+++ b/ietf/secr/meetings/tests.py
@@ -82,6 +82,10 @@ def test_add_meeting(self):
             [cn.slug for cn in new_meeting.group_conflict_types.all()],
             post_data['group_conflict_types'],
         )
+        self.assertEqual(
+            new_meeting.session_request_lock_message, 
+            "Session requests for this meeting have not yet opened.",
+        )
 
     def test_add_meeting_default_conflict_types(self):
         """Add meeting should default to same conflict types as previous meeting"""
diff --git a/ietf/secr/meetings/views.py b/ietf/secr/meetings/views.py
index 8afcf5a11e..47f7b7ffa5 100644
--- a/ietf/secr/meetings/views.py
+++ b/ietf/secr/meetings/views.py
@@ -17,7 +17,7 @@
 from ietf.ietfauth.utils import role_required
 from ietf.utils.mail import send_mail
 from ietf.meeting.forms import duration_string
-from ietf.meeting.helpers import get_meeting, make_materials_directories, populate_important_dates
+from ietf.meeting.helpers import make_materials_directories, populate_important_dates
 from ietf.meeting.models import Meeting, Session, Room, TimeSlot, SchedTimeSessAssignment, Schedule, SchedulingEvent
 from ietf.meeting.utils import add_event_info_to_session_qs
 from ietf.name.models import SessionStatusName
@@ -223,9 +223,8 @@ def add(request):
             )
             meeting.schedule = schedule
 
-            # we want to carry session request lock status over from previous meeting
-            previous_meeting = get_meeting( int(meeting.number) - 1 )
-            meeting.session_request_lock_message = previous_meeting.session_request_lock_message
+            # Create meeting with session requests locked
+            meeting.session_request_lock_message = "Session requests for this meeting have not yet opened."
             meeting.save()
 
             populate_important_dates(meeting)

From 622ded5d2b4db9d4bf948af0e1d0890edaa8bb33 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 22 Nov 2024 12:38:36 -0400
Subject: [PATCH 186/656] fix: ensure csrf cookie for searches (#8260)

Needed on views that include search_form.html in their responses.
---
 ietf/doc/views_search.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py
index 0f1937efb3..f4ad247ff0 100644
--- a/ietf/doc/views_search.py
+++ b/ietf/doc/views_search.py
@@ -56,6 +56,7 @@
 from django.utils.html import strip_tags
 from django.utils.cache import _generate_cache_key # type: ignore
 from django.utils.text import slugify
+from django.views.decorators.csrf import ensure_csrf_cookie
 from django_stubs_ext import QuerySetAny
 
 import debug                            # pyflakes:ignore
@@ -283,6 +284,7 @@ def retrieve_search_results(form, all_types=False):
     return docs
 
 
+@ensure_csrf_cookie
 def search(request):
     """Search for a draft"""
     # defaults for results / meta
@@ -335,6 +337,7 @@ def search(request):
     )
 
 
+@ensure_csrf_cookie
 def frontpage(request):
     form = SearchForm()
     return render(request, 'doc/frontpage.html', {'form':form})

From 9681d9f9786772e3b83a27661444b0c4af0b54f0 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 22 Nov 2024 15:30:54 -0400
Subject: [PATCH 187/656] fix: revert POST for document search requests (#8263)

* Revert "fix: ensure csrf cookie for searches (#8260)"

This reverts commit 622ded5d2b4db9d4bf948af0e1d0890edaa8bb33.

* Revert "refactor: eliminate single-use helper (#8226)"

This reverts commit 6608c9d530b62d10c88d637f949a00fb5fea4526.

* Revert "feat: POST for document search requests (#8206)"

This reverts commit b65a37b6e8f6cfe105cd89c61c7046fff9d21523.

* test: add back test fix

* refactor: eliminate single-use helper
---
 ietf/api/serializer.py                     |   1 -
 ietf/doc/tests.py                          | 155 ++++++---------------
 ietf/doc/views_search.py                   | 120 ++++++----------
 ietf/liaisons/forms.py                     |   1 -
 ietf/liaisons/widgets.py                   |   1 -
 ietf/templates/doc/search/search_form.html |   2 -
 6 files changed, 84 insertions(+), 196 deletions(-)

diff --git a/ietf/api/serializer.py b/ietf/api/serializer.py
index ca34ea649e..27f194c5b5 100644
--- a/ietf/api/serializer.py
+++ b/ietf/api/serializer.py
@@ -146,7 +146,6 @@ def end_object(self, obj):
                                 field_value = None
                         else:
                             field_value = field
-                        # Need QuerySetAny instead of QuerySet until django-stubs 5.0.1
                         if isinstance(field_value, QuerySetAny) or isinstance(field_value, list):
                             self._current[name] = dict([ (rel.pk, self.expand_related(rel, name)) for rel in field_value ])
                         else:
diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index abac10a5e9..f0c8e30626 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -73,163 +73,96 @@
 
 
 class SearchTests(TestCase):
-    def test_search_handles_querystring_parameters(self):
-        """Search parameters via querystring should not actually search"""
-        url = urlreverse("ietf.doc.views_search.search")
-        r = self.client.get(url + "?name=some-document-name&oldDrafts=on")
-        # Check that we got a valid response and that the warning about query string parameters is shown.
-        self.assertContains(
-            r,
-            "Searching via the URL query string is no longer supported.",
-            status_code=200,
-        )
-        # Check that the form was filled in correctly (not an exhaustive check, but different from the
-        # form defaults)
-        pq = PyQuery(r.content)
-        self.assertEqual(
-            pq("form#search_form input#id_name").attr("value"),
-            "some-document-name",
-            "The name field should be set in the SearchForm",
-        )
-        self.assertEqual(
-            pq("form#search_form input#id_olddrafts").attr("checked"),
-            "checked",
-            "The old drafts checkbox should be selected in the SearchForm",
-        )
-        self.assertIsNone(
-            pq("form#search_form input#id_rfcs").attr("checked"),
-            "The RFCs checkbox should not be selected in the SearchForm",
-        )
-        self.assertIsNone(
-            pq("form#search_form input#id_activedrafts").attr("checked"),
-            "The active drafts checkbox should not be selected in the SearchForm",
-        )
-
     def test_search(self):
-        draft = WgDraftFactory(
-            name="draft-ietf-mars-test",
-            group=GroupFactory(acronym="mars", parent=Group.objects.get(acronym="farfut")),
-            authors=[PersonFactory()],
-            ad=PersonFactory(),
-        )
+
+        draft = WgDraftFactory(name='draft-ietf-mars-test',group=GroupFactory(acronym='mars',parent=Group.objects.get(acronym='farfut')),authors=[PersonFactory()],ad=PersonFactory())
         rfc = WgRfcFactory()
         draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="pub-req"))
-        old_draft = IndividualDraftFactory(
-            name="draft-foo-mars-test",
-            authors=[PersonFactory()],
-            title="Optimizing Martian Network Topologies",
-        )
+        old_draft = IndividualDraftFactory(name='draft-foo-mars-test',authors=[PersonFactory()],title="Optimizing Martian Network Topologies")
         old_draft.set_state(State.objects.get(used=True, type="draft", slug="expired"))
-    
-        url = urlreverse("ietf.doc.views_search.search")
-    
+
+        base_url = urlreverse('ietf.doc.views_search.search')
+
         # only show form, no search yet
-        r = self.client.get(url)
+        r = self.client.get(base_url)
         self.assertEqual(r.status_code, 200)
-    
+
         # no match
-        r = self.client.post(url, {"activedrafts": "on", "name": "thisisnotadocumentname"})
+        r = self.client.get(base_url + "?activedrafts=on&name=thisisnotadocumentname")
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "No documents match")
-    
-        r = self.client.post(url, {"rfcs": "on", "name": "xyzzy"})
+
+        r = self.client.get(base_url + "?rfcs=on&name=xyzzy")
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "No documents match")
-    
-        r = self.client.post(url, {"olddrafts": "on", "name": "bar"})
+
+        r = self.client.get(base_url + "?olddrafts=on&name=bar")
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "No documents match")
-    
-        r = self.client.post(url, {"olddrafts": "on", "name": "foo"})
+
+        r = self.client.get(base_url + "?olddrafts=on&name=foo")
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "draft-foo-mars-test")
-    
-        r = self.client.post(url, {"olddrafts": "on", "name": "FoO"})  # mixed case
+
+        r = self.client.get(base_url + "?olddrafts=on&name=FoO")  # mixed case
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "draft-foo-mars-test")
-    
+
         # find by RFC
-        r = self.client.post(url, {"rfcs": "on", "name": rfc.name})
+        r = self.client.get(base_url + "?rfcs=on&name=%s" % rfc.name)
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, rfc.title)
-    
+
         # find by active/inactive
-    
+
         draft.set_state(State.objects.get(type="draft", slug="active"))
-        r = self.client.post(url, {"activedrafts": "on", "name": draft.name})
+        r = self.client.get(base_url + "?activedrafts=on&name=%s" % draft.name)
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-    
+
         draft.set_state(State.objects.get(type="draft", slug="expired"))
-        r = self.client.post(url, {"olddrafts": "on", "name": draft.name})
+        r = self.client.get(base_url + "?olddrafts=on&name=%s" % draft.name)
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-    
+        
         draft.set_state(State.objects.get(type="draft", slug="active"))
-    
+
         # find by title
-        r = self.client.post(url, {"activedrafts": "on", "name": draft.title.split()[0]})
+        r = self.client.get(base_url + "?activedrafts=on&name=%s" % draft.title.split()[0])
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-    
+
         # find by author
-        r = self.client.post(
-            url,
-            {
-                "activedrafts": "on",
-                "by": "author",
-                "author": draft.documentauthor_set.first().person.name_parts()[1],
-            },
-        )
+        r = self.client.get(base_url + "?activedrafts=on&by=author&author=%s" % draft.documentauthor_set.first().person.name_parts()[1])
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-    
+
         # find by group
-        r = self.client.post(
-            url,
-            {"activedrafts": "on", "by": "group", "group": draft.group.acronym},
-        )
+        r = self.client.get(base_url + "?activedrafts=on&by=group&group=%s" % draft.group.acronym)
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-    
-        r = self.client.post(
-            url,
-            {"activedrafts": "on", "by": "group", "group": draft.group.acronym.swapcase()},
-        )
+
+        r = self.client.get(base_url + "?activedrafts=on&by=group&group=%s" % draft.group.acronym.swapcase())
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-    
+
         # find by area
-        r = self.client.post(
-            url,
-            {"activedrafts": "on", "by": "area", "area": draft.group.parent_id},
-        )
+        r = self.client.get(base_url + "?activedrafts=on&by=area&area=%s" % draft.group.parent_id)
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-    
+
         # find by area
-        r = self.client.post(
-            url,
-            {"activedrafts": "on", "by": "area", "area": draft.group.parent_id},
-        )
+        r = self.client.get(base_url + "?activedrafts=on&by=area&area=%s" % draft.group.parent_id)
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-    
+
         # find by AD
-        r = self.client.post(url, {"activedrafts": "on", "by": "ad", "ad": draft.ad_id})
+        r = self.client.get(base_url + "?activedrafts=on&by=ad&ad=%s" % draft.ad_id)
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
-    
+
         # find by IESG state
-        r = self.client.post(
-            url,
-            {
-                "activedrafts": "on",
-                "by": "state",
-                "state": draft.get_state("draft-iesg").pk,
-                "substate": "",
-            },
-        )
+        r = self.client.get(base_url + "?activedrafts=on&by=state&state=%s&substate=" % draft.get_state("draft-iesg").pk)
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.title)
 
@@ -238,15 +171,15 @@ def test_search_became_rfc(self):
         rfc = WgRfcFactory()
         draft.set_state(State.objects.get(type="draft", slug="rfc"))
         draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc)
-        url = urlreverse("ietf.doc.views_search.search")
+        base_url = urlreverse('ietf.doc.views_search.search')
 
         # find by RFC
-        r = self.client.post(url, {"rfcs": "on", "name": rfc.name})
+        r = self.client.get(base_url + f"?rfcs=on&name={rfc.name}")
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, rfc.title)
 
         # find by draft
-        r = self.client.post(url, {"activedrafts": "on", "rfcs": "on", "name": draft.name})
+        r = self.client.get(base_url + f"?activedrafts=on&rfcs=on&name={draft.name}")
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, rfc.title)
 
diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py
index f4ad247ff0..9e9b5e88dd 100644
--- a/ietf/doc/views_search.py
+++ b/ietf/doc/views_search.py
@@ -33,12 +33,11 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-
+import hashlib
+import json
 import re
 import datetime
 import copy
-import hashlib
-import json
 import operator
 
 from collections import defaultdict
@@ -46,18 +45,16 @@
 
 from django import forms
 from django.conf import settings
-from django.contrib import messages
 from django.core.cache import cache, caches
 from django.urls import reverse as urlreverse
 from django.db.models import Q
-from django.http import Http404, HttpResponseBadRequest, HttpResponse, HttpResponseRedirect
+from django.http import Http404, HttpResponseBadRequest, HttpResponse, HttpResponseRedirect, QueryDict
 from django.shortcuts import render
 from django.utils import timezone
 from django.utils.html import strip_tags
 from django.utils.cache import _generate_cache_key # type: ignore
 from django.utils.text import slugify
-from django.views.decorators.csrf import ensure_csrf_cookie
-from django_stubs_ext import QuerySetAny
+
 
 import debug                            # pyflakes:ignore
 
@@ -149,29 +146,6 @@ def clean(self):
             q['irtfstate'] = None
         return q
 
-    def cache_key_fragment(self):
-        """Hash a bound form to get a value for use in a cache key
-        
-        Raises a ValueError if the form is not valid.
-        """
-        def _serialize_value(val):
-            # Need QuerySetAny instead of QuerySet until django-stubs 5.0.1
-            if isinstance(val, QuerySetAny):
-                return [item.pk for item in val]
-            else:
-                return getattr(val, "pk", val)  # use pk if present, else value
-    
-        if not self.is_valid():
-            raise ValueError(f"SearchForm invalid: {self.errors}")
-        contents = {
-            field_name: _serialize_value(field_value)
-            for field_name, field_value in self.cleaned_data.items() 
-            if field_name != "sort" and field_value is not None
-        }
-        contents_json = json.dumps(contents, sort_keys=True)
-        return hashlib.sha512(contents_json.encode("utf-8")).hexdigest()
-
-
 def retrieve_search_results(form, all_types=False):
     """Takes a validated SearchForm and return the results."""
 
@@ -284,65 +258,51 @@ def retrieve_search_results(form, all_types=False):
     return docs
 
 
-@ensure_csrf_cookie
 def search(request):
-    """Search for a draft"""
-    # defaults for results / meta
-    results = []
-    meta = {"by": None, "searching": False}
-
-    if request.method == "POST":
-        form = SearchForm(data=request.POST)
-        if form.is_valid():
-            cache_key = f"doc:document:search:{form.cache_key_fragment()}"
-            cached_val = cache.get(cache_key)
-            if cached_val:
-                [results, meta] = cached_val
-            else:
-                results = retrieve_search_results(form)
-                results, meta = prepare_document_table(
-                    request, results, form.cleaned_data
-                )
-                cache.set(
-                    cache_key, [results, meta]
-                )  # for settings.CACHE_MIDDLEWARE_SECONDS
-                log(f"Search results computed for {form.cleaned_data}")
-            meta["searching"] = True
-    else:
-        if request.GET:
-            # backwards compatibility - fill in the form
-            get_params = request.GET.copy()
-            if "activeDrafts" in request.GET:
-                get_params["activedrafts"] = request.GET["activeDrafts"]
-            if "oldDrafts" in request.GET:
-                get_params["olddrafts"] = request.GET["oldDrafts"]
-            if "subState" in request.GET:
-                get_params["substate"] = request.GET["subState"]
-            form = SearchForm(data=get_params)
-            messages.error(
-                request,
-                (
-                    "Searching via the URL query string is no longer supported. "
-                    "The form below has been filled in with the parameters from your request. "
-                    'To execute your search, please click "Search".'
-                ),
-            )
+    def _get_cache_key(params):
+        fields = set(SearchForm.base_fields) - {'sort'}
+        kwargs = dict([(k, v) for (k, v) in list(params.items()) if k in fields])
+        key = "doc:document:search:" + hashlib.sha512(json.dumps(kwargs, sort_keys=True).encode('utf-8')).hexdigest()
+        return key
+
+    if request.GET:
+        # backwards compatibility
+        get_params = request.GET.copy()
+        if 'activeDrafts' in request.GET:
+            get_params['activedrafts'] = request.GET['activeDrafts']
+        if 'oldDrafts' in request.GET:
+            get_params['olddrafts'] = request.GET['oldDrafts']
+        if 'subState' in request.GET:
+            get_params['substate'] = request.GET['subState']
+
+        form = SearchForm(get_params)
+        if not form.is_valid():
+            return HttpResponseBadRequest("form not valid: %s" % form.errors)
+
+        cache_key = _get_cache_key(get_params)
+        cached_val = cache.get(cache_key)
+        if cached_val:
+            [results, meta] = cached_val
         else:
-            form = SearchForm()
+            results = retrieve_search_results(form)
+            results, meta = prepare_document_table(request, results, get_params)
+            cache.set(cache_key, [results, meta]) # for settings.CACHE_MIDDLEWARE_SECONDS
+            log(f"Search results computed for {get_params}")
+        meta['searching'] = True
+    else:
+        form = SearchForm()
+        results = []
+        meta = { 'by': None, 'searching': False }
+        get_params = QueryDict('')
 
-    return render(
-        request,
-        "doc/search/search.html",
-        context={"form": form, "docs": results, "meta": meta},
+    return render(request, 'doc/search/search.html', {
+        'form':form, 'docs':results, 'meta':meta, 'queryargs':get_params.urlencode() },
     )
 
-
-@ensure_csrf_cookie
 def frontpage(request):
     form = SearchForm()
     return render(request, 'doc/frontpage.html', {'form':form})
 
-
 def search_for_name(request, name):
     def find_unique(n):
         exact = Document.objects.filter(name__iexact=n).first()
diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py
index a75028bf79..1d91041b25 100644
--- a/ietf/liaisons/forms.py
+++ b/ietf/liaisons/forms.py
@@ -203,7 +203,6 @@ def get_results(self):
 class CustomModelMultipleChoiceField(ModelMultipleChoiceField):
     '''If value is a QuerySet, return it as is (for use in widget.render)'''
     def prepare_value(self, value):
-        # Need QuerySetAny instead of QuerySet until django-stubs 5.0.1
         if isinstance(value, QuerySetAny):
             return value
         if (hasattr(value, '__iter__') and
diff --git a/ietf/liaisons/widgets.py b/ietf/liaisons/widgets.py
index 3d4f2d13a5..74368e83f2 100644
--- a/ietf/liaisons/widgets.py
+++ b/ietf/liaisons/widgets.py
@@ -35,7 +35,6 @@ def render(self, name, value, **kwargs):
         html = '<div id="id_%s">' % name
         html += '<span class="d-none showAttachmentsEmpty form-control widget">No files attached</span>'
         html += '<div class="attachedFiles form-control widget">'
-        # Need QuerySetAny instead of QuerySet until django-stubs 5.0.1
         if value and isinstance(value, QuerySetAny):
             for attachment in value:
                 html += '<a class="initialAttach" href="%s">%s</a>&nbsp' % (conditional_escape(attachment.document.get_href()), conditional_escape(attachment.document.title))
diff --git a/ietf/templates/doc/search/search_form.html b/ietf/templates/doc/search/search_form.html
index 6c91894c8c..d4f463ec66 100644
--- a/ietf/templates/doc/search/search_form.html
+++ b/ietf/templates/doc/search/search_form.html
@@ -4,10 +4,8 @@
 {% load widget_tweaks %}
 {% load ietf_filters %}
 <form id="search_form"
-      method="post"
       class="form-horizontal"
       action="{% url 'ietf.doc.views_search.search' %}">
-    {% csrf_token %}
     <!-- [html-validate-disable-block input-missing-label -- labelled via aria-label] -->
     <div class="input-group search_field">
         {{ form.name|add_class:"form-control"|attr:"placeholder:Document name/title/RFC number"|attr:"aria-label:Document name/title/RFC number" }}

From ff5b2e12f390162796135c7b9cd17c80934fb55a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 22 Nov 2024 18:54:15 -0400
Subject: [PATCH 188/656] chore: fix is_authenticated logging (#8266)

"o" instead of "i" to check response headers
---
 ietf/utils/jsonlogger.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/utils/jsonlogger.py b/ietf/utils/jsonlogger.py
index c383ba310f..b02cd7af2b 100644
--- a/ietf/utils/jsonlogger.py
+++ b/ietf/utils/jsonlogger.py
@@ -31,4 +31,4 @@ def add_fields(self, log_record, record, message_dict):
         log_record.setdefault("cf_connecting_ip", record.args["{cf-connecting-ip}i"])
         log_record.setdefault("cf_connecting_ipv6", record.args["{cf-connecting-ipv6}i"])
         log_record.setdefault("cf_ray", record.args["{cf-ray}i"])
-        log_record.setdefault("is_authenticated", record.args["{x-datatracker-is-authenticated}i"])
+        log_record.setdefault("is_authenticated", record.args["{x-datatracker-is-authenticated}o"])

From a45f10c0ce6cbfbc6061049532056892605fa036 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Mon, 25 Nov 2024 15:42:05 +0000
Subject: [PATCH 189/656] fix: increase timeout from 5s to 15s for playwright
 status tests (#8267)

---
 playwright/tests/status/status.spec.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/playwright/tests/status/status.spec.js b/playwright/tests/status/status.spec.js
index daef7a88f1..c70617e2fd 100644
--- a/playwright/tests/status/status.spec.js
+++ b/playwright/tests/status/status.spec.js
@@ -20,7 +20,8 @@ test.describe('site status', () => {
     by: 'Exile is a cool Amiga game'
   }
 
-  test.beforeEach(({ browserName }) => {
+  test.beforeEach(({ page, browserName }) => {
+    page.setDefaultTimeout(15 * 1000) // increase default timeout
     test.skip(browserName === 'firefox', 'bypassing flaky tests on Firefox')
   })
 

From 4b5760240417cac57801b5dc7b47296f94cf9ac3 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 25 Nov 2024 14:51:44 -0400
Subject: [PATCH 190/656] chore: log in-flight request lists on worker term
 (#8272)

* chore: log in-flight request lists on worker term

* style: Black

* chore: suppress empty in-flight logs

* chore: use list consistently
---
 dev/build/gunicorn.conf.py | 70 ++++++++++++++++++++++++++++++++------
 1 file changed, 59 insertions(+), 11 deletions(-)

diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py
index 89d3dd58d3..cabbee0b1e 100644
--- a/dev/build/gunicorn.conf.py
+++ b/dev/build/gunicorn.conf.py
@@ -12,26 +12,25 @@
             "level": "INFO",
             "handlers": ["console"],
             "propagate": False,
-            "qualname": "gunicorn.error"
+            "qualname": "gunicorn.error",
         },
-
         "gunicorn.access": {
             "level": "INFO",
             "handlers": ["access_console"],
             "propagate": False,
-            "qualname": "gunicorn.access"
-        }
+            "qualname": "gunicorn.access",
+        },
     },
     "handlers": {
         "console": {
             "class": "logging.StreamHandler",
             "formatter": "json",
-            "stream": "ext://sys.stdout"
+            "stream": "ext://sys.stdout",
         },
         "access_console": {
             "class": "logging.StreamHandler",
             "formatter": "access_json",
-            "stream": "ext://sys.stdout"
+            "stream": "ext://sys.stdout",
         },
     },
     "formatters": {
@@ -44,14 +43,29 @@
             "class": "ietf.utils.jsonlogger.GunicornRequestJsonFormatter",
             "style": "{",
             "format": "{asctime}{levelname}{message}{name}{process}",
-        }
-    }
+        },
+    },
 }
 
-def pre_request(worker, req):
+# Track in-flight requests and emit a list of what was happeningwhen a worker is terminated.
+# For the default sync worker, there will only be one request per PID, but allow for the
+# possibility of multiple requests in case we switch to a different worker class.
+#
+# This dict is only visible within a single worker, but key by pid to guarantee no conflicts.
+#
+# Use a list rather than a set to allow for the possibility of overlapping identical requests.
+in_flight_by_pid: dict[str, list[str]] = {}  # pid -> list of in-flight requests
+
+
+def _describe_request(req):
+    """Generate a consistent description of a request
+
+    The return value is used identify in-flight requests, so it must not vary between the
+    start and end of handling a request. E.g., do not include a timestamp.
+    """
     client_ip = "-"
     cf_ray = "-"
-    for (header, value) in req.headers:
+    for header, value in req.headers:
         header = header.lower()
         if header == "cf-connecting-ip":
             client_ip = value
@@ -61,4 +75,38 @@ def pre_request(worker, req):
         path = f"{req.path}?{req.query}"
     else:
         path = req.path
-    worker.log.info(f"gunicorn starting to process {req.method} {path} (client_ip={client_ip}, cf_ray={cf_ray})")
+    return f"{req.method} {path} (client_ip={client_ip}, cf_ray={cf_ray})"
+
+
+def pre_request(worker, req):
+    """Log the start of a request and add it to the in-flight list"""
+    request_description = _describe_request(req)
+    worker.log.info(f"gunicorn starting to process {request_description}")
+    in_flight = in_flight_by_pid.setdefault(worker.pid, [])
+    in_flight.append(request_description)
+
+
+def worker_abort(worker):
+    """Emit an error log if any requests were in-flight"""
+    in_flight = in_flight_by_pid.get(worker.pid, [])
+    if len(in_flight) > 0:
+        worker.log.error(
+            f"Aborted worker {worker.pid} with in-flight requests: {', '.join(in_flight)}"
+        )
+
+
+def worker_int(worker):
+    """Emit an error log if any requests were in-flight"""
+    in_flight = in_flight_by_pid.get(worker.pid, [])
+    if len(in_flight) > 0:
+        worker.log.error(
+            f"Interrupted worker {worker.pid} with in-flight requests: {', '.join(in_flight)}"
+        )
+
+
+def post_request(worker, req, environ, resp):
+    """Remove request from in-flight list when we finish handling it"""
+    request_description = _describe_request(req)
+    in_flight = in_flight_by_pid.get(worker.pid, [])
+    if request_description in in_flight:
+        in_flight.remove(request_description)

From 5bb79bb7ca05d4660a5f21c704cb58c0e1668b37 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 26 Nov 2024 09:23:30 -0400
Subject: [PATCH 191/656] ci: fix comment in settings_local.py

---
 k8s/settings_local.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 33ac4f1e38..a63a02e536 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -82,7 +82,7 @@ def _multiline_to_list(s):
 # Set DEBUG if DATATRACKER_DEBUG env var is the word "true"
 DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true"
 
-# DATATRACKER_ALLOWED_HOSTS env var is a comma-separated list of allowed hosts
+# DATATRACKER_ALLOWED_HOSTS env var is a newline-separated list of allowed hosts
 _allowed_hosts_str = os.environ.get("DATATRACKER_ALLOWED_HOSTS", None)
 if _allowed_hosts_str is not None:
     ALLOWED_HOSTS = _multiline_to_list(_allowed_hosts_str)

From 35e3433c2a75fa6452161dd8028df943b3e8167b Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 26 Nov 2024 19:15:50 -0600
Subject: [PATCH 192/656] fix: use lxml 5 (#8279)

---
 requirements.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 5c80d5301e..91647729be 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -39,7 +39,8 @@ jsonfield>=3.1.0    # for SubmissionCheck.  This is https://github.com/bradjaspe
 jsonschema[format]>=4.2.1
 jwcrypto>=1.2    # for signed notifications - this is aspirational, and is not really used.
 logging_tree>=1.9    # Used only by the showloggers management command
-lxml>=4.8.0,<5
+lxml>=5.3.0    # lxml[html_clean] fails on some architectures
+lxml_html_clean>=0.4.1
 markdown>=3.3.6
 types-markdown>=3.3.6
 mock>=4.0.3    # Used only by tests, of course

From 15f3ebd2d5092a91e2feb6673212744384325b8b Mon Sep 17 00:00:00 2001
From: rjsparks <rjsparks@users.noreply.github.com>
Date: Wed, 27 Nov 2024 01:27:15 +0000
Subject: [PATCH 193/656] ci: update base image target version to 20241127T0116

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index c9f91163ce..ef77fef58f 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20241114T1954
+FROM ghcr.io/ietf-tools/datatracker-app-base:20241127T0116
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 7bbb32355d..6420933c22 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20241114T1954
+20241127T0116

From c18900a8e693e5704ae6a0336dba25388ced5896 Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Wed, 27 Nov 2024 14:20:24 +1100
Subject: [PATCH 194/656] feat: Expose important library versions (#7713)

* feat: Expose important library versions

Update `/api/version` to include

```
"other": {
    "xml2rfc": "<version>",
    "weasyprint": "<version>"
},
```

Fixes #3415

* fix: Use importlib

* chore: Reomve additional newline

* fix: Expose libraries that are important for document submission

* fix: Rename IMPORTANT_LIBRARIES as ADVERTISE_VERSIONS
---
 ietf/api/tests.py | 2 ++
 ietf/api/views.py | 8 ++++++++
 ietf/settings.py  | 2 ++
 3 files changed, 12 insertions(+)

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 7dc9c42cc8..a8d6ac4e57 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -936,6 +936,8 @@ def test_api_version(self):
         r = self.client.get(url)
         data = r.json()
         self.assertEqual(data['version'], ietf.__version__+ietf.__patch__)
+        for lib in settings.ADVERTISE_VERSIONS:
+            self.assertIn(lib, data['other'])
         self.assertEqual(data['dumptime'], "2022-08-31 07:10:01 +0000")
         DumpInfo.objects.update(tz='PST8PDT')
         r = self.client.get(url)
diff --git a/ietf/api/views.py b/ietf/api/views.py
index df67cffd56..3e56757528 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -23,6 +23,7 @@
 from django.views.decorators.gzip import gzip_page
 from django.views.generic.detail import DetailView
 from email.message import EmailMessage
+from importlib.metadata import version as metadata_version
 from jwcrypto.jwk import JWK
 from tastypie.exceptions import BadRequest
 from tastypie.serializers import Serializer
@@ -240,9 +241,16 @@ def version(request):
         if dumpinfo.tz != "UTC":
             dumpdate = pytz.timezone(dumpinfo.tz).localize(dumpinfo.date.replace(tzinfo=None))
     dumptime = dumpdate.strftime('%Y-%m-%d %H:%M:%S %z') if dumpinfo else None
+
+    # important libraries
+    __version_extra__ = {}
+    for lib in settings.ADVERTISE_VERSIONS:
+        __version_extra__[lib] = metadata_version(lib)
+
     return HttpResponse(
             json.dumps({
                         'version': ietf.__version__+ietf.__patch__,
+                        'other': __version_extra__,
                         'dumptime': dumptime,
                     }),
                 content_type='application/json',
diff --git a/ietf/settings.py b/ietf/settings.py
index 30f4b367c6..368f61c6e6 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -1273,6 +1273,8 @@ def skip_unreadable_post(record):
 
 PUBLISH_IPR_STATES = ['posted', 'removed', 'removed_objfalse']
 
+ADVERTISE_VERSIONS = ["markdown", "pyang", "rfc2html", "xml2rfc"]
+
 # We provide a secret key only for test and development modes.  It's
 # absolutely vital that django fails to start in production mode unless a
 # secret key has been provided elsewhere, not in this file which is

From c58490bb36ed6c548e064ce3ab4e7c42fd48f0a5 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 27 Nov 2024 16:54:28 -0400
Subject: [PATCH 195/656] feat: django-rest-framework + Person/Email API
 (#8256)

* feat: django-rest-framework + Person/Email API (#8233)

* chore: djangorestframework -> requirements.txt

* chore: auth/perm/schema classes for drf

* chore: settings for drf and friends

* chore: comment that api/serializer.py is not DRF

* feat: URL router for DRF

* feat: simple api/v3/person/{id} endpoint

* fix: actually working demo endpoint

* chore: no auth for PersonViewSet

* ci: params in ci-run-tests.yml

* Revert "ci: params in ci-run-tests.yml"

This reverts commit 03808ddf94afe42b7382ddd3730959987389612b.

* feat: email addresses for person API

* feat: email update api (WIP)

* fix: working Email API endpoint

* chore: annotate address format in api schema

* chore: api adjustments

* feat: expose SpectacularAPIView

At least for now...

* chore: better schema_path_prefix

* feat: permissions for DRF API

* refactor: use permissions classes

* refactor: extract NewEmailForm validation for reuse

* refactor: ietfauth.validators module

* refactor: send new email conf req via helper

* feat: API call to issue new address request

* chore: move datatracker DRF api to /api/core/

* fix: unused import

* fix: lint

* test: drf URL names + API tests (#8248)

* refactor: better drf URL naming

* test: test person-detail view

* test: permissions

* test: add_email tests + stubs

* test: test email update

* test: test 404 vs 403

* fix: fix permissions

* test: test email partial update

* test: assert we have a nonexistent PK

* chore: disable DRF api for now

* chore: fix git inanity

* fix: lint

* test: disable tests of disabled code

* test: more lint
---
 ietf/api/apps.py            |   4 +
 ietf/api/authentication.py  |  19 +++
 ietf/api/permissions.py     |  39 +++++
 ietf/api/routers.py         |  16 ++
 ietf/api/schema.py          |  20 +++
 ietf/api/serializer.py      |   5 +-
 ietf/api/tests_core.py      | 289 ++++++++++++++++++++++++++++++++++++
 ietf/api/urls.py            |  14 +-
 ietf/ietfauth/forms.py      |  27 +---
 ietf/ietfauth/utils.py      |  49 +++++-
 ietf/ietfauth/validators.py |  34 +++++
 ietf/ietfauth/views.py      |  27 +---
 ietf/person/api.py          |  45 ++++++
 ietf/person/serializers.py  |  39 +++++
 ietf/settings.py            |  73 +++++++++
 requirements.txt            |   3 +
 16 files changed, 650 insertions(+), 53 deletions(-)
 create mode 100644 ietf/api/authentication.py
 create mode 100644 ietf/api/permissions.py
 create mode 100644 ietf/api/routers.py
 create mode 100644 ietf/api/schema.py
 create mode 100644 ietf/api/tests_core.py
 create mode 100644 ietf/ietfauth/validators.py
 create mode 100644 ietf/person/api.py
 create mode 100644 ietf/person/serializers.py

diff --git a/ietf/api/apps.py b/ietf/api/apps.py
index 7eca094a62..4549e0d7f2 100644
--- a/ietf/api/apps.py
+++ b/ietf/api/apps.py
@@ -12,4 +12,8 @@ def ready(self):
         interact with the database. See        
         https://docs.djangoproject.com/en/4.2/ref/applications/#django.apps.AppConfig.ready
         """
+        # Populate our API list now that the app registry is set up
         populate_api_list()
+
+        # Import drf-spectacular extensions 
+        import ietf.api.schema  # pyflakes: ignore
diff --git a/ietf/api/authentication.py b/ietf/api/authentication.py
new file mode 100644
index 0000000000..dfab0d72b8
--- /dev/null
+++ b/ietf/api/authentication.py
@@ -0,0 +1,19 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+#
+from rest_framework import authentication
+from django.contrib.auth.models import AnonymousUser
+
+
+class ApiKeyAuthentication(authentication.BaseAuthentication):
+    """API-Key header authentication"""
+
+    def authenticate(self, request):
+        """Extract the authentication token, if present
+        
+        This does not validate the token, it just arranges for it to be available in request.auth.
+        It's up to a Permissions class to validate it for the appropriate endpoint.
+        """
+        token = request.META.get("HTTP_X_API_KEY", None)
+        if token is None:
+            return None
+        return AnonymousUser(), token  # available as request.user and request.auth
diff --git a/ietf/api/permissions.py b/ietf/api/permissions.py
new file mode 100644
index 0000000000..8f7fdd026f
--- /dev/null
+++ b/ietf/api/permissions.py
@@ -0,0 +1,39 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+#
+from rest_framework import permissions
+from ietf.api.ietf_utils import is_valid_token
+
+
+class HasApiKey(permissions.BasePermission):
+    """Permissions class that validates a token using is_valid_token
+    
+    The view class must indicate the relevant endpoint by setting `api_key_endpoint`.
+    Must be used with an Authentication class that puts a token in request.auth.
+    """
+    def has_permission(self, request, view):
+        endpoint = getattr(view, "api_key_endpoint", None)
+        auth_token = getattr(request, "auth", None)
+        if endpoint is not None and auth_token is not None:
+            return is_valid_token(endpoint, auth_token)
+        return False
+
+
+class IsOwnPerson(permissions.BasePermission):
+    """Permission to access own Person object"""
+    def has_object_permission(self, request, view, obj):
+        if not (request.user.is_authenticated and hasattr(request.user, "person")):
+            return False
+        return obj == request.user.person
+
+
+class BelongsToOwnPerson(permissions.BasePermission):
+    """Permission to access objects associated with own Person
+    
+    Requires that the object have a "person" field that indicates ownership.
+    """
+    def has_object_permission(self, request, view, obj):
+        if not (request.user.is_authenticated and hasattr(request.user, "person")):
+            return False
+        return (
+            hasattr(obj, "person") and obj.person == request.user.person
+        )
diff --git a/ietf/api/routers.py b/ietf/api/routers.py
new file mode 100644
index 0000000000..745ddaa811
--- /dev/null
+++ b/ietf/api/routers.py
@@ -0,0 +1,16 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+"""Custom django-rest-framework routers"""
+from django.core.exceptions import ImproperlyConfigured
+from rest_framework import routers
+
+class PrefixedSimpleRouter(routers.SimpleRouter):
+    """SimpleRouter that adds a dot-separated prefix to its basename"""
+    def __init__(self, name_prefix="", *args, **kwargs):
+        self.name_prefix = name_prefix
+        if len(self.name_prefix) == 0 or self.name_prefix[-1] == ".":
+            raise ImproperlyConfigured("Cannot use a name_prefix that is empty or ends with '.'")
+        super().__init__(*args, **kwargs)
+
+    def get_default_basename(self, viewset):
+        basename = super().get_default_basename(viewset)
+        return f"{self.name_prefix}.{basename}"
diff --git a/ietf/api/schema.py b/ietf/api/schema.py
new file mode 100644
index 0000000000..7340149685
--- /dev/null
+++ b/ietf/api/schema.py
@@ -0,0 +1,20 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+#
+from drf_spectacular.extensions import OpenApiAuthenticationExtension
+
+
+class ApiKeyAuthenticationScheme(OpenApiAuthenticationExtension):
+    """Authentication scheme extension for the ApiKeyAuthentication
+
+    Used by drf-spectacular when rendering the OpenAPI schema
+    """
+    target_class = "ietf.api.authentication.ApiKeyAuthentication"
+    name = "apiKeyAuth"
+
+    def get_security_definition(self, auto_schema):
+        return {
+            "type": "apiKey",
+            "description": "Shared secret in the X-Api-Key header",
+            "name": "X-Api-Key",
+            "in": "header",
+        }
diff --git a/ietf/api/serializer.py b/ietf/api/serializer.py
index 27f194c5b5..d5bca430e0 100644
--- a/ietf/api/serializer.py
+++ b/ietf/api/serializer.py
@@ -1,6 +1,9 @@
-# Copyright The IETF Trust 2018-2020, All Rights Reserved
+# Copyright The IETF Trust 2018-2024, All Rights Reserved
 # -*- coding: utf-8 -*-
+"""Serialization utilities
 
+This is _not_ for django-rest-framework!
+"""
 
 import hashlib
 import json
diff --git a/ietf/api/tests_core.py b/ietf/api/tests_core.py
new file mode 100644
index 0000000000..7e45deac8a
--- /dev/null
+++ b/ietf/api/tests_core.py
@@ -0,0 +1,289 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+"""Core API tests"""
+from unittest.mock import patch
+# from unittest.mock import patch, call
+
+from django.urls import reverse as urlreverse, NoReverseMatch
+from rest_framework.test import APIClient
+
+# from ietf.person.factories import PersonFactory, EmailFactory
+# from ietf.person.models import Person
+from ietf.utils.test_utils import TestCase
+
+
+class CoreApiTestCase(TestCase):
+    client_class = APIClient
+
+
+class PersonTests(CoreApiTestCase):
+    # Tests disabled until we activate the DRF URLs in api/urls.py
+    
+    def test_person_detail(self):
+        with self.assertRaises(NoReverseMatch, msg="Re-enable test when this view is enabled"):
+            urlreverse("ietf.api.core_api.person-detail", kwargs={"pk": 1})
+
+        # person = PersonFactory()
+        # other_person = PersonFactory()
+        # url = urlreverse("ietf.api.core_api.person-detail", kwargs={"pk": person.pk})
+        # bad_pk = person.pk + 10000
+        # if Person.objects.filter(pk=bad_pk).exists():
+        #     bad_pk += 10000  # if this doesn't get us clear, something is wrong...
+        #     self.assertFalse(
+        #         Person.objects.filter(pk=bad_pk).exists(),
+        #         "Failed to find a non-existent person pk",
+        #     )
+        # bad_url = urlreverse("ietf.api.core_api.person-detail", kwargs={"pk": bad_pk})
+        # r = self.client.get(bad_url, format="json")
+        # self.assertEqual(r.status_code, 403, "Must be logged in preferred to 404")
+        # r = self.client.get(url, format="json")
+        # self.assertEqual(r.status_code, 403, "Must be logged in")
+        # self.client.login(
+        #     username=other_person.user.username,
+        #     password=other_person.user.username + "+password",
+        # )
+        # r = self.client.get(bad_url, format="json")
+        # self.assertEqual(r.status_code, 404)
+        # r = self.client.get(url, format="json")
+        # self.assertEqual(r.status_code, 403, "Can only retrieve self")
+        # self.client.login(
+        #     username=person.user.username, password=person.user.username + "+password"
+        # )
+        # r = self.client.get(url, format="json")
+        # self.assertEqual(r.status_code, 200)
+        # self.assertEqual(
+        #     r.data,
+        #     {
+        #         "id": person.pk,
+        #         "name": person.name,
+        #         "emails": [
+        #             {
+        #                 "person": person.pk,
+        #                 "address": email.address,
+        #                 "primary": email.primary,
+        #                 "active": email.active,
+        #                 "origin": email.origin,
+        #             }
+        #             for email in person.email_set.all()
+        #         ],
+        #     },
+        # )
+
+    @patch("ietf.person.api.send_new_email_confirmation_request")
+    def test_add_email(self, send_confirmation_mock):
+        with self.assertRaises(NoReverseMatch, msg="Re-enable this test when this view is enabled"):
+            urlreverse("ietf.api.core_api.person-email", kwargs={"pk": 1})
+            
+        # email = EmailFactory(address="old@example.org")
+        # person = email.person
+        # other_person = PersonFactory()
+        # url = urlreverse("ietf.api.core_api.person-email", kwargs={"pk": person.pk})
+        # post_data = {"address": "new@example.org"}
+        # 
+        # r = self.client.post(url, data=post_data, format="json")
+        # self.assertEqual(r.status_code, 403, "Must be logged in")
+        # self.assertFalse(send_confirmation_mock.called)
+        # 
+        # self.client.login(
+        #     username=other_person.user.username,
+        #     password=other_person.user.username + "+password",
+        # )
+        # r = self.client.post(url, data=post_data, format="json")
+        # self.assertEqual(r.status_code, 403, "Can only retrieve self")
+        # self.assertFalse(send_confirmation_mock.called)
+        # 
+        # self.client.login(
+        #     username=person.user.username, password=person.user.username + "+password"
+        # )
+        # r = self.client.post(url, data=post_data, format="json")
+        # self.assertEqual(r.status_code, 200)
+        # self.assertEqual(r.data, {"address": "new@example.org"})
+        # self.assertTrue(send_confirmation_mock.called)
+        # self.assertEqual(
+        #     send_confirmation_mock.call_args, call(person, "new@example.org")
+        # )
+
+
+class EmailTests(CoreApiTestCase):
+    def test_email_update(self):
+        with self.assertRaises(NoReverseMatch, msg="Re-enable this test when the view is enabled"):
+            urlreverse(
+                "ietf.api.core_api.email-detail", kwargs={"pk": "original@example.org"}
+            )
+
+        # email = EmailFactory(
+        #     address="original@example.org", primary=False, active=True, origin="factory"
+        # )
+        # person = email.person
+        # other_person = PersonFactory()
+        # url = urlreverse(
+        #     "ietf.api.core_api.email-detail", kwargs={"pk": "original@example.org"}
+        # )
+        # bad_url = urlreverse(
+        #     "ietf.api.core_api.email-detail",
+        #     kwargs={"pk": "not-original@example.org"},
+        # )
+        # 
+        # r = self.client.put(
+        #     bad_url, data={"primary": True, "active": False}, format="json"
+        # )
+        # self.assertEqual(r.status_code, 403, "Must be logged in preferred to 404")
+        # r = self.client.put(url, data={"primary": True, "active": False}, format="json")
+        # self.assertEqual(r.status_code, 403, "Must be logged in")
+        # 
+        # self.client.login(
+        #     username=other_person.user.username,
+        #     password=other_person.user.username + "+password",
+        # )
+        # r = self.client.put(
+        #     bad_url, data={"primary": True, "active": False}, format="json"
+        # )
+        # self.assertEqual(r.status_code, 404, "No such address")
+        # r = self.client.put(url, data={"primary": True, "active": False}, format="json")
+        # self.assertEqual(r.status_code, 403, "Can only access own addresses")
+        # 
+        # self.client.login(
+        #     username=person.user.username, password=person.user.username + "+password"
+        # )
+        # r = self.client.put(url, data={"primary": True, "active": False}, format="json")
+        # self.assertEqual(r.status_code, 200)
+        # self.assertEqual(
+        #     r.data,
+        #     {
+        #         "person": person.pk,
+        #         "address": "original@example.org",
+        #         "primary": True,
+        #         "active": False,
+        #         "origin": "factory",
+        #     },
+        # )
+        # email.refresh_from_db()
+        # self.assertEqual(email.person, person)
+        # self.assertEqual(email.address, "original@example.org")
+        # self.assertTrue(email.primary)
+        # self.assertFalse(email.active)
+        # self.assertEqual(email.origin, "factory")
+        # 
+        # # address / origin should be immutable
+        # r = self.client.put(
+        #     url,
+        #     data={
+        #         "address": "modified@example.org",
+        #         "primary": True,
+        #         "active": False,
+        #         "origin": "hacker",
+        #     },
+        #     format="json",
+        # )
+        # self.assertEqual(r.status_code, 200)
+        # self.assertEqual(
+        #     r.data,
+        #     {
+        #         "person": person.pk,
+        #         "address": "original@example.org",
+        #         "primary": True,
+        #         "active": False,
+        #         "origin": "factory",
+        #     },
+        # )
+        # email.refresh_from_db()
+        # self.assertEqual(email.person, person)
+        # self.assertEqual(email.address, "original@example.org")
+        # self.assertTrue(email.primary)
+        # self.assertFalse(email.active)
+        # self.assertEqual(email.origin, "factory")
+
+    def test_email_partial_update(self):
+        with self.assertRaises(NoReverseMatch, msg="Re-enable this test when the view is enabled"):
+            urlreverse(
+                "ietf.api.core_api.email-detail", kwargs={"pk": "original@example.org"}
+            )
+
+        # email = EmailFactory(
+        #     address="original@example.org", primary=False, active=True, origin="factory"
+        # )
+        # person = email.person
+        # other_person = PersonFactory()
+        # url = urlreverse(
+        #     "ietf.api.core_api.email-detail", kwargs={"pk": "original@example.org"}
+        # )
+        # bad_url = urlreverse(
+        #     "ietf.api.core_api.email-detail",
+        #     kwargs={"pk": "not-original@example.org"},
+        # )
+        # 
+        # r = self.client.patch(
+        #     bad_url, data={"primary": True}, format="json"
+        # )
+        # self.assertEqual(r.status_code, 403, "Must be logged in preferred to 404")
+        # r = self.client.patch(url, data={"primary": True}, format="json")
+        # self.assertEqual(r.status_code, 403, "Must be logged in")
+        # 
+        # self.client.login(
+        #     username=other_person.user.username,
+        #     password=other_person.user.username + "+password",
+        # )
+        # r = self.client.patch(
+        #     bad_url, data={"primary": True}, format="json"
+        # )
+        # self.assertEqual(r.status_code, 404, "No such address")
+        # r = self.client.patch(url, data={"primary": True}, format="json")
+        # self.assertEqual(r.status_code, 403, "Can only access own addresses")
+        # 
+        # self.client.login(
+        #     username=person.user.username, password=person.user.username + "+password"
+        # )
+        # r = self.client.patch(url, data={"primary": True}, format="json")
+        # self.assertEqual(r.status_code, 200)
+        # self.assertEqual(
+        #     r.data,
+        #     {
+        #         "person": person.pk,
+        #         "address": "original@example.org",
+        #         "primary": True,
+        #         "active": True,
+        #         "origin": "factory",
+        #     },
+        # )
+        # email.refresh_from_db()
+        # self.assertEqual(email.person, person)
+        # self.assertEqual(email.address, "original@example.org")
+        # self.assertTrue(email.primary)
+        # self.assertTrue(email.active)
+        # self.assertEqual(email.origin, "factory")
+        # 
+        # r = self.client.patch(url, data={"active": False}, format="json")
+        # self.assertEqual(r.status_code, 200)
+        # self.assertEqual(
+        #     r.data,
+        #     {
+        #         "person": person.pk,
+        #         "address": "original@example.org",
+        #         "primary": True,
+        #         "active": False,
+        #         "origin": "factory",
+        #     },
+        # )
+        # email.refresh_from_db()
+        # self.assertEqual(email.person, person)
+        # self.assertEqual(email.address, "original@example.org")
+        # self.assertTrue(email.primary)
+        # self.assertFalse(email.active)
+        # self.assertEqual(email.origin, "factory")
+        # 
+        # r = self.client.patch(url, data={"address": "modified@example.org"}, format="json")
+        # self.assertEqual(r.status_code, 200)  # extra fields allowed, but ignored
+        # email.refresh_from_db()
+        # self.assertEqual(email.person, person)
+        # self.assertEqual(email.address, "original@example.org")
+        # self.assertTrue(email.primary)
+        # self.assertFalse(email.active)
+        # self.assertEqual(email.origin, "factory")
+        # 
+        # r = self.client.patch(url, data={"origin": "hacker"}, format="json")
+        # self.assertEqual(r.status_code, 200)  # extra fields allowed, but ignored
+        # email.refresh_from_db()
+        # self.assertEqual(email.person, person)
+        # self.assertEqual(email.address, "original@example.org")
+        # self.assertTrue(email.primary)
+        # self.assertFalse(email.active)
+        # self.assertEqual(email.origin, "factory")
diff --git a/ietf/api/urls.py b/ietf/api/urls.py
index 6846e32747..a9aaaf5805 100644
--- a/ietf/api/urls.py
+++ b/ietf/api/urls.py
@@ -5,12 +5,21 @@
 from django.views.generic import TemplateView
 
 from ietf import api
-from ietf.api import views as api_views
 from ietf.doc import views_ballot
 from ietf.meeting import views as meeting_views
 from ietf.submit import views as submit_views
 from ietf.utils.urls import url
 
+from . import views as api_views
+
+# DRF API routing - disabled until we plan to use it
+# from drf_spectacular.views import SpectacularAPIView
+# from django.urls import path
+# from ietf.person import api as person_api
+# from .routers import PrefixedSimpleRouter 
+# core_router = PrefixedSimpleRouter(name_prefix="ietf.api.core_api")  # core api router
+# core_router.register("email", person_api.EmailViewSet)
+# core_router.register("person", person_api.PersonViewSet)
 
 api.autodiscover()
 
@@ -21,6 +30,9 @@
     url(r'^v1/?$', api_views.top_level),
     # For mailarchive use, requires secretariat role
     url(r'^v2/person/person', api_views.ApiV2PersonExportView.as_view()),
+    # --- DRF API ---
+    # path("core/", include(core_router.urls)),
+    # path("schema/", SpectacularAPIView.as_view()),
     #
     # --- Custom API endpoints, sorted alphabetically ---
     # Email alias information for drafts
diff --git a/ietf/ietfauth/forms.py b/ietf/ietfauth/forms.py
index 9b8ee22e0b..a70f7b6ca1 100644
--- a/ietf/ietfauth/forms.py
+++ b/ietf/ietfauth/forms.py
@@ -6,17 +6,15 @@
 from unidecode import unidecode
 
 from django import forms
-from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.db import models
 from django.contrib.auth.models import User
 
-import debug                            # pyflakes:ignore
-
 from ietf.person.models import Person, Email
 from ietf.mailinglists.models import Allowlisted
 from ietf.utils.text import isascii
 
+from .validators import prevent_at_symbol, prevent_system_name, prevent_anonymous_name, is_allowed_address
 from .widgets import PasswordStrengthInput, PasswordConfirmationInput
 
 
@@ -53,19 +51,6 @@ def ascii_cleaner(supposedly_ascii):
         raise forms.ValidationError("Only unaccented Latin characters are allowed.")
     return supposedly_ascii
 
-def prevent_at_symbol(name):
-    if "@" in name:
-        raise forms.ValidationError("Please fill in name - this looks like an email address (@ is not allowed in names).")
-
-def prevent_system_name(name):
-    name_without_spaces = name.replace(" ", "").replace("\t", "")
-    if "(system)" in name_without_spaces.lower():
-        raise forms.ValidationError("Please pick another name - this name is reserved.")
-
-def prevent_anonymous_name(name):
-    name_without_spaces = name.replace(" ", "").replace("\t", "")
-    if "anonymous" in name_without_spaces.lower():
-        raise forms.ValidationError("Please pick another name - this name is reserved.")
 
 class PersonPasswordForm(forms.ModelForm, PasswordForm):
 
@@ -156,15 +141,7 @@ def clean(self):
 
 
 class NewEmailForm(forms.Form):
-    new_email = forms.EmailField(label="New email address", required=False)
-
-    def clean_new_email(self):
-        email = self.cleaned_data.get("new_email", "")
-        for pat in settings.EXCLUDED_PERSONAL_EMAIL_REGEX_PATTERNS:
-            if re.search(pat, email):
-                raise ValidationError("This email address is not valid in a datatracker account")
-
-        return email
+    new_email = forms.EmailField(label="New email address", required=False, validators=[is_allowed_address])
 
 
 class RoleEmailForm(forms.Form):
diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py
index d8bd67a4e0..b4c6da14ea 100644
--- a/ietf/ietfauth/utils.py
+++ b/ietf/ietfauth/utils.py
@@ -12,6 +12,8 @@
 
 from django.conf import settings
 from django.contrib.auth import REDIRECT_FIELD_NAME
+from django.contrib.sites.models import Site
+from django.core import signing
 from django.core.exceptions import PermissionDenied
 from django.db.models import Q
 from django.http import HttpResponseRedirect
@@ -20,9 +22,10 @@
 import debug                            # pyflakes:ignore
 
 from ietf.group.models import Role, GroupFeatures
-from ietf.person.models import Person
+from ietf.person.models import Email, Person
 from ietf.person.utils import get_dots
 from ietf.doc.utils_bofreq import bofreq_editors
+from ietf.utils.mail import send_mail
 
 def user_is_person(user, person):
     """Test whether user is associated with person."""
@@ -394,3 +397,47 @@ def can_request_rfc_publication(user, doc):
         return False # See the docstring
     else:
         return False
+
+
+def send_new_email_confirmation_request(person: Person, address: str):
+    """Request confirmation of a new email address
+    
+    If the email address is already in use, sends an alert to it. If not, sends a confirmation request.
+    By design, does not indicate which was sent. This is intended to make it a bit harder to scrape addresses
+    with a mindless bot.
+    """
+    auth = signing.dumps([person.user.username, address], salt="add_email")
+    domain = Site.objects.get_current().domain
+    from_email = settings.DEFAULT_FROM_EMAIL
+
+    existing = Email.objects.filter(address=address).first()
+    if existing:
+        subject = f"Attempt to add your email address by {person.name}"
+        send_mail(
+            None,
+            address,
+            from_email,
+            subject,
+            "registration/add_email_exists_email.txt",
+            {
+                "domain": domain,
+                "email": address,
+                "person": person,
+            },
+        )
+    else:
+        subject = f"Confirm email address for {person.name}"
+        send_mail(
+            None,
+            address,
+            from_email,
+            subject,
+            "registration/add_email_email.txt",
+            {
+                "domain": domain,
+                "auth": auth,
+                "email": address,
+                "person": person,
+                "expire": settings.DAYS_TO_EXPIRE_REGISTRATION_LINK,
+            },
+        )
diff --git a/ietf/ietfauth/validators.py b/ietf/ietfauth/validators.py
new file mode 100644
index 0000000000..84684f34d5
--- /dev/null
+++ b/ietf/ietfauth/validators.py
@@ -0,0 +1,34 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+import re
+
+from django import forms
+from django.conf import settings
+from django.core.exceptions import ValidationError
+
+
+def prevent_at_symbol(name):
+    if "@" in name:
+        raise forms.ValidationError(
+            "Please fill in name - this looks like an email address (@ is not allowed in names)."
+        )
+
+
+def prevent_system_name(name):
+    name_without_spaces = name.replace(" ", "").replace("\t", "")
+    if "(system)" in name_without_spaces.lower():
+        raise forms.ValidationError("Please pick another name - this name is reserved.")
+
+
+def prevent_anonymous_name(name):
+    name_without_spaces = name.replace(" ", "").replace("\t", "")
+    if "anonymous" in name_without_spaces.lower():
+        raise forms.ValidationError("Please pick another name - this name is reserved.")
+
+
+def is_allowed_address(value):
+    """Validate that an address complies with datatracker requirements"""
+    for pat in settings.EXCLUDED_PERSONAL_EMAIL_REGEX_PATTERNS:
+        if re.search(pat, value):
+            raise ValidationError(
+                "This email address is not valid in a datatracker account"
+            )
diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py
index 32bb5c92be..23f66ce824 100644
--- a/ietf/ietfauth/views.py
+++ b/ietf/ietfauth/views.py
@@ -65,7 +65,7 @@
 from ietf.ietfauth.forms import ( RegistrationForm, PasswordForm, ResetPasswordForm, TestEmailForm,
                                 ChangePasswordForm, get_person_form, RoleEmailForm,
                                 NewEmailForm, ChangeUsernameForm, PersonPasswordForm)
-from ietf.ietfauth.utils import has_role
+from ietf.ietfauth.utils import has_role, send_new_email_confirmation_request
 from ietf.name.models import ExtResourceName
 from ietf.nomcom.models import NomCom
 from ietf.person.models import Person, Email, Alias, PersonalApiKey, PERSON_API_KEY_VALUES
@@ -297,31 +297,8 @@ def profile(request):
                 to_email = f.cleaned_data["new_email"]
                 if not to_email:
                     continue
-
                 email_confirmations.append(to_email)
-
-                auth = django.core.signing.dumps([person.user.username, to_email], salt="add_email")
-
-                domain = Site.objects.get_current().domain
-                from_email = settings.DEFAULT_FROM_EMAIL
-
-                existing = Email.objects.filter(address=to_email).first()
-                if existing:
-                    subject = 'Attempt to add your email address by %s' % person.name
-                    send_mail(request, to_email, from_email, subject, 'registration/add_email_exists_email.txt', {
-                        'domain': domain,
-                        'email': to_email,
-                        'person': person,
-                    })
-                else:
-                    subject = 'Confirm email address for %s' % person.name
-                    send_mail(request, to_email, from_email, subject, 'registration/add_email_email.txt', {
-                        'domain': domain,
-                        'auth': auth,
-                        'email': to_email,
-                        'person': person,
-                        'expire': settings.DAYS_TO_EXPIRE_REGISTRATION_LINK,
-                    })
+                send_new_email_confirmation_request(person, to_email)
 
             for r in roles:
                 e = r.email_form.cleaned_data["email"]
diff --git a/ietf/person/api.py b/ietf/person/api.py
new file mode 100644
index 0000000000..960785a3d4
--- /dev/null
+++ b/ietf/person/api.py
@@ -0,0 +1,45 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+"""DRF API Views"""
+from rest_framework import mixins, viewsets
+from rest_framework.decorators import action
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.response import Response
+
+from ietf.api.permissions import BelongsToOwnPerson, IsOwnPerson
+from ietf.ietfauth.utils import send_new_email_confirmation_request
+
+from .models import Email, Person
+from .serializers import NewEmailSerializer, EmailSerializer, PersonSerializer
+
+
+class EmailViewSet(mixins.UpdateModelMixin, viewsets.GenericViewSet):
+    """Email viewset
+    
+    Only allows updating an existing email for now.
+    """
+    permission_classes = [IsAuthenticated & BelongsToOwnPerson]
+    queryset = Email.objects.all()
+    serializer_class = EmailSerializer
+    lookup_value_regex = '.+@.+'  # allow @-sign in the pk
+
+
+class PersonViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+    """Person viewset"""
+    permission_classes = [IsAuthenticated & IsOwnPerson]
+    queryset = Person.objects.all()
+    serializer_class = PersonSerializer
+
+    @action(detail=True, methods=["post"], serializer_class=NewEmailSerializer)
+    def email(self, request, pk=None):
+        """Add an email address for this Person
+        
+        Always succeeds if the email address is valid. Causes a confirmation email to be sent to the
+        requested address and completion of that handshake will actually add the email address. If the
+        address already exists, an alert will be sent instead of the confirmation email.
+        """
+        person = self.get_object()
+        serializer = self.get_serializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        # This may or may not actually send a confirmation, but doesn't reveal that to the user.
+        send_new_email_confirmation_request(person, serializer.validated_data["address"])
+        return Response(serializer.data)
diff --git a/ietf/person/serializers.py b/ietf/person/serializers.py
new file mode 100644
index 0000000000..023d77d4bc
--- /dev/null
+++ b/ietf/person/serializers.py
@@ -0,0 +1,39 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+"""DRF Serializers"""
+
+from rest_framework import serializers
+
+from ietf.ietfauth.validators import is_allowed_address
+
+from .models import Email, Person
+
+
+class EmailSerializer(serializers.ModelSerializer):
+    """Email serializer for read/update"""
+
+    address = serializers.EmailField(read_only=True)
+
+    class Meta:
+        model = Email
+        fields = [
+            "person",
+            "address",
+            "primary",
+            "active",
+            "origin",
+        ]
+        read_only_fields = ["person", "address", "origin"]
+
+
+class NewEmailSerializer(serializers.Serializer):
+    """Serialize a new email address request"""
+    address = serializers.EmailField(validators=[is_allowed_address])
+
+
+class PersonSerializer(serializers.ModelSerializer):
+    """Person serializer"""
+    emails = EmailSerializer(many=True, source="email_set")
+
+    class Meta:
+        model = Person
+        fields = ["id", "name", "emails"]
diff --git a/ietf/settings.py b/ietf/settings.py
index 368f61c6e6..6990037585 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -455,6 +455,9 @@ def skip_unreadable_post(record):
     'corsheaders',
     'django_markup',
     'oidc_provider',
+    'drf_spectacular',
+    'drf_standardized_errors',
+    'rest_framework',
     'simple_history',
     'tastypie',
     'widget_tweaks',
@@ -550,6 +553,76 @@ def skip_unreadable_post(record):
         '::1',
 )
 
+# django-rest-framework configuration
+REST_FRAMEWORK = {
+    "DEFAULT_AUTHENTICATION_CLASSES": [
+        "ietf.api.authentication.ApiKeyAuthentication",
+        "rest_framework.authentication.SessionAuthentication",
+    ],
+    "DEFAULT_PERMISSION_CLASSES": [
+        "ietf.api.permissions.HasApiKey",
+    ],
+    "DEFAULT_RENDERER_CLASSES": [
+        "rest_framework.renderers.JSONRenderer",
+    ],
+    "DEFAULT_PARSER_CLASSES": [
+        "rest_framework.parsers.JSONParser",
+    ],
+    "DEFAULT_SCHEMA_CLASS": "drf_standardized_errors.openapi.AutoSchema",
+    "EXCEPTION_HANDLER": "drf_standardized_errors.handler.exception_handler",
+}
+
+# DRF OpenApi schema settings
+SPECTACULAR_SETTINGS = {
+    "TITLE": "Datatracker API",
+    "DESCRIPTION": "Datatracker API",
+    "VERSION": "1.0.0",
+    "SCHEMA_PATH_PREFIX": "/api/",
+    "COMPONENT_SPLIT_REQUEST": True,
+    "COMPONENT_NO_READ_ONLY_REQUIRED": True,
+    "SERVERS": [
+        {"url": "http://localhost:8000", "description": "local dev server"},
+        {"url": "https://datatracker.ietf.org", "description": "production server"},
+    ],
+    # The following settings are needed for drf-standardized-errors
+    "ENUM_NAME_OVERRIDES": {
+        "ValidationErrorEnum": "drf_standardized_errors.openapi_serializers.ValidationErrorEnum.choices",
+        "ClientErrorEnum": "drf_standardized_errors.openapi_serializers.ClientErrorEnum.choices",
+        "ServerErrorEnum": "drf_standardized_errors.openapi_serializers.ServerErrorEnum.choices",
+        "ErrorCode401Enum": "drf_standardized_errors.openapi_serializers.ErrorCode401Enum.choices",
+        "ErrorCode403Enum": "drf_standardized_errors.openapi_serializers.ErrorCode403Enum.choices",
+        "ErrorCode404Enum": "drf_standardized_errors.openapi_serializers.ErrorCode404Enum.choices",
+        "ErrorCode405Enum": "drf_standardized_errors.openapi_serializers.ErrorCode405Enum.choices",
+        "ErrorCode406Enum": "drf_standardized_errors.openapi_serializers.ErrorCode406Enum.choices",
+        "ErrorCode415Enum": "drf_standardized_errors.openapi_serializers.ErrorCode415Enum.choices",
+        "ErrorCode429Enum": "drf_standardized_errors.openapi_serializers.ErrorCode429Enum.choices",
+        "ErrorCode500Enum": "drf_standardized_errors.openapi_serializers.ErrorCode500Enum.choices",
+    },
+    "POSTPROCESSING_HOOKS": ["drf_standardized_errors.openapi_hooks.postprocess_schema_enums"],
+}
+
+# DRF Standardized Errors settings
+DRF_STANDARDIZED_ERRORS = {
+    # enable the standardized errors when DEBUG=True for unhandled exceptions.
+    # By default, this is set to False so you're able to view the traceback in
+    # the terminal and get more information about the exception.
+    "ENABLE_IN_DEBUG_FOR_UNHANDLED_EXCEPTIONS": False,
+    # ONLY the responses that correspond to these status codes will appear
+    # in the API schema.
+    "ALLOWED_ERROR_STATUS_CODES": [
+        "400",
+        # "401",
+        # "403",
+        "404",
+        # "405",
+        # "406",
+        # "415",
+        # "429",
+        # "500",
+    ],
+
+}
+
 # no slash at end
 IDTRACKER_BASE_URL = "https://datatracker.ietf.org"
 RFCDIFF_BASE_URL = "https://author-tools.ietf.org/iddiff"
diff --git a/requirements.txt b/requirements.txt
index 91647729be..f974113d8f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,8 +24,11 @@ django-stubs>=4.2.7,<5    # The django-stubs version used determines the the myp
 django-tastypie>=0.14.7,<0.15.0    # Version must be locked in sync with version of Django
 django-vite>=2.0.2,<3
 django-widget-tweaks>=1.4.12
+djangorestframework>=3.15,<4
 djlint>=1.0.0    # To auto-indent templates via "djlint --profile django --reformat"
 docutils>=0.18.1    # Used only by dbtemplates for RestructuredText
+drf-spectacular>=0.27
+drf-standardized-errors[openapi] >= 0.14
 types-docutils>=0.18.1
 factory-boy>=3.3
 github3.py>=3.2.0

From e5b6e330b18ad24f029266715186b4f8efc3c25d Mon Sep 17 00:00:00 2001
From: rjsparks <rjsparks@users.noreply.github.com>
Date: Wed, 27 Nov 2024 21:05:52 +0000
Subject: [PATCH 196/656] ci: update base image target version to 20241127T2054

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index ef77fef58f..7af27e7d13 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20241127T0116
+FROM ghcr.io/ietf-tools/datatracker-app-base:20241127T2054
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 6420933c22..e4b05ed700 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20241127T0116
+20241127T2054

From f76137eaae3dee819d62e9592f9360d4241e7098 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 5 Dec 2024 10:31:09 -0400
Subject: [PATCH 197/656] fix: disable raw/include in RST (#8300)

* fix: disable raw/include in RST

* fix: suppress warnings
---
 ietf/settings.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ietf/settings.py b/ietf/settings.py
index 6990037585..cf8abe9f4d 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -1149,11 +1149,14 @@ def skip_unreadable_post(record):
 MARKUP_SETTINGS = {
     'restructuredtext': {
         'settings_overrides': {
+            'report_level': 3,  # error (3) or severe (4) only
             'initial_header_level': 3,
             'doctitle_xform': False,
             'footnote_references': 'superscript',
             'trim_footnote_reference_space': True,
             'default_reference_context': 'view',
+            'raw_enabled': False,  # critical for security
+            'file_insertion_enabled': False,  # critical for security
             'link_base': ''
         }
     }

From b39b80fe1a3f218ed0cb25dc7bdb520a89454ebf Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 5 Dec 2024 08:46:14 -0600
Subject: [PATCH 198/656] fix: test file existence using metadata (#8292)

* fix: test file existance using metadata

* fix: use Path more

* fix: don't read the file to see if it exists

* fix: more conservative error handling

* chore: remove unused import
---
 ietf/doc/models.py | 18 ++++++++++++++----
 ietf/doc/tests.py  | 15 +++++++++++++++
 ietf/doc/utils.py  |  2 +-
 3 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index 077502db11..03698c80c3 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -4,7 +4,6 @@
 
 import datetime
 import logging
-import io
 import os
 
 import django.db
@@ -530,16 +529,27 @@ def replaces(self):
     def replaced_by(self):
         return set([ r.document for r in self.related_that("replaces") ])
 
-    def text(self, size = -1):
+    def _text_path(self):
         path = self.get_file_name()
         root, ext =  os.path.splitext(path)
         txtpath = root+'.txt'
         if ext != '.txt' and os.path.exists(txtpath):
             path = txtpath
+        return path
+    
+    def text_exists(self):
+        path = Path(self._text_path())
+        return path.exists()
+
+    def text(self, size = -1):
+        path = Path(self._text_path())
+        if not path.exists():
+            return None
         try:
-            with io.open(path, 'rb') as file:
+            with path.open('rb') as file:
                 raw = file.read(size)
-        except IOError:
+        except IOError as e:
+            log.log(f"Error reading text for {path}: {e}")
             return None
         text = None
         try:
diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index f0c8e30626..0630fcd8d4 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -3318,3 +3318,18 @@ def test_investigate(self):
             self.assertEqual(r.status_code, 200)
             q = PyQuery(r.content)
             self.assertEqual(len(q("#id_name_fragment.is-invalid")), 1)
+
+class LogIOErrorTests(TestCase):
+
+    def test_doc_text_io_error(self):
+
+        d = IndividualDraftFactory()
+
+        with mock.patch("ietf.doc.models.Path") as path_cls_mock:
+            with mock.patch("ietf.doc.models.log.log") as log_mock:
+                path_cls_mock.return_value.exists.return_value = True
+                path_cls_mock.return_value.open.return_value.__enter__.return_value.read.side_effect = IOError("Bad things happened")
+                text = d.text()
+                self.assertIsNone(text)
+                self.assertTrue(log_mock.called)
+                self.assertIn("Bad things happened", log_mock.call_args[0][0])
diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index a30430829a..b2bc0997b1 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -1081,7 +1081,7 @@ def build_file_urls(doc: Union[Document, DocHistory]):
             label = "plain text" if t == "txt" else t
             file_urls.append((label, base + doc.name + "-" + doc.rev + "." + t))
 
-        if doc.text():
+        if doc.text_exists():
             file_urls.append(("htmlized", urlreverse('ietf.doc.views_doc.document_html', kwargs=dict(name=doc.name, rev=doc.rev))))
             file_urls.append(("pdfized", urlreverse('ietf.doc.views_doc.document_pdfized', kwargs=dict(name=doc.name, rev=doc.rev))))
         file_urls.append(("bibtex", urlreverse('ietf.doc.views_doc.document_bibtex',kwargs=dict(name=doc.name,rev=doc.rev))))

From 64c5ef16bd96cd8f2cc960a0224cfa502663c7ac Mon Sep 17 00:00:00 2001
From: Greg Wood <ghwood@ietf.org>
Date: Thu, 5 Dec 2024 09:50:07 -0500
Subject: [PATCH 199/656] fix: clarify replace wording (#8244)

* Clarify "Replace" in I-D submission form

Update help text In the "Replacement Information" section of
the "Status" tab

* refactor: update node, eslint, neostandard + fix esm (#8083)

* chore: update dependencies

* fix: eslint + neostandard

* fix: add corepack prompt env var to init script

* docs: Update README.md

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>

* ci: update base image target version to 20241114T1703

* ci: fix tests.yml workflow

* fix: clarify "Replace" in I-D submission form #8205

update quote syntax issue on previous update help text In the "Replacement Information" section of the "Status" tab to fix #8059

* Revert "refactor: update node, eslint, neostandard + fix esm (#8083)"

This reverts commit 649879efd745470f6e0cc6768d889f45640e1505.

* Revert "ci: update base image target version to 20241114T1703"

This reverts commit f11144017ed788bc7a38a1d028127434f0d50eb4.

* Revert "ci: fix tests.yml workflow"

This reverts commit 39231321c49291565a39608b98740e098c74dda7.

* Update forms.py

---------

Co-authored-by: Nicolas Giard <github@ngpixel.com>
Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
Co-authored-by: NGPixel <NGPixel@users.noreply.github.com>
---
 ietf/submit/forms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py
index bed87b77c2..183e081242 100644
--- a/ietf/submit/forms.py
+++ b/ietf/submit/forms.py
@@ -502,7 +502,7 @@ def clean_name(self):
         return name
 
 class ReplacesForm(forms.Form):
-    replaces = SearchableDocumentsField(required=False, help_text="Any Internet-Drafts that this document replaces (approval required for replacing an Internet-Draft you are not the author of)")
+    replaces = SearchableDocumentsField(required=False, help_text='Do not enter anything here if you are only submitting the next version of your Internet-Draft. Only enter items here if this submission is intended to replace an I-D with a different name. A typical use of this field is to note what individual I-Ds are replaced by a new -00 Working group I-D. Note that additional approval will be required to replace an I-D for which you are not an author.')
 
     def __init__(self, *args, **kwargs):
         self.name = kwargs.pop("name")

From 167752ba765e3ca740ee26b5beca268fc9a5c4aa Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 6 Dec 2024 13:17:55 -0400
Subject: [PATCH 200/656] feat: log ASN (#8309)

* feat: log ip_src_asnum in nginx

* feat: log asn from gunicorn
---
 dev/build/gunicorn.conf.py | 5 ++++-
 ietf/utils/jsonlogger.py   | 1 +
 k8s/nginx-logging.conf     | 7 +++++--
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py
index cabbee0b1e..6666a0d37d 100644
--- a/dev/build/gunicorn.conf.py
+++ b/dev/build/gunicorn.conf.py
@@ -64,18 +64,21 @@ def _describe_request(req):
     start and end of handling a request. E.g., do not include a timestamp.
     """
     client_ip = "-"
+    asn = "-"
     cf_ray = "-"
     for header, value in req.headers:
         header = header.lower()
         if header == "cf-connecting-ip":
             client_ip = value
+        elif header == "x-ip-src-asnum":
+            asn = value
         elif header == "cf-ray":
             cf_ray = value
     if req.query:
         path = f"{req.path}?{req.query}"
     else:
         path = req.path
-    return f"{req.method} {path} (client_ip={client_ip}, cf_ray={cf_ray})"
+    return f"{req.method} {path} (client_ip={client_ip}, asn={asn}, cf_ray={cf_ray})"
 
 
 def pre_request(worker, req):
diff --git a/ietf/utils/jsonlogger.py b/ietf/utils/jsonlogger.py
index b02cd7af2b..6502cab0cb 100644
--- a/ietf/utils/jsonlogger.py
+++ b/ietf/utils/jsonlogger.py
@@ -31,4 +31,5 @@ def add_fields(self, log_record, record, message_dict):
         log_record.setdefault("cf_connecting_ip", record.args["{cf-connecting-ip}i"])
         log_record.setdefault("cf_connecting_ipv6", record.args["{cf-connecting-ipv6}i"])
         log_record.setdefault("cf_ray", record.args["{cf-ray}i"])
+        log_record.setdefault("asn", record.args["{x-ip-src-asnum}i"])
         log_record.setdefault("is_authenticated", record.args["{x-datatracker-is-authenticated}o"])
diff --git a/k8s/nginx-logging.conf b/k8s/nginx-logging.conf
index 3c4ade4614..0bc7deca81 100644
--- a/k8s/nginx-logging.conf
+++ b/k8s/nginx-logging.conf
@@ -1,4 +1,6 @@
-# Define JSON log format - must be loaded before config that references it
+# Define JSON log format - must be loaded before config that references it.
+# Note that each line is fully enclosed in single quotes. Commas in arrays are
+# intentionally inside the single quotes.
 log_format  ietfjson  escape=json
   '{'
     '"time":"$${keepempty}time_iso8601",'
@@ -16,5 +18,6 @@ log_format  ietfjson  escape=json
     '"x_forwarded_proto":"$${keepempty}http_x_forwarded_proto",'
     '"cf_connecting_ip":"$${keepempty}http_cf_connecting_ip",'
     '"cf_connecting_ipv6":"$${keepempty}http_cf_connecting_ipv6",'
-    '"cf_ray":"$${keepempty}http_cf_ray"'
+    '"cf_ray":"$${keepempty}http_cf_ray",'
+    '"asn":"$${keepempty}http_x_ip_src_asnum"'
   '}';

From 3055d17eb1c8f91a2152142bfce975b6dd2e82c1 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 9 Dec 2024 10:33:03 -0600
Subject: [PATCH 201/656] fix: remove unreliable statistics (#8307)

---
 ietf/settings.py                              |   4 +-
 ietf/stats/tests.py                           | 127 +--
 ietf/stats/views.py                           | 894 +-----------------
 ietf/templates/stats/document_stats.html      |  86 --
 .../document_stats_author_affiliation.html    | 113 ---
 .../document_stats_author_citations.html      |  72 --
 .../document_stats_author_continent.html      |  69 --
 .../stats/document_stats_author_country.html  | 136 ---
 .../document_stats_author_documents.html      |  69 --
 .../stats/document_stats_author_hindex.html   |  83 --
 .../stats/document_stats_authors.html         |  68 --
 .../stats/document_stats_format.html          |  63 --
 .../stats/document_stats_formlang.html        |  63 --
 .../templates/stats/document_stats_pages.html |  62 --
 .../templates/stats/document_stats_words.html |  62 --
 .../stats/document_stats_yearly.html          |  52 -
 .../includes/number_with_details_cell.html    |  15 -
 ietf/templates/stats/index.html               |   9 +-
 ietf/templates/stats/meeting_stats.html       |  35 -
 .../stats/meeting_stats_continent.html        |  61 --
 .../stats/meeting_stats_country.html          |  97 --
 .../stats/meeting_stats_overview.html         | 160 ----
 22 files changed, 18 insertions(+), 2382 deletions(-)
 delete mode 100644 ietf/templates/stats/document_stats.html
 delete mode 100644 ietf/templates/stats/document_stats_author_affiliation.html
 delete mode 100644 ietf/templates/stats/document_stats_author_citations.html
 delete mode 100644 ietf/templates/stats/document_stats_author_continent.html
 delete mode 100644 ietf/templates/stats/document_stats_author_country.html
 delete mode 100644 ietf/templates/stats/document_stats_author_documents.html
 delete mode 100644 ietf/templates/stats/document_stats_author_hindex.html
 delete mode 100644 ietf/templates/stats/document_stats_authors.html
 delete mode 100644 ietf/templates/stats/document_stats_format.html
 delete mode 100644 ietf/templates/stats/document_stats_formlang.html
 delete mode 100644 ietf/templates/stats/document_stats_pages.html
 delete mode 100644 ietf/templates/stats/document_stats_words.html
 delete mode 100644 ietf/templates/stats/document_stats_yearly.html
 delete mode 100644 ietf/templates/stats/includes/number_with_details_cell.html
 delete mode 100644 ietf/templates/stats/meeting_stats.html
 delete mode 100644 ietf/templates/stats/meeting_stats_continent.html
 delete mode 100644 ietf/templates/stats/meeting_stats_country.html
 delete mode 100644 ietf/templates/stats/meeting_stats_overview.html

diff --git a/ietf/settings.py b/ietf/settings.py
index cf8abe9f4d..7c3dc7fa16 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -809,8 +809,8 @@ def skip_unreadable_post(record):
 SESSION_REQUEST_FROM_EMAIL = 'IETF Meeting Session Request Tool <session-request@ietf.org>' 
 
 SECRETARIAT_SUPPORT_EMAIL = "support@ietf.org"
-SECRETARIAT_ACTION_EMAIL = "ietf-action@ietf.org"
-SECRETARIAT_INFO_EMAIL = "ietf-info@ietf.org"
+SECRETARIAT_ACTION_EMAIL = SECRETARIAT_SUPPORT_EMAIL
+SECRETARIAT_INFO_EMAIL = SECRETARIAT_SUPPORT_EMAIL
 
 # Put real password in settings_local.py
 IANA_SYNC_PASSWORD = "secret"
diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py
index f0e8a19c4a..47027277be 100644
--- a/ietf/stats/tests.py
+++ b/ietf/stats/tests.py
@@ -13,22 +13,16 @@
 import debug    # pyflakes:ignore
 
 from django.urls import reverse as urlreverse
-from django.utils import timezone
 
 from ietf.utils.test_utils import login_testing_unauthorized, TestCase
 import ietf.stats.views
 
-from ietf.submit.models import Submission
-from ietf.doc.factories import WgDraftFactory, WgRfcFactory
-from ietf.doc.models import Document, State, RelatedDocument, NewRevisionDocEvent, DocumentAuthor
+
 from ietf.group.factories import RoleFactory
-from ietf.meeting.factories import MeetingFactory, AttendedFactory
+from ietf.meeting.factories import MeetingFactory
 from ietf.person.factories import PersonFactory
-from ietf.person.models import Person, Email
-from ietf.name.models import FormalLanguageName, DocRelationshipName, CountryName
 from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory
-from ietf.stats.models import MeetingRegistration, CountryAlias
-from ietf.stats.factories import MeetingRegistrationFactory
+from ietf.stats.models import MeetingRegistration
 from ietf.stats.tasks import fetch_meeting_attendance_task
 from ietf.stats.utils import get_meeting_registration_data, FetchStats, fetch_attendance_from_meetings
 from ietf.utils.timezone import date_today
@@ -41,121 +35,14 @@ def test_stats_index(self):
         self.assertEqual(r.status_code, 200)
 
     def test_document_stats(self):
-        WgRfcFactory()
-        draft = WgDraftFactory()
-        DocumentAuthor.objects.create(
-            document=draft,
-            person=Person.objects.get(email__address="aread@example.org"),
-            email=Email.objects.get(address="aread@example.org"),
-            country="Germany",
-            affiliation="IETF",
-            order=1
-        )
-
-        # create some data for the statistics
-        Submission.objects.create(
-            authors=[ { "name": "Some Body", "email": "somebody@example.com", "affiliation": "Some Inc.", "country": "US" }],
-            pages=30,
-            rev=draft.rev,
-            words=4000,
-            draft=draft,
-            file_types=".txt",
-            state_id="posted",
-        )
-
-        draft.formal_languages.add(FormalLanguageName.objects.get(slug="xml"))
-        Document.objects.filter(pk=draft.pk).update(words=4000)
-        # move it back so it shows up in the yearly summaries
-        NewRevisionDocEvent.objects.filter(doc=draft, rev=draft.rev).update(
-            time=timezone.now() - datetime.timedelta(days=500))
-
-        referencing_draft = Document.objects.create(
-            name="draft-ietf-mars-referencing",
-            type_id="draft",
-            title="Referencing",
-            stream_id="ietf",
-            abstract="Test",
-            rev="00",
-            pages=2,
-            words=100
-            )
-        referencing_draft.set_state(State.objects.get(used=True, type="draft", slug="active"))
-        RelatedDocument.objects.create(
-            source=referencing_draft,
-            target=draft,
-            relationship=DocRelationshipName.objects.get(slug="refinfo")
-        )
-        NewRevisionDocEvent.objects.create(
-            type="new_revision",
-            by=Person.objects.get(name="(System)"),
-            doc=referencing_draft,
-            desc="New revision available",
-            rev=referencing_draft.rev,
-            time=timezone.now() - datetime.timedelta(days=1000)
-        )
+        r = self.client.get(urlreverse("ietf.stats.views.document_stats"))
+        self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index"))
 
 
-        # check redirect
-        url = urlreverse(ietf.stats.views.document_stats)
-
-        authors_url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": "authors" })
-
-        r = self.client.get(url)
-        self.assertEqual(r.status_code, 302)
-        self.assertTrue(authors_url in r["Location"])
-
-        # check various stats types
-        for stats_type in ["authors", "pages", "words", "format", "formlang",
-                           "author/documents", "author/affiliation", "author/country",
-                           "author/continent", "author/citations", "author/hindex",
-                           "yearly/affiliation", "yearly/country", "yearly/continent"]:
-            for document_type in ["", "rfc", "draft"]:
-                for time_choice in ["", "5y"]:
-                    url = urlreverse(ietf.stats.views.document_stats, kwargs={ "stats_type": stats_type })
-                    r = self.client.get(url, {
-                        "type": document_type,
-                        "time": time_choice,
-                    })
-                    self.assertEqual(r.status_code, 200)
-                    q = PyQuery(r.content)
-                    self.assertTrue(q('#chart'))
-                    if not stats_type.startswith("yearly"):
-                        self.assertTrue(q('table.stats-data'))
-
     def test_meeting_stats(self):
-        # create some data for the statistics
-        meeting = MeetingFactory(type_id='ietf', date=date_today(), number="96")
-        MeetingRegistrationFactory(first_name='John', last_name='Smith', country_code='US', email="john.smith@example.us", meeting=meeting, attended=True)
-        CountryAlias.objects.get_or_create(alias="US", country=CountryName.objects.get(slug="US"))
-        p = MeetingRegistrationFactory(first_name='Jaume', last_name='Guillaume', country_code='FR', email="jaume.guillaume@example.fr", meeting=meeting, attended=False).person
-        CountryAlias.objects.get_or_create(alias="FR", country=CountryName.objects.get(slug="FR"))
-        AttendedFactory(session__meeting=meeting,person=p)
-        # check redirect
-        url = urlreverse(ietf.stats.views.meeting_stats)
-
-        authors_url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": "overview" })
-
-        r = self.client.get(url)
-        self.assertEqual(r.status_code, 302)
-        self.assertTrue(authors_url in r["Location"])
-
-        # check various stats types
-        for stats_type in ["overview", "country", "continent"]:
-            url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": stats_type })
-            r = self.client.get(url)
-            self.assertEqual(r.status_code, 200)
-            q = PyQuery(r.content)
-            self.assertTrue(q('#chart'))
-            if stats_type == "overview":
-                self.assertTrue(q('table.stats-data'))
+        r = self.client.get(urlreverse("ietf.stats.views.meeting_stats"))
+        self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index"))
 
-        for stats_type in ["country", "continent"]:
-            url = urlreverse(ietf.stats.views.meeting_stats, kwargs={ "stats_type": stats_type, "num": meeting.number })
-            r = self.client.get(url)
-            self.assertEqual(r.status_code, 200)
-            q = PyQuery(r.content)
-            self.assertTrue(q('#chart'))
-            self.assertTrue(q('table.stats-data'))
                 
     def test_known_country_list(self):
         # check redirect
diff --git a/ietf/stats/views.py b/ietf/stats/views.py
index ea73d9f4fc..504d84e86d 100644
--- a/ietf/stats/views.py
+++ b/ietf/stats/views.py
@@ -2,25 +2,18 @@
 # -*- coding: utf-8 -*-
 
 
-import os
 import calendar
 import datetime
-import email.utils
 import itertools
 import json
 import dateutil.relativedelta
 from collections import defaultdict
 
-from django.conf import settings
 from django.contrib.auth.decorators import login_required
-from django.core.cache import cache
-from django.db.models import Count, Q, Subquery, OuterRef
 from django.http import HttpResponseRedirect
-from django.shortcuts import get_object_or_404, render
+from django.shortcuts import render
 from django.urls import reverse as urlreverse
-from django.utils import timezone
-from django.utils.safestring import mark_safe
-from django.utils.text import slugify
+
 
 import debug                            # pyflakes:ignore
 
@@ -29,18 +22,12 @@
                                ReviewAssignmentData,
                                sum_period_review_assignment_stats,
                                sum_raw_review_assignment_aggregations)
-from ietf.submit.models import Submission
 from ietf.group.models import Role, Group
 from ietf.person.models import Person
-from ietf.name.models import ReviewResultName, CountryName, DocRelationshipName, ReviewAssignmentStateName
-from ietf.person.name import plain_name
-from ietf.doc.models import Document, RelatedDocument, State, DocEvent
-from ietf.meeting.models import Meeting
-from ietf.stats.models import MeetingRegistration, CountryAlias
-from ietf.stats.utils import get_aliased_affiliations, get_aliased_countries, compute_hirsch_index
+from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName
 from ietf.ietfauth.utils import has_role
 from ietf.utils.response import permission_denied
-from ietf.utils.timezone import date_today, DEADLINE_TZINFO, RPC_TZINFO
+from ietf.utils.timezone import date_today, DEADLINE_TZINFO
 
 
 def stats_index(request):
@@ -135,632 +122,8 @@ def add_labeled_top_series_from_bins(chart_data, bins, limit):
         })
 
 def document_stats(request, stats_type=None):
-    def build_document_stats_url(stats_type_override=Ellipsis, get_overrides=None):
-        if get_overrides is None:
-            get_overrides={}
-        kwargs = {
-            "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override,
-        }
-
-        return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.items() if v is not None }) + generate_query_string(request.GET, get_overrides)
-
-    # the length limitation is to keep the key shorter than memcached's limit
-    # of 250 after django has added the key_prefix and key_version parameters
-    cache_key = ("stats:document_stats:%s:%s" % (stats_type, slugify(request.META.get('QUERY_STRING',''))))[:228]
-    data = cache.get(cache_key)
-    if not data:
-        names_limit = settings.STATS_NAMES_LIMIT
-        # statistics types
-        possible_document_stats_types = add_url_to_choices([
-            ("authors", "Number of authors"),
-            ("pages", "Pages"),
-            ("words", "Words"),
-            ("format", "Format"),
-            ("formlang", "Formal languages"),
-        ], lambda slug: build_document_stats_url(stats_type_override=slug))
-
-        possible_author_stats_types = add_url_to_choices([
-            ("author/documents", "Number of documents"),
-            ("author/affiliation", "Affiliation"),
-            ("author/country", "Country"),
-            ("author/continent", "Continent"),
-            ("author/citations", "Citations"),
-            ("author/hindex", "h-index"),
-        ], lambda slug: build_document_stats_url(stats_type_override=slug))
-
-        possible_yearly_stats_types = add_url_to_choices([
-            ("yearly/affiliation", "Affiliation"),
-            ("yearly/country", "Country"),
-            ("yearly/continent", "Continent"),
-        ], lambda slug: build_document_stats_url(stats_type_override=slug))
-
-
-        if not stats_type:
-            return HttpResponseRedirect(build_document_stats_url(stats_type_override=possible_document_stats_types[0][0]))
-
-
-        possible_document_types = add_url_to_choices([
-            ("", "All"),
-            ("rfc", "RFCs"),
-            ("draft", "Internet-Drafts"),
-        ], lambda slug: build_document_stats_url(get_overrides={ "type": slug }))
-
-        document_type = get_choice(request, "type", possible_document_types) or ""
-
-
-        possible_time_choices = add_url_to_choices([
-            ("", "All time"),
-            ("5y", "Past 5 years"),
-        ], lambda slug: build_document_stats_url(get_overrides={ "time": slug }))
-
-        time_choice = request.GET.get("time") or ""
-
-        from_time = None
-        if "y" in time_choice:
-            try:
-                y = int(time_choice.rstrip("y"))
-                from_time = timezone.now() - dateutil.relativedelta.relativedelta(years=y)
-            except ValueError:
-                pass
-
-        chart_data = []
-        table_data = []
-        stats_title = ""
-        template_name = stats_type.replace("/", "_")
-        bin_size = 1
-        alias_data = []
-        eu_countries = None
-
-
-        if any(stats_type == t[0] for t in possible_document_stats_types):
-            # filter documents
-            document_filters = Q(type__in=["draft","rfc"]) # TODO - review lots of "rfc is a draft" assumptions below
-
-            rfc_state = State.objects.get(type="rfc", slug="published")
-            if document_type == "rfc":
-                document_filters &= Q(states=rfc_state)
-            elif document_type == "draft":
-                document_filters &= ~Q(states=rfc_state)
-
-            if from_time:
-                # this is actually faster than joining in the database,
-                # despite the round-trip back and forth
-                docs_within_time_constraint = list(Document.objects.filter(
-                    type="draft",
-                    docevent__time__gte=from_time,
-                    docevent__type__in=["published_rfc", "new_revision"],
-                ).values_list("pk",flat=True))
-
-                document_filters &= Q(pk__in=docs_within_time_constraint)
-
-            document_qs = Document.objects.filter(document_filters)
-
-            if document_type == "rfc":
-                doc_label = "RFC"
-            elif document_type == "draft":
-                doc_label = "draft"
-            else:
-                doc_label = "document"
-
-            total_docs = document_qs.values_list("name").distinct().count()
-
-            if stats_type == "authors":
-                stats_title = "Number of authors for each {}".format(doc_label)
-
-                bins = defaultdict(set)
-
-                for name, author_count in document_qs.values_list("name").annotate(Count("documentauthor")).values_list("name","documentauthor__count"):
-                    bins[author_count or 0].add(name)
-
-                series_data = []
-                for author_count, names in sorted(bins.items(), key=lambda t: t[0]):
-                    percentage = len(names) * 100.0 / (total_docs or 1)
-                    series_data.append((author_count, percentage))
-                    table_data.append((author_count, percentage, len(names), list(names)[:names_limit]))
-
-                chart_data.append({ "data": series_data })
-
-            elif stats_type == "pages":
-                stats_title = "Number of pages for each {}".format(doc_label)
-
-                bins = defaultdict(set)
-
-                for name, pages in document_qs.values_list("name", "pages"):
-                    bins[pages or 0].add(name)
-
-                series_data = []
-                for pages, names in sorted(bins.items(), key=lambda t: t[0]):
-                    percentage = len(names) * 100.0 / (total_docs or 1)
-                    if pages is not None:
-                        series_data.append((pages, len(names)))
-                        table_data.append((pages, percentage, len(names), list(names)[:names_limit]))
-
-                chart_data.append({ "data": series_data })
-
-            elif stats_type == "words":
-                stats_title = "Number of words for each {}".format(doc_label)
-
-                bin_size = 500
-
-                bins = defaultdict(set)
-
-                for name, words in document_qs.values_list("name", "words"):
-                    bins[put_into_bin(words, bin_size)].add(name)
-
-                series_data = []
-                for (value, words), names in sorted(bins.items(), key=lambda t: t[0][0]):
-                    percentage = len(names) * 100.0 / (total_docs or 1)
-                    if words is not None:
-                        series_data.append((value, len(names)))
-
-                    table_data.append((words, percentage, len(names), list(names)[:names_limit]))
-
-                chart_data.append({ "data": series_data })
-
-            elif stats_type == "format":
-                stats_title = "Submission formats for each {}".format(doc_label)
-
-                bins = defaultdict(set)
-
-                # on new documents, we should have a Submission row with the file types
-                submission_types = {}
-
-                for doc_name, file_types in Submission.objects.values_list("draft", "file_types").order_by("submission_date", "id"):
-                    submission_types[doc_name] = file_types
-
-                doc_names_with_missing_types = {}
-                for doc_name, doc_type, rev in document_qs.values_list("name", "type_id", "rev"):
-                    types = submission_types.get(doc_name)
-                    if types:
-                        for dot_ext in types.split(","):
-                            bins[dot_ext.lstrip(".").upper()].add(doc_name)
-
-                    else:
-
-                        if doc_type == "rfc":
-                            filename = doc_name
-                        else:
-                            filename = doc_name + "-" + rev
-
-                        doc_names_with_missing_types[filename] = doc_name
-
-                # look up the remaining documents on disk
-                for filename in itertools.chain(os.listdir(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR), os.listdir(settings.RFC_PATH)):
-                    t = filename.split(".", 1)
-                    if len(t) != 2:
-                        continue
-
-                    basename, ext = t
-                    ext = ext.lower()
-                    if not any(ext==allowlisted_ext for allowlisted_ext in settings.DOCUMENT_FORMAT_ALLOWLIST):
-                        continue
-
-                    name = doc_names_with_missing_types.get(basename)
-
-                    if name:
-                        bins[ext.upper()].add(name)
-
-                series_data = []
-                for fmt, names in sorted(bins.items(), key=lambda t: t[0]):
-                    percentage = len(names) * 100.0 / (total_docs or 1)
-                    series_data.append((fmt, len(names)))
-
-                    table_data.append((fmt, percentage, len(names), list(names)[:names_limit]))
-
-                chart_data.append({ "data": series_data })
-
-            elif stats_type == "formlang":
-                stats_title = "Formal languages used for each {}".format(doc_label)
-
-                bins = defaultdict(set)
-
-                for name, formal_language_name in document_qs.values_list("name", "formal_languages__name"):
-                    bins[formal_language_name or ""].add(name)
-
-                series_data = []
-                for formal_language, names in sorted(bins.items(), key=lambda t: t[0]):
-                    percentage = len(names) * 100.0 / (total_docs or 1)
-                    if formal_language is not None:
-                        series_data.append((formal_language, len(names)))
-                        table_data.append((formal_language, percentage, len(names), list(names)[:names_limit]))
-
-                chart_data.append({ "data": series_data })
-
-        elif any(stats_type == t[0] for t in possible_author_stats_types):
-            person_filters = Q(documentauthor__document__type="draft")
-
-            # filter persons
-            rfc_state = State.objects.get(type="rfc", slug="published")
-            if document_type == "rfc":
-                person_filters &= Q(documentauthor__document__states=rfc_state)
-            elif document_type == "draft":
-                person_filters &= ~Q(documentauthor__document__states=rfc_state)
-
-            if from_time:
-                # this is actually faster than joining in the database,
-                # despite the round-trip back and forth
-                docs_within_time_constraint = set(Document.objects.filter(
-                    type="draft",
-                    docevent__time__gte=from_time,
-                    docevent__type__in=["published_rfc", "new_revision"],
-                ).values_list("pk"))
-
-                person_filters &= Q(documentauthor__document__in=docs_within_time_constraint)
-
-            person_qs = Person.objects.filter(person_filters)
-
-            if document_type == "rfc":
-                doc_label = "RFC"
-            elif document_type == "draft":
-                doc_label = "draft"
-            else:
-                doc_label = "document"
-
-            if stats_type == "author/documents":
-                stats_title = "Number of {}s per author".format(doc_label)
-
-                bins = defaultdict(set)
-
-                person_qs = Person.objects.filter(person_filters)
-
-                for name, document_count in person_qs.values_list("name").annotate(Count("documentauthor")):
-                    bins[document_count or 0].add(name)
-
-                total_persons = count_bins(bins)
-
-                series_data = []
-                for document_count, names in sorted(bins.items(), key=lambda t: t[0]):
-                    percentage = len(names) * 100.0 / (total_persons or 1)
-                    series_data.append((document_count, percentage))
-                    plain_names = sorted([ plain_name(n) for n in names ])
-                    table_data.append((document_count, percentage, len(plain_names), list(plain_names)[:names_limit]))
-
-                chart_data.append({ "data": series_data })
-
-            elif stats_type == "author/affiliation":
-                stats_title = "Number of {} authors per affiliation".format(doc_label)
-
-                bins = defaultdict(set)
-
-                person_qs = Person.objects.filter(person_filters)
-
-                # Since people don't write the affiliation names in the
-                # same way, and we don't want to go back and edit them
-                # either, we transform them here.
-
-                name_affiliation_set = {
-                    (name, affiliation)
-                    for name, affiliation in person_qs.values_list("name", "documentauthor__affiliation")
-                }
-
-                aliases = get_aliased_affiliations(affiliation for _, affiliation in name_affiliation_set)
-
-                for name, affiliation in name_affiliation_set:
-                    bins[aliases.get(affiliation, affiliation)].add(name)
-
-                prune_unknown_bin_with_known(bins)
-                total_persons = count_bins(bins)
-
-                series_data = []
-                for affiliation, names in sorted(bins.items(), key=lambda t: t[0].lower()):
-                    percentage = len(names) * 100.0 / (total_persons or 1)
-                    if affiliation:
-                        series_data.append((affiliation, len(names)))
-                    plain_names = sorted([ plain_name(n) for n in names ])
-                    table_data.append((affiliation, percentage, len(plain_names), list(plain_names)[:names_limit]))
-
-                series_data.sort(key=lambda t: t[1], reverse=True)
-                series_data = series_data[:30]
-
-                chart_data.append({ "data": series_data })
-
-                for alias, name in sorted(aliases.items(), key=lambda t: t[1]):
-                    alias_data.append((name, alias))
-
-            elif stats_type == "author/country":
-                stats_title = "Number of {} authors per country".format(doc_label)
-
-                bins = defaultdict(set)
-
-                person_qs = Person.objects.filter(person_filters)
-
-                # Since people don't write the country names in the
-                # same way, and we don't want to go back and edit them
-                # either, we transform them here.
-
-                name_country_set = {
-                    (name, country)
-                    for name, country in person_qs.values_list("name", "documentauthor__country")
-                }
-
-                aliases = get_aliased_countries(country for _, country in name_country_set)
-
-                countries = { c.name: c for c in CountryName.objects.all() }
-                eu_name = "EU"
-                eu_countries = { c for c in countries.values() if c.in_eu }
-
-                for name, country in name_country_set:
-                    country_name = aliases.get(country, country)
-                    bins[country_name].add(name)
-
-                    c = countries.get(country_name)
-                    if c and c.in_eu:
-                        bins[eu_name].add(name)
-
-                prune_unknown_bin_with_known(bins)
-                total_persons = count_bins(bins)
+    return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index"))
 
-                series_data = []
-                for country, names in sorted(bins.items(), key=lambda t: t[0].lower()):
-                    percentage = len(names) * 100.0 / (total_persons or 1)
-                    if country:
-                        series_data.append((country, len(names)))
-                    plain_names = sorted([ plain_name(n) for n in names ])
-                    table_data.append((country, percentage, len(plain_names), list(plain_names)[:names_limit]))
-
-                series_data.sort(key=lambda t: t[1], reverse=True)
-                series_data = series_data[:30]
-
-                chart_data.append({ "data": series_data })
-
-                for alias, country_name in aliases.items():
-                    alias_data.append((country_name, alias, countries.get(country_name)))
-
-                alias_data.sort()
-
-            elif stats_type == "author/continent":
-                stats_title = "Number of {} authors per continent".format(doc_label)
-
-                bins = defaultdict(set)
-
-                person_qs = Person.objects.filter(person_filters)
-
-                name_country_set = {
-                    (name, country)
-                    for name, country in person_qs.values_list("name", "documentauthor__country")
-                }
-
-                aliases = get_aliased_countries(country for _, country in name_country_set)
-
-                country_to_continent = dict(CountryName.objects.values_list("name", "continent__name"))
-
-                for name, country in name_country_set:
-                    country_name = aliases.get(country, country)
-                    continent_name = country_to_continent.get(country_name, "")
-                    bins[continent_name].add(name)
-
-                prune_unknown_bin_with_known(bins)
-                total_persons = count_bins(bins)
-
-                series_data = []
-                for continent, names in sorted(bins.items(), key=lambda t: t[0].lower()):
-                    percentage = len(names) * 100.0 / (total_persons or 1)
-                    if continent:
-                        series_data.append((continent, len(names)))
-                    plain_names = sorted([ plain_name(n) for n in names ])
-                    table_data.append((continent, percentage, len(plain_names), list(plain_names)[:names_limit]))
-
-                series_data.sort(key=lambda t: t[1], reverse=True)
-
-                chart_data.append({ "data": series_data })
-
-            elif stats_type == "author/citations":
-                stats_title = "Number of citations of {}s written by author".format(doc_label)
-
-                bins = defaultdict(set)
-
-                cite_relationships = list(DocRelationshipName.objects.filter(slug__in=['refnorm', 'refinfo', 'refunk', 'refold']))
-                person_filters &= Q(documentauthor__document__relateddocument__relationship__in=cite_relationships)
-
-                person_qs = Person.objects.filter(person_filters)
-
-                for name, citations in person_qs.values_list("name").annotate(Count("documentauthor__document__relateddocument")):
-                    bins[citations or 0].add(name)
-
-                total_persons = count_bins(bins)
-
-                series_data = []
-                for citations, names in sorted(bins.items(), key=lambda t: t[0], reverse=True):
-                    percentage = len(names) * 100.0 / (total_persons or 1)
-                    series_data.append((citations, percentage))
-                    plain_names = sorted([ plain_name(n) for n in names ])
-                    table_data.append((citations, percentage, len(plain_names), list(plain_names)[:names_limit]))
-
-                chart_data.append({ "data": sorted(series_data, key=lambda t: t[0]) })
-
-            elif stats_type == "author/hindex":
-                stats_title = "h-index for {}s written by author".format(doc_label)
-
-                bins = defaultdict(set)
-
-                cite_relationships = list(DocRelationshipName.objects.filter(slug__in=['refnorm', 'refinfo', 'refunk', 'refold']))
-                person_filters &= Q(documentauthor__document__relateddocument__relationship__in=cite_relationships)
-
-                person_qs = Person.objects.filter(person_filters)
-
-                values = person_qs.values_list("name", "documentauthor__document").annotate(Count("documentauthor__document__relateddocument"))
-                for name, ts in itertools.groupby(values.order_by("name"), key=lambda t: t[0]):
-                    h_index = compute_hirsch_index([citations for _, document, citations in ts])
-                    bins[h_index or 0].add(name)
-
-                total_persons = count_bins(bins)
-
-                series_data = []
-                for citations, names in sorted(bins.items(), key=lambda t: t[0], reverse=True):
-                    percentage = len(names) * 100.0 / (total_persons or 1)
-                    series_data.append((citations, percentage))
-                    plain_names = sorted([ plain_name(n) for n in names ])
-                    table_data.append((citations, percentage, len(plain_names), list(plain_names)[:names_limit]))
-
-                chart_data.append({ "data": sorted(series_data, key=lambda t: t[0]) })
-
-        elif any(stats_type == t[0] for t in possible_yearly_stats_types):
-
-            # filter persons
-            rfc_state = State.objects.get(type="rfc", slug="published")
-            if document_type == "rfc":
-                person_filters = Q(documentauthor__document__type="rfc")
-                person_filters &= Q(documentauthor__document__states=rfc_state)
-            elif document_type == "draft":
-                person_filters = Q(documentauthor__document__type="draft")
-                person_filters &= ~Q(documentauthor__document__states=rfc_state)
-            else:
-                person_filters = Q(documentauthor__document__type="rfc")
-                person_filters |= Q(documentauthor__document__type="draft")
-
-            doc_years = defaultdict(set)
-
-            draftevent_qs = DocEvent.objects.filter(
-                doc__type="draft",
-                type = "new_revision",
-            ).values_list("doc","time").order_by("doc")
-
-            for doc_id, time in draftevent_qs.iterator():
-                # RPC_TZINFO is used to match the timezone handling in Document.pub_date()
-                doc_years[doc_id].add(time.astimezone(RPC_TZINFO).year)
-
-            rfcevent_qs = (
-                DocEvent.objects.filter(doc__type="rfc", type="published_rfc")
-                .annotate(
-                    draft=Subquery(
-                        RelatedDocument.objects.filter(
-                            target=OuterRef("doc__pk"), relationship_id="became_rfc"
-                        ).values_list("source", flat=True)[:1]
-                    )
-                )
-                .values_list("doc", "time")
-                .order_by("doc")
-            )
-
-            for doc_id, time in rfcevent_qs.iterator():
-                doc_years[doc_id].add(time.astimezone(RPC_TZINFO).year)
-
-            person_qs = Person.objects.filter(person_filters)
-
-            if document_type == "rfc":
-                doc_label = "RFC"
-            elif document_type == "draft":
-                doc_label = "draft"
-            else:
-                doc_label = "document"
-
-            template_name = "yearly"
-
-            years_from = from_time.year if from_time else 1
-            years_to = timezone.now().year - 1
-
-
-            if stats_type == "yearly/affiliation":
-                stats_title = "Number of {} authors per affiliation over the years".format(doc_label)
-
-                person_qs = Person.objects.filter(person_filters)
-
-                name_affiliation_doc_set = {
-                    (name, affiliation, doc)
-                    for name, affiliation, doc in person_qs.values_list("name", "documentauthor__affiliation", "documentauthor__document")
-                }
-
-                aliases = get_aliased_affiliations(affiliation for _, affiliation, _ in name_affiliation_doc_set)
-
-                bins = defaultdict(set)
-                for name, affiliation, doc in name_affiliation_doc_set:
-                    a = aliases.get(affiliation, affiliation)
-                    if a:
-                        years = doc_years.get(doc)
-                        if years:
-                            for year in years:
-                                if years_from <= year <= years_to:
-                                    bins[(year, a)].add(name)
-
-                add_labeled_top_series_from_bins(chart_data, bins, limit=8)
-
-            elif stats_type == "yearly/country":
-                stats_title = "Number of {} authors per country over the years".format(doc_label)
-
-                person_qs = Person.objects.filter(person_filters)
-
-                name_country_doc_set = {
-                    (name, country, doc)
-                    for name, country, doc in person_qs.values_list("name", "documentauthor__country", "documentauthor__document")
-                }
-
-                aliases = get_aliased_countries(country for _, country, _ in name_country_doc_set)
-
-                countries = { c.name: c for c in CountryName.objects.all() }
-                eu_name = "EU"
-                eu_countries = { c for c in countries.values() if c.in_eu }
-
-                bins = defaultdict(set)
-
-                for name, country, doc in name_country_doc_set:
-                    country_name = aliases.get(country, country)
-                    c = countries.get(country_name)
-
-                    years = doc_years.get(doc)
-                    if country_name and years:
-                        for year in years:
-                            if years_from <= year <= years_to:
-                                bins[(year, country_name)].add(name)
-
-                                if c and c.in_eu:
-                                    bins[(year, eu_name)].add(name)
-
-                add_labeled_top_series_from_bins(chart_data, bins, limit=8)
-
-
-            elif stats_type == "yearly/continent":
-                stats_title = "Number of {} authors per continent".format(doc_label)
-
-                person_qs = Person.objects.filter(person_filters)
-
-                name_country_doc_set = {
-                    (name, country, doc)
-                    for name, country, doc in person_qs.values_list("name", "documentauthor__country", "documentauthor__document")
-                }
-
-                aliases = get_aliased_countries(country for _, country, _ in name_country_doc_set)
-
-                country_to_continent = dict(CountryName.objects.values_list("name", "continent__name"))
-
-                bins = defaultdict(set)
-
-                for name, country, doc in name_country_doc_set:
-                    country_name = aliases.get(country, country)
-                    continent_name = country_to_continent.get(country_name, "")
-
-                    if continent_name:
-                        years = doc_years.get(doc)
-                        if years:
-                            for year in years:
-                                if years_from <= year <= years_to:
-                                    bins[(year, continent_name)].add(name)
-
-                add_labeled_top_series_from_bins(chart_data, bins, limit=8)
-
-        data = {
-            "chart_data": mark_safe(json.dumps(chart_data)),
-            "table_data": table_data,
-            "stats_title": stats_title,
-            "possible_document_stats_types": possible_document_stats_types,
-            "possible_author_stats_types": possible_author_stats_types,
-            "possible_yearly_stats_types": possible_yearly_stats_types,
-            "stats_type": stats_type,
-            "possible_document_types": possible_document_types,
-            "document_type": document_type,
-            "possible_time_choices": possible_time_choices,
-            "time_choice": time_choice,
-            "doc_label": doc_label,
-            "bin_size": bin_size,
-            "show_aliases_url": build_document_stats_url(get_overrides={ "showaliases": "1" }),
-            "hide_aliases_url": build_document_stats_url(get_overrides={ "showaliases": None }),
-            "alias_data": alias_data,
-            "eu_countries": sorted(eu_countries or [], key=lambda c: c.name),
-            "content_template": "stats/document_stats_{}.html".format(template_name),
-        }
-        # Logs are full of these, but nobody is using them
-        # log("Cache miss for '%s'.  Data size: %sk" % (cache_key, len(str(data))/1000))
-        cache.set(cache_key, data, 24*60*60)
-    return render(request, "stats/document_stats.html", data)
 
 def known_countries_list(request, stats_type=None, acronym=None):
     countries = CountryName.objects.prefetch_related("countryalias_set")
@@ -774,252 +137,7 @@ def known_countries_list(request, stats_type=None, acronym=None):
     })
 
 def meeting_stats(request, num=None, stats_type=None):
-    meeting = None
-    if num is not None:
-        meeting = get_object_or_404(Meeting, number=num, type="ietf")
-
-    def build_meeting_stats_url(number=None, stats_type_override=Ellipsis, get_overrides=None):
-        if get_overrides is None:
-            get_overrides = {}
-        kwargs = {
-            "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override,
-        }
-
-        if number is not None:
-            kwargs["num"] = number
-
-        return urlreverse(meeting_stats, kwargs={ k: v for k, v in kwargs.items() if v is not None }) + generate_query_string(request.GET, get_overrides)
-
-    cache_key = ("stats:meeting_stats:%s:%s:%s" % (num, stats_type, slugify(request.META.get('QUERY_STRING',''))))[:228]
-    data = cache.get(cache_key)
-    if not data:
-        names_limit = settings.STATS_NAMES_LIMIT
-        # statistics types
-        if meeting:
-            possible_stats_types = add_url_to_choices([
-                ("country", "Country"),
-                ("continent", "Continent"),
-            ], lambda slug: build_meeting_stats_url(number=meeting.number, stats_type_override=slug))
-        else:
-            possible_stats_types = add_url_to_choices([
-                ("overview", "Overview"),
-                ("country", "Country"),
-                ("continent", "Continent"),
-            ], lambda slug: build_meeting_stats_url(number=None, stats_type_override=slug))
-
-        if not stats_type:
-            return HttpResponseRedirect(build_meeting_stats_url(number=num, stats_type_override=possible_stats_types[0][0]))
-
-        chart_data = []
-        piechart_data = []
-        table_data = []
-        stats_title = ""
-        template_name = stats_type
-        bin_size = 1
-        eu_countries = None
-
-        def get_country_mapping(attendees):
-            return {
-                alias.alias: alias.country
-                for alias in CountryAlias.objects.filter(alias__in=set(r.country_code for r in attendees)).select_related("country", "country__continent")
-                if alias.alias.isupper()
-            }
-
-        def reg_name(r):
-            return email.utils.formataddr(((r.first_name + " " + r.last_name).strip(), r.email))
-
-        if meeting and any(stats_type == t[0] for t in possible_stats_types):
-            attendees = MeetingRegistration.objects.filter(
-                meeting=meeting,
-                reg_type__in=['onsite', 'remote']
-            ).filter(
-                Q( attended=True) | Q( checkedin=True )
-            )
-
-            if stats_type == "country":
-                stats_title = "Number of attendees for {} {} per country".format(meeting.type.name, meeting.number)
-
-                bins = defaultdict(set)
-
-                country_mapping = get_country_mapping(attendees)
-
-                eu_name = "EU"
-                eu_countries = set(CountryName.objects.filter(in_eu=True))
-
-                for r in attendees:
-                    name = reg_name(r)
-                    c = country_mapping.get(r.country_code)
-                    bins[c.name if c else ""].add(name)
-
-                    if c and c.in_eu:
-                        bins[eu_name].add(name)
-
-                prune_unknown_bin_with_known(bins)
-                total_attendees = count_bins(bins)
-
-                series_data = []
-                for country, names in sorted(bins.items(), key=lambda t: t[0].lower()):
-                    percentage = len(names) * 100.0 / (total_attendees or 1)
-                    if country:
-                        series_data.append((country, len(names)))
-                    table_data.append((country, percentage, len(names), list(names)[:names_limit]))
-
-                    if country and country != eu_name:
-                        piechart_data.append({ "name": country, "y": percentage })
-
-                series_data.sort(key=lambda t: t[1], reverse=True)
-                series_data = series_data[:20]
-
-                piechart_data.sort(key=lambda d: d["y"], reverse=True)
-                pie_cut_off = 8
-                piechart_data = piechart_data[:pie_cut_off] + [{ "name": "Other", "y": sum(d["y"] for d in piechart_data[pie_cut_off:])}]
-
-                chart_data.append({ "data": series_data })
-
-            elif stats_type == "continent":
-                stats_title = "Number of attendees for {} {} per continent".format(meeting.type.name, meeting.number)
-
-                bins = defaultdict(set)
-
-                country_mapping = get_country_mapping(attendees)
-
-                for r in attendees:
-                    name = reg_name(r)
-                    c = country_mapping.get(r.country_code)
-                    bins[c.continent.name if c else ""].add(name)
-
-                prune_unknown_bin_with_known(bins)
-                total_attendees = count_bins(bins)
-
-                series_data = []
-                for continent, names in sorted(bins.items(), key=lambda t: t[0].lower()):
-                    percentage = len(names) * 100.0 / (total_attendees or 1)
-                    if continent:
-                        series_data.append((continent, len(names)))
-                    table_data.append((continent, percentage, len(names), list(names)[:names_limit]))
-
-                series_data.sort(key=lambda t: t[1], reverse=True)
-
-                chart_data.append({ "data": series_data })
-
-
-        elif not meeting and any(stats_type == t[0] for t in possible_stats_types):
-            template_name = "overview"
-
-            attendees = MeetingRegistration.objects.filter(
-                meeting__type="ietf",
-                attended=True,
-                reg_type__in=['onsite', 'remote']
-            ).filter(
-                Q( attended=True) | Q( checkedin=True )  
-            ).select_related('meeting')
-
-            if stats_type == "overview":
-                stats_title = "Number of attendees per meeting"
-
-                continents = {}
-                
-                meetings = Meeting.objects.filter(type='ietf', date__lte=date_today()).order_by('number')
-                for m in meetings:
-                    country = CountryName.objects.get(slug=m.country)
-                    continents[country.continent.name] = country.continent.name
-
-                bins = defaultdict(set)
-
-                for r in attendees:
-                    meeting_number = int(r.meeting.number)
-                    name = reg_name(r)
-                    bins[meeting_number].add(name)
-
-                series_data = {}
-                for continent in list(continents.keys()):
-                    series_data[continent] = []
-
-                for m in meetings:
-                    country = CountryName.objects.get(slug=m.country)
-                    url = build_meeting_stats_url(number=m.number,
-                                                  stats_type_override="country")
-                    for continent in list(continents.keys()):
-                        if continent == country.continent.name:
-                            d = {
-                                "name": "IETF {} - {}, {}".format(int(m.number), m.city, country),
-                                "x": int(m.number),
-                                "y": m.attendees,
-                                "date": m.date.strftime("%d %b %Y"),
-                                "url": url,
-                                }
-                        else:
-                            d = {
-                                "x": int(m.number),
-                                "y": 0,
-                                }
-                        series_data[continent].append(d)
-                    table_data.append((m, url,
-                                       m.attendees, country))
-
-                for continent in list(continents.keys()):
-#                    series_data[continent].sort(key=lambda t: t[0]["x"])
-                    chart_data.append( { "name": continent,
-                                         "data": series_data[continent] })
-                    
-                table_data.sort(key=lambda t: int(t[0].number), reverse=True)
-
-            elif stats_type == "country":
-                stats_title = "Number of attendees per country across meetings"
-
-                country_mapping = get_country_mapping(attendees)
-
-                eu_name = "EU"
-                eu_countries = set(CountryName.objects.filter(in_eu=True))
-
-                bins = defaultdict(set)
-
-                for r in attendees:
-                    meeting_number = int(r.meeting.number)
-                    name = reg_name(r)
-                    c = country_mapping.get(r.country_code)
-
-                    if c:
-                        bins[(meeting_number, c.name)].add(name)
-                        if c.in_eu:
-                            bins[(meeting_number, eu_name)].add(name)
-
-                add_labeled_top_series_from_bins(chart_data, bins, limit=8)
-
-
-            elif stats_type == "continent":
-                stats_title = "Number of attendees per continent across meetings"
-
-                country_mapping = get_country_mapping(attendees)
-
-                bins = defaultdict(set)
-
-                for r in attendees:
-                    meeting_number = int(r.meeting.number)
-                    name = reg_name(r)
-                    c = country_mapping.get(r.country_code)
-
-                    if c:
-                        bins[(meeting_number, c.continent.name)].add(name)
-
-                add_labeled_top_series_from_bins(chart_data, bins, limit=8)
-        data = {
-            "chart_data": mark_safe(json.dumps(chart_data)),
-            "piechart_data": mark_safe(json.dumps(piechart_data)),
-            "table_data": table_data,
-            "stats_title": stats_title,
-            "possible_stats_types": possible_stats_types,
-            "stats_type": stats_type,
-            "bin_size": bin_size,
-            "meeting": meeting,
-            "eu_countries": sorted(eu_countries or [], key=lambda c: c.name),
-            "content_template": "stats/meeting_stats_{}.html".format(template_name),
-        }
-        # Logs are full of these, but nobody is using them...
-        # log("Cache miss for '%s'.  Data size: %sk" % (cache_key, len(str(data))/1000))
-        cache.set(cache_key, data, 24*60*60)
-    #
-    return render(request, "stats/meeting_stats.html", data)
+    return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index"))
 
 
 @login_required
diff --git a/ietf/templates/stats/document_stats.html b/ietf/templates/stats/document_stats.html
deleted file mode 100644
index 4e66bed37e..0000000000
--- a/ietf/templates/stats/document_stats.html
+++ /dev/null
@@ -1,86 +0,0 @@
-{% extends "base.html" %}
-{% load origin %}
-{% load ietf_filters static %}
-{% block title %}{{ stats_title }}{% endblock %}
-{% block pagehead %}
-    <link rel="stylesheet" href="{% static "ietf/css/list.css" %}">
-    <link rel="stylesheet" href="{% static "ietf/css/highcharts.css" %}">
-{% endblock %}
-{% block content %}
-    {% origin %}
-    <h1>Internet-Draft and RFC statistics</h1>
-    <div class="row my-3">
-        <label class="fw-bold col-sm-2 col-form-label">Documents:</label>
-        <div class="btn-group col-sm-10">
-            {% for slug, label, url in possible_document_stats_types %}
-                <a class="btn btn-outline-primary
-                          {% if slug == stats_type %}
-                              active
-                          {% endif %}"
-                   href="{{ url }}">{{ label }}</a>
-            {% endfor %}
-        </div>
-    </div>
-    <div class="row my-3">
-        <label class="fw-bold col-sm-2 col-form-label">Authors:</label>
-        <div class="btn-group col-sm-10">
-            {% for slug, label, url in possible_author_stats_types %}
-                <a class="btn btn-outline-primary
-                          {% if slug == stats_type %}
-                              active
-                          {% endif %}"
-                   href="{{ url }}">{{ label }}</a>
-            {% endfor %}
-        </div>
-    </div>
-    <div class="row my-3">
-        <label class="fw-bold col-sm-2 col-form-label">Yearly:</label>
-        <div class="btn-group col-sm-10">
-            {% for slug, label, url in possible_yearly_stats_types %}
-                <a class="btn btn-outline-primary
-                          {% if slug == stats_type %}
-                              active
-                          {% endif %}"
-                   href="{{ url }}">{{ label }}</a>
-            {% endfor %}
-        </div>
-    </div>
-    <h2>Options</h2>
-    <div class="row my-3">
-        <label class="fw-bold col-sm-2 col-form-label">Document type:</label>
-        <div class="btn-group col-sm-10">
-            {% for slug, label, url in possible_document_types %}
-                <a class="btn btn-outline-primary
-                          {% if slug == document_type %}
-                              active
-                          {% endif %}"
-                   href="{{ url }}">{{ label }}</a>
-            {% endfor %}
-        </div>
-    </div>
-    <div class="row my-3">
-        <label class="fw-bold col-sm-2 col-form-label">Time:</label>
-        <div class="btn-group col-sm-10">
-            {% for slug, label, url in possible_time_choices %}
-                <a class="btn btn-outline-primary
-                          {% if slug == time_choice %}
-                              active
-                          {% endif %}"
-                   href="{{ url }}">{{ label }}</a>
-            {% endfor %}
-        </div>
-    </div>
-    <div class="alert alert-info my-3">
-        <b>Please Note:</b> The author information in the datatracker about RFCs
-        with numbers lower than about 1300 and Internet-Drafts from before 2001 is
-        unreliable and in many cases absent.  For this reason, statistics on these
-        pages does not show correct author stats for corpus selections that involve such
-        documents.
-    </div>
-    {% include content_template %}
-{% endblock %}
-{% block js %}
-    <script src="{% static 'ietf/js/highcharts.js' %}"></script>
-    <script src="{% static 'ietf/js/stats.js' %}"></script>
-    <script src="{% static "ietf/js/list.js" %}"></script>
-{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/stats/document_stats_author_affiliation.html b/ietf/templates/stats/document_stats_author_affiliation.html
deleted file mode 100644
index 9c798cb924..0000000000
--- a/ietf/templates/stats/document_stats_author_affiliation.html
+++ /dev/null
@@ -1,113 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'column'
-        },
-        plotOptions: {
-            column: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            type: "category",
-            title: {
-                text: 'Affiliation'
-            }
-        },
-        yAxis: {
-            title: {
-                text: 'Number of authors'
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.points[0].key + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-<h2>Data</h2>
-<table class="table table-sm table-striped tablesorter stats-data">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="affiliation">Affiliation</th>
-            <th scope="col" data-sort="percent">Percentage of authors</th>
-            <th scope="col" data-sort="count">Authors</th>
-        </tr>
-    </thead>
-    {% if table_data %}
-        <tbody>
-            {% for affiliation, percentage, count, names in table_data %}
-                <tr>
-                    <td>{{ affiliation|default:"(unknown)" }}</td>
-                    <td>{{ percentage|floatformat:2 }}%</td>
-                    <td>{% include "stats/includes/number_with_details_cell.html" %}</td>
-                </tr>
-            {% endfor %}
-        </tbody>
-    {% endif %}
-</table>
-<p>
-    The statistics are based entirely on the author affiliation
-    provided with each Internet-Draft. Since this may vary across documents, an
-    author may be counted with more than one affiliation, making the
-    total sum more than 100%.
-</p>
-<h2>Affiliation Aliases</h2>
-<p>
-    In generating the above statistics, some heuristics have been
-    applied to determine the affiliations of each author.
-</p>
-{% if request.GET.showaliases %}
-    <p>
-        <a href="{{ hide_aliases_url }}" class="btn btn-primary">Hide generated aliases</a>
-    </p>
-    {% if request.user.is_staff %}
-        <p>
-            Note: since you're an admin, you can
-            <a href="{% url "admin:stats_affiliationalias_add" %}">add an extra known alias</a>
-            or see the
-            <a href="{% url "admin:stats_affiliationalias_changelist" %}">existing known aliases</a>
-            and
-            <a href="{% url "admin:stats_affiliationignoredending_changelist" %}">generally ignored endings</a>.
-        </p>
-    {% endif %}
-    {% if alias_data %}
-        <table class="table table-sm table-striped tablesorter">
-            <thead>
-                <tr>
-                    <th scope="col" data-sort="affiliation">Affiliation</th>
-                    <th scope="col" data-sort="alias">Alias</th>
-                </tr>
-            </thead>
-            {% if alias_data %}
-                <tbody>
-                    {% for name, alias in alias_data %}
-                        <tr>
-                            <td>{{ name|default:"(unknown)" }}</td>
-                            <td>{{ alias }}</td>
-                        </tr>
-                    {% endfor %}
-                </tbody>
-            {% endif %}
-        </table>
-    {% endif %}
-{% else %}
-    <p>
-        <a href="{{ show_aliases_url }}" class="btn btn-primary">Show generated aliases</a>
-    </p>
-{% endif %}
\ No newline at end of file
diff --git a/ietf/templates/stats/document_stats_author_citations.html b/ietf/templates/stats/document_stats_author_citations.html
deleted file mode 100644
index ae89335fae..0000000000
--- a/ietf/templates/stats/document_stats_author_citations.html
+++ /dev/null
@@ -1,72 +0,0 @@
-{% load origin %}{% origin %}
-<div id="chart"></div>
-
-<script>
-    var chartConf = {
-        chart: {
-            type: 'area'
-        },
-        plotOptions: {
-            area: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            title: {
-                text: 'Number of citations of {{ doc_label }}s by author'
-            },
-            max: 500
-        },
-        yAxis: {
-            title: {
-                text: 'Percentage of authors'
-            },
-            labels: {
-                formatter: function () {
-                    return this.value + '%';
-                }
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.x + ' ' + (this.x == 1 ? "citation" : 'citations') + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y.toFixed(1) + '%';
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-
-<h2>Data</h2>
-
-<table class="table table-sm table-striped tablesorter stats-data">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="num">Citations</th>
-            <th scope="col" data-sort="percent">Percentage of authors</th>
-            <th scope="col" data-sort="authors">Authors</th>
-        </tr>
-    </thead>
-    {% if table_data %}
-        <tbody>
-            {% for citations, percentage, count, names in table_data %}
-                <tr>
-                    <td>{{ citations }}</td>
-                    <td>{{ percentage|floatformat:2 }}%</td>
-                    <td>{% include "stats/includes/number_with_details_cell.html" with content_limit=10 %}</td>
-                </tr>
-            {% endfor %}
-        </tbody>
-    {% endif %}
-</table>
-
-<p>Note that the citation counts do not exclude self-references.</p>
diff --git a/ietf/templates/stats/document_stats_author_continent.html b/ietf/templates/stats/document_stats_author_continent.html
deleted file mode 100644
index 5554ac341e..0000000000
--- a/ietf/templates/stats/document_stats_author_continent.html
+++ /dev/null
@@ -1,69 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'column'
-        },
-        plotOptions: {
-            column: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            type: "category",
-            title: {
-                text: 'Continent'
-            }
-        },
-        yAxis: {
-            title: {
-                text: 'Number of authors'
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.points[0].key + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-<h2>Data</h2>
-<table class="table table-sm table-striped tablesorter stats-data">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="continent">Continent</th>
-            <th scope="col" data-sort="percent">Percentage of authors</th>
-            <th scope="col" data-sort="count">Authors</th>
-        </tr>
-    </thead>
-    {% if table_data %}
-        <tbody>
-            {% for continent, percentage, count, names in table_data %}
-                <tr>
-                    <td>{{ continent|default:"(unknown)" }}</td>
-                    <td>{{ percentage|floatformat:2 }}%</td>
-                    <td>{% include "stats/includes/number_with_details_cell.html" %}</td>
-                </tr>
-            {% endfor %}
-        </tbody>
-    {% endif %}
-</table>
-<p>
-    The statistics are based entirely on the author addresses provided
-    with each Internet-Draft. Since this varies across documents, a traveling
-    author may be counted in more than country, making the total sum
-    more than 100%.
-</p>
\ No newline at end of file
diff --git a/ietf/templates/stats/document_stats_author_country.html b/ietf/templates/stats/document_stats_author_country.html
deleted file mode 100644
index 72299cc397..0000000000
--- a/ietf/templates/stats/document_stats_author_country.html
+++ /dev/null
@@ -1,136 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'column'
-        },
-        plotOptions: {
-            column: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            type: "category",
-            title: {
-                text: 'Country'
-            }
-        },
-        yAxis: {
-            title: {
-                text: 'Number of authors'
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.points[0].key + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-<h2>Data</h2>
-<table class="table table-sm table-striped tablesorter stats-data">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="country">Country</th>
-            <th scope="col" class="text-end" data-sort="percent">Percentage of authors</th>
-            <th scope="col" class="text-end" data-sort="authors">Authors</th>
-        </tr>
-    </thead>
-    {% if table_data %}
-        <tbody>
-            {% for country, percentage, count, names in table_data %}
-                <tr>
-                    <td>{{ country|default:"(unknown)" }}</td>
-                    <td class="text-end">{{ percentage|floatformat:2 }}%</td>
-                    <td class="text-end">{% include "stats/includes/number_with_details_cell.html" %}</td>
-                </tr>
-            {% endfor %}
-        </tbody>
-    {% endif %}
-</table>
-<p>
-    The statistics are based entirely on the author addresses provided
-    with each Internet-Draft. Since this varies across documents, a traveling
-    author may be counted in more than country, making the total sum
-    more than 100%.
-</p>
-<p>
-    In case no country information is found for an author in the time
-    period, the author is counted as (unknown).
-</p>
-<p>
-    EU (European Union) is not a country, but has been added for reference, as the sum of
-    all current EU member countries:
-    {% for c in eu_countries %}
-        {{ c.name }}{% if not forloop.last %},{% endif %}
-    {% endfor %}
-    .
-</p>
-<h2>Country Aliases</h2>
-<p>
-    In generating the above statistics, some heuristics have been
-    applied to figure out which country each author is from.
-</p>
-{% if request.GET.showaliases %}
-    <p>
-        <a href="{{ hide_aliases_url }}" class="btn btn-primary">Hide generated aliases</a>
-    </p>
-    {% if request.user.is_staff %}
-        <p class="alert alert-info my-3">
-            Note: since you're an admin, some extra links are visible. You
-            can either correct a document author entry directly in case the
-            information is obviously missing or add an alias if an unknown
-            <a href="{% url "admin:name_countryname_changelist" %}">country name</a>
-            is being used.
-        </p>
-    {% endif %}
-    {% if alias_data %}
-        <table class="table table-sm table-striped tablesorter">
-            <thead>
-                <th scope="col" data-sort="country">Country</th>
-                <th scope="col" data-sort="alias">Alias</th>
-            </thead>
-            {% if alias_data %}
-                <tbody>
-                {% for name, alias, country in alias_data %}
-                    <tr>
-                        <td>
-                            {% if country and request.user.is_staff %}
-                                <a href="{% url "admin:name_countryname_change" country.pk %}">{{ name|default:"(unknown)" }}</a>
-                            {% else %}
-                                {{ name|default:"(unknown)" }}
-                            {% endif %}
-                        </td>
-                        <td>
-                            {{ alias }}
-                            {% if request.user.is_staff and name != "EU" %}
-                                <a class="float-end btn btn-primary btn-sm"
-                                   href="{% url "admin:doc_documentauthor_changelist" %}?country={{ alias|urlencode }}">
-                                    Matching authors
-                                </a>
-                            {% endif %}
-                        </td>
-                    </tr>
-                {% endfor %}
-                </tbody>
-            {% endif %}
-        </table>
-    {% endif %}
-{% else %}
-    <p>
-        <a href="{{ show_aliases_url }}" class="btn btn-primary">Show generated aliases</a>
-    </p>
-{% endif %}
\ No newline at end of file
diff --git a/ietf/templates/stats/document_stats_author_documents.html b/ietf/templates/stats/document_stats_author_documents.html
deleted file mode 100644
index 28e33e6737..0000000000
--- a/ietf/templates/stats/document_stats_author_documents.html
+++ /dev/null
@@ -1,69 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'column'
-        },
-        plotOptions: {
-            column: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            tickInterval: 1,
-            title: {
-                text: 'Author of number of {{ doc_label }}s'
-            },
-            max: 20
-        },
-        yAxis: {
-            title: {
-                text: 'Percentage of authors'
-            },
-            labels: {
-                formatter: function () {
-                    return this.value + '%';
-                }
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.x + ' ' + (this.x == 1 ? "{{ doc_label }}" : '{{ doc_label }}s') + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y.toFixed(1) + '%';
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-<h2>Data</h2>
-<table class="table table-sm table-striped tablesorter stats-data">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="num">Documents</th>
-            <th scope="col" data-sort="percent">Percentage of authors</th>
-            <th scope="col" data-sort="count">Authors</th>
-        </tr>
-    </thead>
-    {% if table_data %}
-        <tbody>
-            {% for document_count, percentage, count, names in table_data %}
-                <tr>
-                    <td>{{ document_count }}</td>
-                    <td>{{ percentage|floatformat:2 }}%</td>
-                    <td>{% include "stats/includes/number_with_details_cell.html" with content_limit=10 %}</td>
-                </tr>
-            {% endfor %}
-        </tbody>
-    {% endif %}
-</table>
diff --git a/ietf/templates/stats/document_stats_author_hindex.html b/ietf/templates/stats/document_stats_author_hindex.html
deleted file mode 100644
index ab3215d355..0000000000
--- a/ietf/templates/stats/document_stats_author_hindex.html
+++ /dev/null
@@ -1,83 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'column'
-        },
-        plotOptions: {
-            column: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            tickInterval: 1,
-            title: {
-                text: 'h-index of {{ doc_label }}s by author'
-            }
-        },
-        yAxis: {
-            title: {
-                text: 'Percentage of authors'
-            },
-            labels: {
-                formatter: function () {
-                    return this.value + '%';
-                }
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + ' h-index ' + this.x + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y.toFixed(1) + '%';
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-<h2>Data</h2>
-<table class="table table-sm table-striped tablesorter stats-data">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="num">h-index</th>
-            <th scope="col" data-sort="percent">Percentage of authors</th>
-            <th scope="col" data-sort="Authors">Authors</th>
-        </tr>
-    </thead>
-    {% if table_data %}
-        <tbody>
-            {% for h_index, percentage, count, names in table_data %}
-                <tr>
-                    <td>{{ h_index }}</td>
-                    <td>{{ percentage|floatformat:2 }}%</td>
-                    <td>{% include "stats/includes/number_with_details_cell.html" with content_limit=25 %}</td>
-                </tr>
-            {% endfor %}
-        </tbody>
-    {% endif %}
-</table>
-<p>
-    Hirsch index or h-index is a
-    <a href="https://www.wikipedia.org/wiki/H-index">
-        measure of the
-        productivity and impact of the publications of an author
-    </a>.
-    An
-    author with an h-index of 5 has had 5 publications each cited at
-    least 5 times - to increase the index to 6, the 5 publications plus
-    1 more would have to have been cited at least 6 times, each. Thus a
-    high h-index requires many highly-cited publications.
-</p>
-<p>
-    Note that the h-index calculations do not exclude self-references.
-</p>
diff --git a/ietf/templates/stats/document_stats_authors.html b/ietf/templates/stats/document_stats_authors.html
deleted file mode 100644
index 5c1bbbdf4c..0000000000
--- a/ietf/templates/stats/document_stats_authors.html
+++ /dev/null
@@ -1,68 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'column'
-        },
-        plotOptions: {
-            column: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            tickInterval: 1,
-            title: {
-                text: 'Number of authors'
-            }
-        },
-        yAxis: {
-            title: {
-                text: 'Percentage of {{ doc_label }}s'
-            },
-            labels: {
-                formatter: function () {
-                    return this.value + '%';
-                }
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.x + ' ' + (this.x == 1 ? "author" : 'authors') + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y.toFixed(1) + '%';
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-<h3>Data</h3>
-<table class="table table-sm table-striped tablesorter stats-data">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="num">Authors</th>
-            <th scope="col" data-sort="percent">Percentage of {{ doc_label }}s</th>
-            <th scope="col" data-sort="count">{{ doc_label|capfirst }}s</th>
-        </tr>
-    </thead>
-    {% if table_data %}
-        <tbody>
-            {% for author_count, percentage, count, names in table_data %}
-                <tr>
-                    <td>{{ author_count }}</td>
-                    <td>{{ percentage|floatformat:2 }}%</td>
-                    <td>{% include "stats/includes/number_with_details_cell.html" %}</td>
-                </tr>
-            {% endfor %}
-        </tbody>
-    {% endif %}
-</table>
diff --git a/ietf/templates/stats/document_stats_format.html b/ietf/templates/stats/document_stats_format.html
deleted file mode 100644
index 32c25fe378..0000000000
--- a/ietf/templates/stats/document_stats_format.html
+++ /dev/null
@@ -1,63 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'column'
-        },
-        plotOptions: {
-            column: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            type: "category",
-            title: {
-                text: 'Format'
-            }
-        },
-        yAxis: {
-            title: {
-                text: 'Number of {{ doc_label }}s'
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.points[0].key + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-<h2>Data</h2>
-<table class="table table-sm table-striped tablesorter stats-data">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="format">Format</th>
-            <th scope="col" data-sort="percent">Percentage of {{ doc_label }}s</th>
-            <th scope="col" data-sort="label">{{ doc_label|capfirst }}s</th>
-        </tr>
-    </thead>
-    {% if table_data %}
-        <tbody>
-            {% for pages, percentage, count, names in table_data %}
-                <tr>
-                    <td>{{ pages }}</td>
-                    <td>{{ percentage|floatformat:2 }}%</td>
-                    <td>{% include "stats/includes/number_with_details_cell.html" %}</td>
-                </tr>
-            {% endfor %}
-        </tbody>
-    {% endif %}
-</table>
\ No newline at end of file
diff --git a/ietf/templates/stats/document_stats_formlang.html b/ietf/templates/stats/document_stats_formlang.html
deleted file mode 100644
index 217d79e3ef..0000000000
--- a/ietf/templates/stats/document_stats_formlang.html
+++ /dev/null
@@ -1,63 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'column'
-        },
-        plotOptions: {
-            column: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            type: "category",
-            title: {
-                text: 'Formal language'
-            }
-        },
-        yAxis: {
-            title: {
-                text: 'Number of {{ doc_label }}s'
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.points[0].key + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-<h2>Data</h2>
-<table class="table table-sm table-striped tablesorter stats-data">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="formal">Formal language</th>
-            <th scope="col" data-sort="percent">Percentage of {{ doc_label }}s</th>
-            <th scope="col" data-sort="count">{{ doc_label|capfirst }}s</th>
-        </tr>
-    </thead>
-    {% if table_data %}
-        <tbody>
-            {% for formal_language, percentage, count, names in table_data %}
-                <tr>
-                    <td>{{ formal_language }}</td>
-                    <td>{{ percentage|floatformat:2 }}%</td>
-                    <td>{% include "stats/includes/number_with_details_cell.html" %}</td>
-                </tr>
-            {% endfor %}
-        </tbody>
-    {% endif %}
-</table>
\ No newline at end of file
diff --git a/ietf/templates/stats/document_stats_pages.html b/ietf/templates/stats/document_stats_pages.html
deleted file mode 100644
index 73231b0e90..0000000000
--- a/ietf/templates/stats/document_stats_pages.html
+++ /dev/null
@@ -1,62 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'line'
-        },
-        plotOptions: {
-            line: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            title: {
-                text: 'Number of pages'
-            }
-        },
-        yAxis: {
-            title: {
-                text: 'Number of {{ doc_label }}s'
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.x + ' ' + (this.x == 1 ? "page" : 'pages') + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-<h2>Data</h2>
-<table class="table table-sm table-striped tablesorter stats-data">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="num">Pages</th>
-            <th scope="col" data-sort="percent">Percentage of {{ doc_label }}s</th>
-            <th scope="col" data-sort="count">{{ doc_label|capfirst }}s</th>
-        </tr>
-    </thead>
-    {% if table_data %}
-        <tbody>
-            {% for pages, percentage, count, names in table_data %}
-                <tr>
-                    <td>{{ pages }}</td>
-                    <td>{{ percentage|floatformat:2 }}%</td>
-                    <td>{% include "stats/includes/number_with_details_cell.html" %}</td>
-                </tr>
-            {% endfor %}
-        </tbody>
-    {% endif %}
-</table>
diff --git a/ietf/templates/stats/document_stats_words.html b/ietf/templates/stats/document_stats_words.html
deleted file mode 100644
index 4e8c15e937..0000000000
--- a/ietf/templates/stats/document_stats_words.html
+++ /dev/null
@@ -1,62 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'line'
-        },
-        plotOptions: {
-            line: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            title: {
-                text: 'Number of words'
-            }
-        },
-        yAxis: {
-            title: {
-                text: 'Number of {{ doc_label }}s'
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.x + ' - ' + (this.x + {{ bin_size }} - 1) + ' ' + (this.x == 1 ? "word" : 'words') + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-<h2>Data</h2>
-<table class="table table-sm table-striped tablesorter stats-data">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="num">Words</th>
-            <th scope="col" data-sort="percent">Percentage of {{ doc_label }}s</th>
-            <th scope="col" data-sort="count">{{ doc_label|capfirst }}s</th>
-        </tr>
-    </thead>
-    {% if table_data %}
-        <tbody>
-            {% for pages, percentage, count, names in table_data %}
-                <tr>
-                    <td>{{ pages }}</td>
-                    <td>{{ percentage|floatformat:2 }}%</td>
-                    <td>{% include "stats/includes/number_with_details_cell.html" %}</td>
-                </tr>
-            {% endfor %}
-        </tbody>
-    {% endif %}
-</table>
diff --git a/ietf/templates/stats/document_stats_yearly.html b/ietf/templates/stats/document_stats_yearly.html
deleted file mode 100644
index b819255ced..0000000000
--- a/ietf/templates/stats/document_stats_yearly.html
+++ /dev/null
@@ -1,52 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'line',
-        },
-        plotOptions: {
-            line: {
-                marker: {
-                    enabled: false
-                },
-                animation: false
-            }
-        },
-        legend: {
-            align: "right",
-            verticalAlign: "middle",
-            layout: "vertical",
-            enabled: true
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            tickInterval: 1,
-            title: {
-                text: 'Year'
-            }
-        },
-        yAxis: {
-            min: 0,
-            title: {
-                text: 'Authors active in year'
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.x + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + this.series.name + ': ' + this.y;
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
\ No newline at end of file
diff --git a/ietf/templates/stats/includes/number_with_details_cell.html b/ietf/templates/stats/includes/number_with_details_cell.html
deleted file mode 100644
index a5e88113ca..0000000000
--- a/ietf/templates/stats/includes/number_with_details_cell.html
+++ /dev/null
@@ -1,15 +0,0 @@
-{% load person_filters %}
-{% if content_limit and count <= content_limit %}
-    {% for n in names %}
-        {% with n|person_by_name as person %}
-            {% if person %}
-                {% person_link person %}
-            {% else %}
-                {{ n }}
-            {% endif %}
-            <br>
-        {% endwith %}
-    {% endfor %}
-{% else %}
-    {{ count }}
-{% endif %}
\ No newline at end of file
diff --git a/ietf/templates/stats/index.html b/ietf/templates/stats/index.html
index 1c5026013c..2000168525 100644
--- a/ietf/templates/stats/index.html
+++ b/ietf/templates/stats/index.html
@@ -10,15 +10,12 @@ <h1>
         Statistics on...
     </p>
     <ul>
-        <li>
-            <a href="{% url "ietf.stats.views.document_stats" %}">Internet-Drafts and RFCs (authors, countries, formats, ...)</a>
-        </li>
-        <li>
-            <a href="{% url "ietf.stats.views.meeting_stats" %}">Meeting attendance</a>
-        </li>
         <li>
             <a rel="nofollow" href="{% url "ietf.stats.views.review_stats" %}">Reviews of Internet-Drafts in review teams</a>
             (requires login)
         </li>
     </ul>
+    <p>
+        Statistics on meetings and authorship are not currently available.
+    </p>
 {% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/stats/meeting_stats.html b/ietf/templates/stats/meeting_stats.html
deleted file mode 100644
index 606caffde0..0000000000
--- a/ietf/templates/stats/meeting_stats.html
+++ /dev/null
@@ -1,35 +0,0 @@
-{% extends "base.html" %}
-{% load origin %}
-{% load ietf_filters static django_bootstrap5 %}
-{% block title %}{{ stats_title }}{% endblock %}
-{% block pagehead %}
-    <link rel="stylesheet" href="{% static 'ietf/css/list.css' %}">
-    <link rel="stylesheet" href="{% static "ietf/css/highcharts.css" %}">
-{% endblock %}
-{% block content %}
-    {% origin %}
-    <h1>Meeting Statistics</h1>
-    {% if meeting %}
-        <p>
-            <a href="{% url "ietf.stats.views.meeting_stats" %}">&laquo; Back to overview</a>
-        </p>
-    {% endif %}
-    <div class="row my-3">
-        <label class="fw-bold col-sm-2 col-form-label">Attendees:</label>
-        <div class="btn-group col-sm-10">
-            {% for slug, label, url in possible_stats_types %}
-                <a class="btn btn-outline-primary
-                          {% if slug == stats_type %}
-                              active
-                          {% endif %}"
-                   href="{{ url }}">{{ label }}</a>
-            {% endfor %}
-        </div>
-    </div>
-    <div class="document-stats">{% include content_template %}</div>
-{% endblock %}
-{% block js %}
-    <script src="{% static 'ietf/js/highcharts.js' %}"></script>
-    <script src="{% static 'ietf/js/stats.js' %}"></script>
-    <script src="{% static "ietf/js/list.js" %}"></script>
-{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/stats/meeting_stats_continent.html b/ietf/templates/stats/meeting_stats_continent.html
deleted file mode 100644
index 42ca03a409..0000000000
--- a/ietf/templates/stats/meeting_stats_continent.html
+++ /dev/null
@@ -1,61 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'column'
-        },
-        plotOptions: {
-            column: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            type: "category",
-            title: {
-                text: 'Continent'
-            }
-        },
-        yAxis: {
-            title: {
-                text: 'Number of attendees'
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.points[0].key + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-<h2>Data</h2>
-<table class="table table-sm table-striped stats-data tablesorter">
-    <thead>
-        <tr>
-            <th scope="col" data-sort="continent">Continent</th>
-            <th scope="col" data-sort="percent">Percentage of attendees</th>
-            <th scope="col" data-sort="count">Attendees</th>
-        </tr>
-    </thead>
-    <tbody>
-        {% for continent, percentage, count, names in table_data %}
-            <tr>
-                <td>{{ continent|default:"(unknown)" }}</td>
-                <td>{{ percentage|floatformat:2 }}%</td>
-                <td>{% include "stats/includes/number_with_details_cell.html" %}</td>
-            </tr>
-        {% endfor %}
-    </tbody>
-</table>
diff --git a/ietf/templates/stats/meeting_stats_country.html b/ietf/templates/stats/meeting_stats_country.html
deleted file mode 100644
index cebbad3c9f..0000000000
--- a/ietf/templates/stats/meeting_stats_country.html
+++ /dev/null
@@ -1,97 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart"></div>
-<script>
-    var chartConf = {
-        chart: {
-            type: 'column'
-        },
-        plotOptions: {
-            column: {
-                animation: false
-            }
-        },
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            type: "category",
-            title: {
-                text: 'Country'
-            }
-        },
-        yAxis: {
-            title: {
-                text: 'Number of attendees'
-            }
-        },
-        tooltip: {
-            formatter: function () {
-                var s = '<b>' + this.points[0].key + '</b>';
-
-                $.each(this.points, function () {
-                    s += '<br>' + chartConf.yAxis.title.text + ': ' + this.y;
-                });
-
-                return s;
-            },
-            shared: true
-        },
-        series: {{ chart_data }}
-    };
-</script>
-<div id="pie-chart"></div>
-<script>
-    var pieChartConf = {
-        chart: {
-            type: 'pie'
-        },
-        plotOptions: {
-            pie: {
-                animation: false,
-                dataLabels: {
-                    enabled: true,
-                    format: "{point.name}: {point.percentage:.1f}%"
-                },
-                enableMouseTracking: false
-            }
-        },
-        title: {
-            text: "Countries at IETF {{ meeting.number }}"
-        },
-        tooltip: {
-        },
-        series: [ {
-            name: "Countries",
-            colorByPoint: true,
-            data: {{ piechart_data }}
-        }]
-    };
-</script>
-<h2>Data</h2>
-<table class="table table-sm stats-data tablesorter">
-    <thead>
-        <tr>
-            <th scope="col">Country</th>
-            <th scope="col">Percentage of attendees</th>
-            <th scope="col">Attendees</th>
-        </tr>
-    </thead>
-    <tbody>
-        {% for country, percentage, count, names in table_data %}
-            <tr>
-                <td>{{ country|default:"(unknown)" }}</td>
-                <td>{{ percentage|floatformat:2 }}%</td>
-                <td>{% include "stats/includes/number_with_details_cell.html" %}</td>
-            </tr>
-        {% endfor %}
-    </tbody>
-</table>
-<p>
-    EU (European Union) is not a country, but has been added for reference, as the sum of
-    all current EU member countries:
-    {% for c in eu_countries %}
-        {{ c.name }}{% if not forloop.last %},{% endif %}
-    {% endfor %}
-    .
-</p>
\ No newline at end of file
diff --git a/ietf/templates/stats/meeting_stats_overview.html b/ietf/templates/stats/meeting_stats_overview.html
deleted file mode 100644
index 1136e458b8..0000000000
--- a/ietf/templates/stats/meeting_stats_overview.html
+++ /dev/null
@@ -1,160 +0,0 @@
-{% load origin %}
-{% origin %}
-<div id="chart" class="chart-{{ stats_type }}"></div>
-<script>
-    var chartConf = {
-
-        {% if stats_type == "overview" %}
-
-            chart: {
-                type: 'column',
-            },
-            colors: [ 'blue', 'red', 'cyan', 'orange', 'black',
-                'green', 'yellow' ],
-            plotOptions: {
-                series: {
-                    stacking: 'normal',
-                    point: {
-                        events: {
-                            click: function () {
-                                location.href = this.options.url;
-                            }
-                        }
-                    },
-                },
-                column: {
-                    marker: {
-                        enabled: false
-                    },
-                    animation: false
-                }
-            },
-            tooltip: {
-                formatter: function () {
-                    var s = '<b>' + this.point.name + '</b>';
-                    s += '<br>' + this.point.date;
-                    s += '<br>' + this.series.name + '<br> Attendees: ' + this.y;
-                    return s;
-                },
-                shared: false
-            },
-            legend: {
-                title: {
-                    text: "Continent of the venue",
-                },
-                borderWidth: 1,
-                align: "center",
-                verticalAlign: "bottom",
-                layout: "horizontal",
-                enabled: true
-            },
-
-
-        {% else %}
-
-            chart: {
-                type: 'line',
-            },
-            plotOptions: {
-                line: {
-                    marker: {
-                        enabled: false
-                    },
-                    animation: false
-                }
-            },
-            tooltip: {
-                formatter: function () {
-                    var s = '<b>' + "IETF " + this.x + '</b>';
-
-                    $.each(this.points, function () {
-                        s += '<br>' + this.series.name + ': ' + this.y;
-                    });
-
-                    return s;
-                },
-                shared: true
-            },
-            legend: {
-                align: "right",
-                verticalAlign: "middle",
-                layout: "vertical",
-                enabled: true
-            },
-
-
-        {% endif %}
-
-        title: {
-            text: '{{ stats_title|escapejs }}'
-        },
-        xAxis: {
-            tickInterval: 1,
-            title: {
-                text: 'Meeting'
-            }
-        },
-        yAxis: {
-            min: 0,
-            title: {
-                text: 'Attendees at meeting'
-            }
-        },
-        exporting: {
-            buttons: {
-                contextButton: {
-                    menuItems: [
-                        'printChart',
-                        'separator',
-                        'downloadPNG',
-                        'downloadJPEG',
-                        'downloadSVG',
-                        'separator',
-                        'downloadCSV'
-                    ]
-                }
-            }
-        },
-        series: {{ chart_data }}
-    };
-</script>
-{% if table_data %}
-    <h2>Data</h2>
-    <table class="table table-sm table-striped stats-data tablesorter">
-        <thead>
-            <tr>
-                <th scope="col" data-sort="num">Meeting</th>
-                <th scope="col" data-sort="date">Date</th>
-                <th scope="col" data-sort="city">City</th>
-                <th scope="col" data-sort="country">Country</th>
-                <th scope="col" data-sort="continent">Continent</th>
-                <th scope="col" data-sort="count">Attendees</th>
-            </tr>
-        </thead>
-        <tbody>
-            {% for meeting, url, count, country in table_data %}
-                <tr>
-                    {% if meeting.get_number > 71 %}
-                        <td>
-                            <a href="{{ url }}">{{ meeting.number }}</a>
-                        </td>
-                        <td>{{ meeting.date }}</td>
-                        <td>
-                            <a href="{{ url }}">{{ meeting.city }}</a>
-                        </td>
-                        <td>{{ country.name }}</td>
-                        <td>{{ country.continent }}</td>
-                        <td>{% include "stats/includes/number_with_details_cell.html" %}</td>
-                    {% else %}
-                        <td>{{ meeting.number }}</td>
-                        <td>{{ meeting.date }}</td>
-                        <td>{{ meeting.city }}</td>
-                        <td>{{ country.name }}</td>
-                        <td>{{ country.continent }}</td>
-                        <td>{% include "stats/includes/number_with_details_cell.html" %}</td>
-                    {% endif %}
-                </tr>
-            {% endfor %}
-        </tbody>
-    </table>
-{% endif %}

From 2ec7a71edfad24176034a1aa6a46ce66ddfb396b Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 9 Dec 2024 10:34:53 -0600
Subject: [PATCH 202/656] chore: remove unused setting from various
 settings_local templates (#8311)

---
 dev/deploy-to-container/settings_local.py | 1 -
 dev/diff/settings_local.py                | 1 -
 dev/tests/settings_local.py               | 1 -
 docker/configs/settings_local.py          | 1 -
 4 files changed, 4 deletions(-)

diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py
index ae698e20b6..07bf0a7511 100644
--- a/dev/deploy-to-container/settings_local.py
+++ b/dev/deploy-to-container/settings_local.py
@@ -40,7 +40,6 @@
 
 SUBMIT_YANG_CATALOG_MODEL_DIR = '/assets/ietf-ftp/yang/catalogmod/'
 SUBMIT_YANG_DRAFT_MODEL_DIR = '/assets/ietf-ftp/yang/draftmod/'
-SUBMIT_YANG_INVAL_MODEL_DIR = '/assets/ietf-ftp/yang/invalmod/'
 SUBMIT_YANG_IANA_MODEL_DIR = '/assets/ietf-ftp/yang/ianamod/'
 SUBMIT_YANG_RFC_MODEL_DIR   = '/assets/ietf-ftp/yang/rfcmod/'
 
diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py
index 774c7797cf..6bcee46b61 100644
--- a/dev/diff/settings_local.py
+++ b/dev/diff/settings_local.py
@@ -37,7 +37,6 @@
 
 SUBMIT_YANG_CATALOG_MODEL_DIR = '/assets/ietf-ftp/yang/catalogmod/'
 SUBMIT_YANG_DRAFT_MODEL_DIR = '/assets/ietf-ftp/yang/draftmod/'
-SUBMIT_YANG_INVAL_MODEL_DIR = '/assets/ietf-ftp/yang/invalmod/'
 SUBMIT_YANG_IANA_MODEL_DIR = '/assets/ietf-ftp/yang/ianamod/'
 SUBMIT_YANG_RFC_MODEL_DIR   = '/assets/ietf-ftp/yang/rfcmod/'
 
diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py
index 20941359d4..afadb3760b 100644
--- a/dev/tests/settings_local.py
+++ b/dev/tests/settings_local.py
@@ -36,7 +36,6 @@
 
 SUBMIT_YANG_CATALOG_MODEL_DIR = '/assets/ietf-ftp/yang/catalogmod/'
 SUBMIT_YANG_DRAFT_MODEL_DIR = '/assets/ietf-ftp/yang/draftmod/'
-SUBMIT_YANG_INVAL_MODEL_DIR = '/assets/ietf-ftp/yang/invalmod/'
 SUBMIT_YANG_IANA_MODEL_DIR = '/assets/ietf-ftp/yang/ianamod/'
 SUBMIT_YANG_RFC_MODEL_DIR   = '/assets/ietf-ftp/yang/rfcmod/'
 
diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py
index 5d9859c19b..a1c19c80cf 100644
--- a/docker/configs/settings_local.py
+++ b/docker/configs/settings_local.py
@@ -26,7 +26,6 @@
 
 SUBMIT_YANG_CATALOG_MODEL_DIR = '/assets/ietf-ftp/yang/catalogmod/'
 SUBMIT_YANG_DRAFT_MODEL_DIR = '/assets/ietf-ftp/yang/draftmod/'
-SUBMIT_YANG_INVAL_MODEL_DIR = '/assets/ietf-ftp/yang/invalmod/'
 SUBMIT_YANG_IANA_MODEL_DIR = '/assets/ietf-ftp/yang/ianamod/'
 SUBMIT_YANG_RFC_MODEL_DIR   = '/assets/ietf-ftp/yang/rfcmod/'
 

From 6f1c308ab3142a8f52df6d417767cb583d38a957 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 9 Dec 2024 14:56:09 -0400
Subject: [PATCH 203/656] chore: drop unused cf-connecting-ipv6 header (#8319)

Only used in certain configurations of Pseudo IPv4.
---
 ietf/utils/jsonlogger.py | 1 -
 k8s/nginx-logging.conf   | 1 -
 2 files changed, 2 deletions(-)

diff --git a/ietf/utils/jsonlogger.py b/ietf/utils/jsonlogger.py
index 6502cab0cb..1fc453ad9e 100644
--- a/ietf/utils/jsonlogger.py
+++ b/ietf/utils/jsonlogger.py
@@ -29,7 +29,6 @@ def add_fields(self, log_record, record, message_dict):
         log_record.setdefault("x_forwarded_for", record.args["{x-forwarded-for}i"])
         log_record.setdefault("x_forwarded_proto", record.args["{x-forwarded-proto}i"])
         log_record.setdefault("cf_connecting_ip", record.args["{cf-connecting-ip}i"])
-        log_record.setdefault("cf_connecting_ipv6", record.args["{cf-connecting-ipv6}i"])
         log_record.setdefault("cf_ray", record.args["{cf-ray}i"])
         log_record.setdefault("asn", record.args["{x-ip-src-asnum}i"])
         log_record.setdefault("is_authenticated", record.args["{x-datatracker-is-authenticated}o"])
diff --git a/k8s/nginx-logging.conf b/k8s/nginx-logging.conf
index 0bc7deca81..673d7a29ab 100644
--- a/k8s/nginx-logging.conf
+++ b/k8s/nginx-logging.conf
@@ -17,7 +17,6 @@ log_format  ietfjson  escape=json
     '"x_forwarded_for":"$${keepempty}http_x_forwarded_for",'
     '"x_forwarded_proto":"$${keepempty}http_x_forwarded_proto",'
     '"cf_connecting_ip":"$${keepempty}http_cf_connecting_ip",'
-    '"cf_connecting_ipv6":"$${keepempty}http_cf_connecting_ipv6",'
     '"cf_ray":"$${keepempty}http_cf_ray",'
     '"asn":"$${keepempty}http_x_ip_src_asnum"'
   '}';

From 8e325829a36515344800ef64b850f1a5af2f5160 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 10 Dec 2024 09:57:08 -0600
Subject: [PATCH 204/656] chore: pin django-oidc-provider until we can adapt to
 changes in 0.8.3 (#8320)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index f974113d8f..66397091ad 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,7 +17,7 @@ django-csp>=3.7
 django-cors-headers>=3.11.0
 django-debug-toolbar>=3.2.4
 django-markup>=1.5    # Limited use - need to reconcile against direct use of markdown
-django-oidc-provider>=0.8.1    # 0.8 dropped Django 2 support
+django-oidc-provider==0.8.2    # 0.8.3 changes logout flow and claim return
 django-referrer-policy>=1.0
 django-simple-history>=3.0.0
 django-stubs>=4.2.7,<5    # The django-stubs version used determines the the mypy version indicated below

From 6b77807c05145230c337809322ffa96606c140fd Mon Sep 17 00:00:00 2001
From: rjsparks <rjsparks@users.noreply.github.com>
Date: Tue, 10 Dec 2024 16:08:48 +0000
Subject: [PATCH 205/656] ci: update base image target version to 20241210T1557

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 7af27e7d13..bab7c1fab3 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20241127T2054
+FROM ghcr.io/ietf-tools/datatracker-app-base:20241210T1557
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index e4b05ed700..bca669bb51 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20241127T2054
+20241210T1557

From 9b372a31b4c70a031bb20a1cb2ee8189551d01ce Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 12 Dec 2024 13:40:49 -0400
Subject: [PATCH 206/656] chore: update import for python-json-logger (#8330)

The "jsonlogger" module became "json" in 3.1.0
---
 ietf/utils/jsonlogger.py | 4 ++--
 requirements.txt         | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ietf/utils/jsonlogger.py b/ietf/utils/jsonlogger.py
index 1fc453ad9e..589132977d 100644
--- a/ietf/utils/jsonlogger.py
+++ b/ietf/utils/jsonlogger.py
@@ -1,9 +1,9 @@
 # Copyright The IETF Trust 2024, All Rights Reserved
-from pythonjsonlogger import jsonlogger
+from pythonjsonlogger.json import JsonFormatter
 import time
 
 
-class DatatrackerJsonFormatter(jsonlogger.JsonFormatter):
+class DatatrackerJsonFormatter(JsonFormatter):
     converter = time.gmtime  # use UTC
     default_msec_format = "%s.%03d"  # '.' instead of ','
 
diff --git a/requirements.txt b/requirements.txt
index 66397091ad..ec5fc60b5f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -59,7 +59,7 @@ pyopenssl>=22.0.0    # Used by urllib3.contrib, which is used by PyQuery but not
 pyquery>=1.4.3
 python-dateutil>=2.8.2
 types-python-dateutil>=2.8.2
-python-json-logger>=2.0.7
+python-json-logger>=3.1.0
 python-magic==0.4.18    # Versions beyond the yanked .19 and .20 introduce form failures
 pymemcache>=4.0.0  # for django.core.cache.backends.memcached.PyMemcacheCache
 python-mimeparse>=1.6    # from TastyPie

From a2f27d3d516e6914d947f013198bba2f19ece402 Mon Sep 17 00:00:00 2001
From: rjsparks <rjsparks@users.noreply.github.com>
Date: Thu, 12 Dec 2024 17:53:09 +0000
Subject: [PATCH 207/656] ci: update base image target version to 20241212T1741

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index bab7c1fab3..a923bf693f 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20241210T1557
+FROM ghcr.io/ietf-tools/datatracker-app-base:20241212T1741
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index bca669bb51..b5d33714f2 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20241210T1557
+20241212T1741

From 70ab7112165ffa1e686649311bb14b51721d6f77 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 12 Dec 2024 18:48:44 -0400
Subject: [PATCH 208/656] refactor: separate concerns / rename notify_events
 (#8328)

* refactor: separate signal receiver from work

* test: split test to match code structure

* test: fix test

* refactor: reorg signals in community app
---
 ietf/community/apps.py                        | 12 ++++
 ietf/community/models.py                      | 35 +---------
 ietf/community/signals.py                     | 44 ++++++++++++
 ietf/community/tests.py                       | 67 ++++++++++---------
 ietf/utils/management/commands/loadrelated.py |  4 +-
 5 files changed, 96 insertions(+), 66 deletions(-)
 create mode 100644 ietf/community/apps.py
 create mode 100644 ietf/community/signals.py

diff --git a/ietf/community/apps.py b/ietf/community/apps.py
new file mode 100644
index 0000000000..ab0a6d6054
--- /dev/null
+++ b/ietf/community/apps.py
@@ -0,0 +1,12 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+
+from django.apps import AppConfig
+
+
+class CommunityConfig(AppConfig):
+    name = "ietf.community"
+
+    def ready(self):
+        """Initialize the app after the registry is populated"""
+        # implicitly connects @receiver-decorated signals
+        from . import signals  # pyflakes: ignore
diff --git a/ietf/community/models.py b/ietf/community/models.py
index 0407730107..6945918f9a 100644
--- a/ietf/community/models.py
+++ b/ietf/community/models.py
@@ -1,19 +1,14 @@
 # Copyright The IETF Trust 2012-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-
-from django.conf import settings
-from django.db import models, transaction
-from django.db.models import signals
+from django.db import models
 from django.urls import reverse as urlreverse
 
-from ietf.doc.models import Document, DocEvent, State
+from ietf.doc.models import Document, State
 from ietf.group.models import Group
 from ietf.person.models import Person, Email
 from ietf.utils.models import ForeignKey
 
-from .tasks import notify_event_to_subscribers_task
-
 
 class CommunityList(models.Model):
     person = ForeignKey(Person, blank=True, null=True)
@@ -98,29 +93,3 @@ class EmailSubscription(models.Model):
 
     def __str__(self):
         return "%s to %s (%s changes)" % (self.email, self.community_list, self.notify_on)
-
-
-def notify_events(sender, instance, **kwargs):
-    if not isinstance(instance, DocEvent):
-        return
-
-    if not kwargs.get("created", False):
-        return  # only notify on creation
-
-    if instance.doc.type_id != 'draft':
-        return
-
-    if getattr(instance, "skip_community_list_notification", False):
-        return
-    
-    # kludge alert: queuing a celery task in response to a signal can cause unexpected attempts to
-    # start a Celery task during tests. To prevent this, don't queue a celery task if we're running
-    # tests.
-    if settings.SERVER_MODE != "test":
-        # Wrap in on_commit in case a transaction is open
-        transaction.on_commit(
-            lambda: notify_event_to_subscribers_task.delay(event_id=instance.pk)
-        )
-
-
-signals.post_save.connect(notify_events)
diff --git a/ietf/community/signals.py b/ietf/community/signals.py
new file mode 100644
index 0000000000..20ee761129
--- /dev/null
+++ b/ietf/community/signals.py
@@ -0,0 +1,44 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+
+from django.conf import settings
+from django.db import transaction
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+
+from ietf.doc.models import DocEvent
+from .tasks import notify_event_to_subscribers_task
+
+
+def notify_of_event(event: DocEvent):
+    """Send subscriber notification emails for a 'draft'-related DocEvent
+
+    If the event is attached to a draft of type 'doc', queues a task to send notification emails to
+    community list subscribers. No emails will be sent when SERVER_MODE is 'test'.
+    """
+    if event.doc.type_id != "draft":
+        return
+
+    if getattr(event, "skip_community_list_notification", False):
+        return
+
+    # kludge alert: queuing a celery task in response to a signal can cause unexpected attempts to
+    # start a Celery task during tests. To prevent this, don't queue a celery task if we're running
+    # tests.
+    if settings.SERVER_MODE != "test":
+        # Wrap in on_commit in case a transaction is open
+        transaction.on_commit(
+            lambda: notify_event_to_subscribers_task.delay(event_id=event.pk)
+        )
+
+
+# dispatch_uid ensures only a single signal receiver binding is made
+@receiver(post_save, dispatch_uid="notify_of_events_receiver_uid")
+def notify_of_events_receiver(sender, instance, **kwargs):
+    """Call notify_of_event after saving a new DocEvent"""
+    if not isinstance(instance, DocEvent):
+        return
+
+    if not kwargs.get("created", False):
+        return  # only notify on creation
+
+    notify_of_event(instance)
diff --git a/ietf/community/tests.py b/ietf/community/tests.py
index 743242f11b..9bd7789958 100644
--- a/ietf/community/tests.py
+++ b/ietf/community/tests.py
@@ -1,7 +1,6 @@
 # Copyright The IETF Trust 2016-2023, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-
 import mock
 from pyquery import PyQuery
 
@@ -11,6 +10,7 @@
 import debug                            # pyflakes:ignore
 
 from ietf.community.models import CommunityList, SearchRule, EmailSubscription
+from ietf.community.signals import notify_of_event
 from ietf.community.utils import docs_matching_community_list_rule, community_list_rules_matching_doc
 from ietf.community.utils import reset_name_contains_index_for_rule, notify_event_to_subscribers
 from ietf.community.tasks import notify_event_to_subscribers_task
@@ -431,53 +431,58 @@ def test_subscription_for_group(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
 
-    # Mock out the on_commit call so we can tell whether the task was actually queued
-    @mock.patch("ietf.submit.views.transaction.on_commit", side_effect=lambda x: x())
-    @mock.patch("ietf.community.models.notify_event_to_subscribers_task")
-    def test_notification_signal_receiver(self, mock_notify_task, mock_on_commit):
-        """Saving a DocEvent should notify subscribers
+    @mock.patch("ietf.community.signals.notify_of_event")
+    def test_notification_signal_receiver(self, mock_notify_of_event):
+        """Saving a newly created DocEvent should notify subscribers
         
-        This implicitly tests that notify_events is hooked up to the post_save signal.
+        This implicitly tests that notify_of_event_receiver is hooked up to the post_save signal.
         """
         # Arbitrary model that's not a DocEvent
-        person = PersonFactory()
-        mock_notify_task.reset_mock()  # clear any calls that resulted from the factories
-        # be careful overriding SERVER_MODE - we do it here because the method
-        # under test does not make this call when in "test" mode
-        with override_settings(SERVER_MODE="not-test"):
-            person.save()
-        self.assertFalse(mock_notify_task.delay.called)
-        
+        person = PersonFactory.build()  # builds but does not save...
+        mock_notify_of_event.reset_mock()  # clear any calls that resulted from the factories
+        person.save()
+        self.assertFalse(mock_notify_of_event.called)
+    
         # build a DocEvent that is not yet persisted
         doc = DocumentFactory()
-        d = DocEventFactory.build(by=person, doc=doc)
-        # mock_notify_task.reset_mock()  # clear any calls that resulted from the factories
+        event = DocEventFactory.build(by=person, doc=doc)  # builds but does not save...
+        mock_notify_of_event.reset_mock()  # clear any calls that resulted from the factories
+        event.save()
+        self.assertEqual(mock_notify_of_event.call_count, 1, "notify_task should be run on creation of DocEvent")
+        self.assertEqual(mock_notify_of_event.call_args, mock.call(event))
+
+        # save the existing DocEvent and see that no notification is sent    
+        mock_notify_of_event.reset_mock()
+        event.save()
+        self.assertFalse(mock_notify_of_event.called, "notify_task should not be run save of on existing DocEvent")
+
+    # Mock out the on_commit call so we can tell whether the task was actually queued
+    @mock.patch("ietf.submit.views.transaction.on_commit", side_effect=lambda x: x())
+    @mock.patch("ietf.community.signals.notify_event_to_subscribers_task")
+    def test_notify_of_event(self, mock_notify_task, mock_on_commit):
+        """The community notification task should be called as intended"""
+        person = PersonFactory()  # builds but does not save...
+        doc = DocumentFactory()
+        event = DocEventFactory(by=person, doc=doc)
         # be careful overriding SERVER_MODE - we do it here because the method
         # under test does not make this call when in "test" mode
         with override_settings(SERVER_MODE="not-test"):
-            d.save()
-        self.assertEqual(mock_notify_task.delay.call_count, 1, "notify_task should be run on creation of DocEvent")
-        self.assertEqual(mock_notify_task.delay.call_args, mock.call(event_id = d.pk))
-        
-        mock_notify_task.reset_mock()
-        with override_settings(SERVER_MODE="not-test"):
-            d.save()
-        self.assertFalse(mock_notify_task.delay.called, "notify_task should not be run save of on existing DocEvent")
-        
+            notify_of_event(event)
+        self.assertTrue(mock_notify_task.delay.called, "notify_task should run for a DocEvent on a draft")
         mock_notify_task.reset_mock()
-        d = DocEventFactory.build(by=person, doc=doc)
-        d.skip_community_list_notification = True
+
+        event.skip_community_list_notification = True
         # be careful overriding SERVER_MODE - we do it here because the method
         # under test does not make this call when in "test" mode
         with override_settings(SERVER_MODE="not-test"):
-            d.save()
+            notify_of_event(event)
         self.assertFalse(mock_notify_task.delay.called, "notify_task should not run when skip_community_list_notification is set")
 
-        d = DocEventFactory.build(by=person, doc=DocumentFactory(type_id="rfc"))
+        event = DocEventFactory.build(by=person, doc=DocumentFactory(type_id="rfc"))
         # be careful overriding SERVER_MODE - we do it here because the method
         # under test does not make this call when in "test" mode
         with override_settings(SERVER_MODE="not-test"):
-            d.save()
+            notify_of_event(event)
         self.assertFalse(mock_notify_task.delay.called, "notify_task should not run on a document with type 'rfc'")
 
     @mock.patch("ietf.utils.mail.send_mail_text")
diff --git a/ietf/utils/management/commands/loadrelated.py b/ietf/utils/management/commands/loadrelated.py
index da9d00d5dc..d8ae19dc77 100644
--- a/ietf/utils/management/commands/loadrelated.py
+++ b/ietf/utils/management/commands/loadrelated.py
@@ -23,7 +23,7 @@
 
 import debug                            # pyflakes:ignore
 
-from ietf.community.models import notify_events
+from ietf.community.signals import notify_of_events_receiver
 
 class Command(loaddata.Command):
     help = ("""
@@ -62,7 +62,7 @@ def handle(self, *args, **options):
         #
         self.serialization_formats = serializers.get_public_serializer_formats()
         #
-        post_save.disconnect(notify_events)
+        post_save.disconnect(notify_of_events_receiver())
         #
         connection = connections[self.using]
         self.fixture_count = 0

From c747e972017142c81af5162207155ca31df7aeba Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 13 Dec 2024 11:48:19 -0600
Subject: [PATCH 209/656] fix: keep draft-iesg state on expiration. Update
 action holders. (#8321)

* fix: keep draft-iesg state on expiration. Update action holders

* feat: task to repair docs in dead because expiry

* fix: restore all to-date flows through update_action_holders

* fix: Fetch the System user following more regular conventions

* fix: better signal test
---
 ietf/doc/expire.py      | 62 +++++++++++++++++++++--------
 ietf/doc/tasks.py       |  6 +++
 ietf/doc/tests_draft.py | 86 ++++++++++++++++++++++++++++++++++++++---
 ietf/doc/tests_tasks.py |  7 ++++
 ietf/doc/utils.py       | 82 ++++++++++++++++++++++-----------------
 ietf/doc/views_draft.py |  3 +-
 6 files changed, 186 insertions(+), 60 deletions(-)

diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py
index 681ca963a3..0581c9a03f 100644
--- a/ietf/doc/expire.py
+++ b/ietf/doc/expire.py
@@ -3,6 +3,8 @@
 # expiry of Internet-Drafts
 
 
+import debug    # pyflakes:ignore
+
 from django.conf import settings
 from django.utils import timezone
 
@@ -11,12 +13,12 @@
 
 from typing import List, Optional      # pyflakes:ignore
 
+from ietf.doc.utils import new_state_change_event, update_action_holders
 from ietf.utils import log
 from ietf.utils.mail import send_mail
-from ietf.doc.models import Document, DocEvent, State, IESG_SUBSTATE_TAGS
+from ietf.doc.models import Document, DocEvent, State, StateDocEvent
 from ietf.person.models import Person 
 from ietf.meeting.models import Meeting
-from ietf.doc.utils import add_state_change_event, update_action_holders
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.utils.timezone import date_today, datetime_today, DEADLINE_TZINFO
 
@@ -161,24 +163,11 @@ def expire_draft(doc):
 
     events = []
 
-    # change the state
-    if doc.latest_event(type='started_iesg_process'):
-        new_state = State.objects.get(used=True, type="draft-iesg", slug="dead")
-        prev_state = doc.get_state(new_state.type_id)
-        prev_tags = doc.tags.filter(slug__in=IESG_SUBSTATE_TAGS)
-        if new_state != prev_state:
-            doc.set_state(new_state)
-            doc.tags.remove(*prev_tags)
-            e = add_state_change_event(doc, system, prev_state, new_state, prev_tags=prev_tags, new_tags=[])
-            if e:
-                events.append(e)
-            e = update_action_holders(doc, prev_state, new_state, prev_tags=prev_tags, new_tags=[])
-            if e:
-                events.append(e)
-
     events.append(DocEvent.objects.create(doc=doc, rev=doc.rev, by=system, type="expired_document", desc="Document has expired"))
 
+    prev_draft_state=doc.get_state("draft")
     doc.set_state(State.objects.get(used=True, type="draft", slug="expired"))
+    events.append(update_action_holders(doc, prev_draft_state, doc.get_state("draft"),[],[]))
     doc.save_with_history(events)
 
 def clean_up_draft_files():
@@ -238,3 +227,42 @@ def move_file_to(subdir):
         except Document.DoesNotExist:
             # All uses of this past 2014 seem related to major system failures.
             move_file_to("unknown_ids")
+
+
+def repair_dead_on_expire():
+    by = Person.objects.get(name="(System)")
+    id_exists = State.objects.get(type="draft-iesg", slug="idexists")
+    dead = State.objects.get(type="draft-iesg", slug="dead")
+    dead_drafts = Document.objects.filter(
+        states__type="draft-iesg", states__slug="dead", type_id="draft"
+    )
+    for d in dead_drafts:
+        dead_event = d.latest_event(
+            StateDocEvent, state_type="draft-iesg", state__slug="dead"
+        )
+        if dead_event is not None:
+            if d.docevent_set.filter(type="expired_document").exists():
+                closest_expiry = min(
+                    [
+                        abs(e.time - dead_event.time)
+                        for e in d.docevent_set.filter(type="expired_document")
+                    ]
+                )
+                if closest_expiry.total_seconds() < 60:
+                    d.set_state(id_exists)
+                    events = []
+                    e = DocEvent(
+                        doc=d,
+                        rev=d.rev,
+                        type="added_comment",
+                        by=by,
+                        desc="IESG Dead state was set due only to document expiry - changing IESG state to ID-Exists",
+                    )
+                    e.skip_community_list_notification = True
+                    e.save()
+                    events.append(e)
+                    e = new_state_change_event(d, by, dead, id_exists)
+                    e.skip_community_list_notification = True
+                    e.save()
+                    events.append(e)
+                    d.save_with_history(events)
diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py
index f1de459dd8..b7f89e1f9b 100644
--- a/ietf/doc/tasks.py
+++ b/ietf/doc/tasks.py
@@ -18,6 +18,7 @@
     in_draft_expire_freeze,
     get_expired_drafts,
     expirable_drafts,
+    repair_dead_on_expire,
     send_expire_notice_for_draft,
     expire_draft,
     clean_up_draft_files,
@@ -61,6 +62,11 @@ def expire_ids_task():
         raise
 
 
+@shared_task
+def repair_dead_on_expire_task():
+    repair_dead_on_expire()
+
+
 @shared_task
 def notify_expirations_task(notify_days=14):
     for doc in get_soon_to_expire_drafts(notify_days):
diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py
index 09a7b38999..84959625c9 100644
--- a/ietf/doc/tests_draft.py
+++ b/ietf/doc/tests_draft.py
@@ -19,10 +19,10 @@
 
 import debug                            # pyflakes:ignore
 
-from ietf.doc.expire import get_expired_drafts, send_expire_notice_for_draft, expire_draft
-from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, WgDraftFactory, RgDraftFactory, DocEventFactory
+from ietf.doc.expire import expirable_drafts, get_expired_drafts, repair_dead_on_expire, send_expire_notice_for_draft, expire_draft
+from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, StateDocEventFactory, WgDraftFactory, RgDraftFactory, DocEventFactory
 from ietf.doc.models import ( Document, DocReminder, DocEvent,
-    ConsensusDocEvent, LastCallDocEvent, RelatedDocument, State, TelechatDocEvent, 
+    ConsensusDocEvent, LastCallDocEvent, RelatedDocument, State, StateDocEvent, TelechatDocEvent, 
     WriteupDocEvent, DocRelationshipName, IanaExpertDocEvent )
 from ietf.doc.utils import get_tags_for_stream_id, create_ballot_if_not_open
 from ietf.doc.views_draft import AdoptDraftForm
@@ -36,7 +36,7 @@
 from ietf.utils.test_utils import login_testing_unauthorized
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import TestCase
-from ietf.utils.timezone import date_today, datetime_from_date, DEADLINE_TZINFO
+from ietf.utils.timezone import date_today, datetime_today, datetime_from_date, DEADLINE_TZINFO
 
 
 class ChangeStateTests(TestCase):
@@ -763,13 +763,16 @@ def test_expire_drafts(self):
         txt = "%s-%s.txt" % (draft.name, draft.rev)
         self.write_draft_file(txt, 5000)
 
+        self.assertFalse(expirable_drafts(Document.objects.filter(pk=draft.pk)).exists())
+        draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="idexists"))
+        self.assertTrue(expirable_drafts(Document.objects.filter(pk=draft.pk)).exists())
         expire_draft(draft)
 
         draft = Document.objects.get(name=draft.name)
         self.assertEqual(draft.get_state_slug(), "expired")
-        self.assertEqual(draft.get_state_slug("draft-iesg"), "dead")
+        self.assertEqual(draft.get_state_slug("draft-iesg"), "idexists")
         self.assertTrue(draft.latest_event(type="expired_document"))
-        self.assertCountEqual(draft.action_holders.all(), [])
+        self.assertEqual(draft.action_holders.count(), 0)
         self.assertIn('Removed all action holders', draft.latest_event(type='changed_action_holders').desc)
         self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, txt)))
         self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, txt)))
@@ -842,6 +845,77 @@ def test_clean_up_draft_files(self):
         self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, txt)))
         self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, txt)))
 
+    @mock.patch("ietf.community.signals.notify_of_event")
+    def test_repair_dead_on_expire(self, mock_notify):
+
+        # Create a draft in iesg idexists - ensure it doesn't get new docevents.
+        # Create a draft in iesg dead with no expires within the window - ensure it doesn't get new docevents and its state doesn't change.
+        # Create a draft in iesg dead with an expiry in the window - ensure it gets the right doc events, iesg state changes, draft state doesn't change.
+        last_year = datetime_today() - datetime.timedelta(days=365)
+
+        not_dead = WgDraftFactory(name="draft-not-dead")
+        not_dead_event_count = not_dead.docevent_set.count()
+
+        dead_not_from_expires = WgDraftFactory(name="draft-dead-not-from-expiring")
+        dead_not_from_expires.set_state(
+            State.objects.get(type="draft-iesg", slug="dead")
+        )
+        StateDocEventFactory(
+            doc=dead_not_from_expires, state=("draft-iesg", "dead"), time=last_year
+        )
+        DocEventFactory(
+            doc=dead_not_from_expires,
+            type="expired_document",
+            time=last_year + datetime.timedelta(days=1),
+        )
+        dead_not_from_expires_event_count = dead_not_from_expires.docevent_set.count()
+
+        dead_from_expires = []
+        dead_from_expires_event_count = dict()
+        for delta in [-5, 5]:
+            d = WgDraftFactory(
+                name=f"draft-dead-from-expiring-just-{'before' if delta<0 else 'after'}"
+            )
+            d.set_state(State.objects.get(type="draft-iesg", slug="dead"))
+            StateDocEventFactory(doc=d, state=("draft-iesg", "dead"), time=last_year)
+            DocEventFactory(
+                doc=d,
+                type="expired_document",
+                time=last_year + datetime.timedelta(seconds=delta),
+            )
+            dead_from_expires.append(d)
+            dead_from_expires_event_count[d] = d.docevent_set.count()
+
+        notified_during_factory_work = mock_notify.call_count
+        for call_args in mock_notify.call_args_list:
+            e = call_args.args[0]
+            self.assertTrue(isinstance(e,DocEvent))
+            self.assertFalse(hasattr(e,"skip_community_list_notification"))
+
+        repair_dead_on_expire()
+
+        self.assertEqual(not_dead.docevent_set.count(), not_dead_event_count)
+        self.assertEqual(
+            dead_not_from_expires.docevent_set.count(),
+            dead_not_from_expires_event_count,
+        )
+        for d in dead_from_expires:
+            self.assertEqual(
+                d.docevent_set.count(), dead_from_expires_event_count[d] + 2
+            )
+            self.assertIn(
+                "due only to document expiry", d.latest_event(type="added_comment").desc
+            )
+            self.assertEqual(
+                d.latest_event(StateDocEvent).desc,
+                "IESG state changed to <b>I-D Exists</b> from Dead",
+            )
+        self.assertEqual(mock_notify.call_count, 4+notified_during_factory_work)
+        for call_args in mock_notify.call_args_list[-4:]:
+            e = call_args.args[0]
+            self.assertTrue(isinstance(e,DocEvent))
+            self.assertTrue(hasattr(e,"skip_community_list_notification"))
+            self.assertTrue(e.skip_community_list_notification)
 
 class ExpireLastCallTests(TestCase):
     def test_expire_last_call(self):
diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py
index b75f58656b..135b52f604 100644
--- a/ietf/doc/tests_tasks.py
+++ b/ietf/doc/tests_tasks.py
@@ -1,4 +1,6 @@
 # Copyright The IETF Trust 2024, All Rights Reserved
+
+import debug    # pyflakes:ignore
 import datetime
 import mock
 
@@ -19,6 +21,7 @@
     generate_idnits2_rfcs_obsoleted_task,
     generate_idnits2_rfc_status_task,
     notify_expirations_task,
+    repair_dead_on_expire_task,
 )
 
 class TaskTests(TestCase):
@@ -96,6 +99,10 @@ def test_expire_last_calls_task(self, mock_get_expired, mock_expire):
         self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1]))
         self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2]))
 
+    @mock.patch("ietf.doc.tasks.repair_dead_on_expire")
+    def test_repair_dead_on_expire_task(self, mock_repair):
+        repair_dead_on_expire_task()
+        self.assertEqual(mock_repair.call_count, 1)
 
 class Idnits2SupportTests(TestCase):
     settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR']
diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index b2bc0997b1..ff19dfbde6 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -491,8 +491,9 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None,
     
     Returns an event describing the change which should be passed to doc.save_with_history()
     
-    Only cares about draft-iesg state changes. Places where other state types are updated
-    may not call this method. If you add rules for updating action holders on other state
+    Only cares about draft-iesg state changes and draft expiration. 
+    Places where other state types are updated may not call this method. 
+    If you add rules for updating action holders on other state
     types, be sure this is called in the places that change that state.
     """
     # Should not call this with different state types
@@ -511,41 +512,50 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None,
     
     # Remember original list of action holders to later check if it changed
     prev_set = list(doc.action_holders.all())
-    
-    # Update the action holders. To get this right for people with more
-    # than one relationship to the document, do removals first, then adds.
-    # Remove outdated action holders
-    iesg_state_changed = (prev_state != new_state) and (getattr(new_state, "type_id", None) == "draft-iesg") 
-    if iesg_state_changed:
-        # Clear the action_holders list on a state change. This will reset the age of any that get added back.
+
+    if new_state and new_state.type_id=="draft" and new_state.slug=="expired":
         doc.action_holders.clear()
-    if tags.removed("need-rev"):
-        # Removed the 'need-rev' tag - drop authors from the action holders list
-        DocumentActionHolder.objects.filter(document=doc, person__in=doc.authors()).delete()
-    elif tags.added("need-rev"):
-        # Remove the AD if we're asking for a new revision
-        DocumentActionHolder.objects.filter(document=doc, person=doc.ad).delete()
-
-    # Add new action holders
-    if doc.ad:
-        # AD is an action holder unless specified otherwise for the new state
-        if iesg_state_changed and new_state.slug not in DocumentActionHolder.CLEAR_ACTION_HOLDERS_STATES:
-            doc.action_holders.add(doc.ad)
-        # If AD follow-up is needed, make sure they are an action holder 
-        if tags.added("ad-f-up"):
-            doc.action_holders.add(doc.ad)
-    # Authors get the action if a revision is needed
-    if tags.added("need-rev"):
-        for auth in doc.authors():
-            doc.action_holders.add(auth)
-
-    # Now create an event if we changed the set
-    return add_action_holder_change_event(
-        doc, 
-        Person.objects.get(name='(System)'), 
-        prev_set,
-        reason='IESG state changed',
-    )
+        return add_action_holder_change_event(
+            doc, 
+            Person.objects.get(name='(System)'), 
+            prev_set,
+            reason='draft expired',
+        )
+    else:
+        # Update the action holders. To get this right for people with more
+        # than one relationship to the document, do removals first, then adds.
+        # Remove outdated action holders
+        iesg_state_changed = (prev_state != new_state) and (getattr(new_state, "type_id", None) == "draft-iesg") 
+        if iesg_state_changed:
+            # Clear the action_holders list on a state change. This will reset the age of any that get added back.
+            doc.action_holders.clear()
+        if tags.removed("need-rev"):
+            # Removed the 'need-rev' tag - drop authors from the action holders list
+            DocumentActionHolder.objects.filter(document=doc, person__in=doc.authors()).delete()
+        elif tags.added("need-rev"):
+            # Remove the AD if we're asking for a new revision
+            DocumentActionHolder.objects.filter(document=doc, person=doc.ad).delete()
+
+        # Add new action holders
+        if doc.ad:
+            # AD is an action holder unless specified otherwise for the new state
+            if iesg_state_changed and new_state.slug not in DocumentActionHolder.CLEAR_ACTION_HOLDERS_STATES:
+                doc.action_holders.add(doc.ad)
+            # If AD follow-up is needed, make sure they are an action holder 
+            if tags.added("ad-f-up"):
+                doc.action_holders.add(doc.ad)
+        # Authors get the action if a revision is needed
+        if tags.added("need-rev"):
+            for auth in doc.authors():
+                doc.action_holders.add(auth)
+
+        # Now create an event if we changed the set
+        return add_action_holder_change_event(
+            doc, 
+            Person.objects.get(name='(System)'), 
+            prev_set,
+            reason='IESG state changed',
+        )
 
 
 def update_documentauthors(doc, new_docauthors, by=None, basis=None):
diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py
index d709aedd42..34104b2005 100644
--- a/ietf/doc/views_draft.py
+++ b/ietf/doc/views_draft.py
@@ -95,7 +95,8 @@ def change_state(request, name):
     and logging the change as a comment."""
     doc = get_object_or_404(Document, name=name)
 
-    if (not doc.latest_event(type="started_iesg_process")) or doc.get_state_slug() == "expired":
+    # Steer ADs towards "Begin IESG Processing"
+    if doc.get_state_slug("draft-iesg")=="idexists" and not has_role(request.user,"Secretariat"):
         raise Http404
 
     login = request.user.person

From 981f7aea3e3641356de92207f2529622764e8828 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 13 Dec 2024 15:14:13 -0600
Subject: [PATCH 210/656] chore: remove unused setting (#8336)

---
 ietf/settings.py            | 1 -
 test/settings_local_test.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/ietf/settings.py b/ietf/settings.py
index 7c3dc7fa16..4e678b3808 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -750,7 +750,6 @@ def skip_unreadable_post(record):
 STATUS_CHANGE_PATH = '/a/ietfdata/doc/status-change'
 AGENDA_PATH = '/a/www/www6s/proceedings/'
 MEETINGHOST_LOGO_PATH = AGENDA_PATH  # put these in the same place as other proceedings files
-IPR_DOCUMENT_PATH = '/a/www/ietf-ftp/ietf/IPR/'
 # Move drafts to this directory when they expire
 INTERNET_DRAFT_ARCHIVE_DIR = '/a/ietfdata/doc/draft/collection/draft-archive/'
 # The following directory contains copies of all drafts - it used to be
diff --git a/test/settings_local_test.py b/test/settings_local_test.py
index 06d810e4af..7097f76459 100644
--- a/test/settings_local_test.py
+++ b/test/settings_local_test.py
@@ -5,6 +5,5 @@
 
 SERVER_MODE = 'test'
 
-IPR_DOCUMENT_PATH = '/home/ietf/adm/IPR/'
 
 SITE_ID = 1

From d59e102c9f57893a18d56246dc8a8e8fb5414f32 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 13 Dec 2024 15:17:01 -0600
Subject: [PATCH 211/656] fix: cleanup ftp copy of repository on move to
 archive (#8335)

---
 ietf/doc/expire.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py
index 0581c9a03f..63955d091a 100644
--- a/ietf/doc/expire.py
+++ b/ietf/doc/expire.py
@@ -150,10 +150,17 @@ def move_file(f):
                     pass
                 else:
                     raise
+    
+    def remove_ftp_copy(f):
+        mark = Path(settings.FTP_DIR) / "internet-drafts" / f
+        if mark.exists():
+            mark.unlink()
+
 
     src_dir = Path(settings.INTERNET_DRAFT_PATH)
     for file in src_dir.glob("%s-%s.*" % (doc.name, rev)):
         move_file(str(file.name))
+        remove_ftp_copy(str(file.name))
 
 def expire_draft(doc):
     # clean up files

From 98b251487dfa685f0b318e6a02b47a84f6f65876 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 13 Dec 2024 19:15:44 -0500
Subject: [PATCH 212/656] chore(deps): bump nanoid in /dev/deploy-to-container
 in the npm group (#8293)

Bumps the npm group in /dev/deploy-to-container with 1 update: [nanoid](https://github.com/ai/nanoid).


Updates `nanoid` from 5.0.8 to 5.0.9
- [Release notes](https://github.com/ai/nanoid/releases)
- [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md)
- [Commits](https://github.com/ai/nanoid/compare/5.0.8...5.0.9)

---
updated-dependencies:
- dependency-name: nanoid
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/deploy-to-container/package-lock.json | 14 +++++++-------
 dev/deploy-to-container/package.json      |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json
index d787f6aa9e..08b6eb6c62 100644
--- a/dev/deploy-to-container/package-lock.json
+++ b/dev/deploy-to-container/package-lock.json
@@ -8,7 +8,7 @@
       "dependencies": {
         "dockerode": "^4.0.2",
         "fs-extra": "^11.2.0",
-        "nanoid": "5.0.8",
+        "nanoid": "5.0.9",
         "nanoid-dictionary": "5.0.0-beta.1",
         "slugify": "1.6.6",
         "tar": "^7.4.3",
@@ -546,9 +546,9 @@
       "optional": true
     },
     "node_modules/nanoid": {
-      "version": "5.0.8",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.8.tgz",
-      "integrity": "sha512-TcJPw+9RV9dibz1hHUzlLVy8N4X9TnwirAjrU08Juo6BNKggzVfP2ZJ/3ZUSq15Xl5i85i+Z89XBO90pB2PghQ==",
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.9.tgz",
+      "integrity": "sha512-Aooyr6MXU6HpvvWXKoVoXwKMs/KyVakWwg7xQfv5/S/RIgJMy0Ifa45H9qqYy7pTCszrHzP21Uk4PZq2HpEM8Q==",
       "funding": [
         {
           "type": "github",
@@ -1346,9 +1346,9 @@
       "optional": true
     },
     "nanoid": {
-      "version": "5.0.8",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.8.tgz",
-      "integrity": "sha512-TcJPw+9RV9dibz1hHUzlLVy8N4X9TnwirAjrU08Juo6BNKggzVfP2ZJ/3ZUSq15Xl5i85i+Z89XBO90pB2PghQ=="
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.9.tgz",
+      "integrity": "sha512-Aooyr6MXU6HpvvWXKoVoXwKMs/KyVakWwg7xQfv5/S/RIgJMy0Ifa45H9qqYy7pTCszrHzP21Uk4PZq2HpEM8Q=="
     },
     "nanoid-dictionary": {
       "version": "5.0.0-beta.1",
diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json
index 530d1f3b8b..8f16279cea 100644
--- a/dev/deploy-to-container/package.json
+++ b/dev/deploy-to-container/package.json
@@ -4,7 +4,7 @@
   "dependencies": {
     "dockerode": "^4.0.2",
     "fs-extra": "^11.2.0",
-    "nanoid": "5.0.8",
+    "nanoid": "5.0.9",
     "nanoid-dictionary": "5.0.0-beta.1",
     "slugify": "1.6.6",
     "tar": "^7.4.3",

From 4a1d29e86ca7bf1d5fca42f1061769c5383ff859 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Mon, 16 Dec 2024 09:51:33 -0500
Subject: [PATCH 213/656] feat: add wiki button to agenda list for hackathon
 sessions (#8133)

* feat: add wiki button to agenda list for hackathon sessions

* fix: update client/agenda/AgendaScheduleList.vue

Co-authored-by: Matthew Holloway <matthew@holloway.co.nz>

* fix: broken tests

---------

Co-authored-by: Matthew Holloway <matthew@holloway.co.nz>
Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 client/agenda/AgendaScheduleList.vue    | 14 +++++++++++-
 client/shared/urls.json                 |  1 +
 ietf/templates/meeting/agenda.html      |  2 +-
 playwright/tests/meeting/agenda.spec.js | 30 ++++++++++++++++++-------
 4 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index 0cac7e39ee..e7c14797ee 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -284,6 +284,7 @@ const meetingEvents = computed(() => {
     const purposesWithoutLinks = ['admin', 'closed_meeting', 'officehours', 'social']
     if (item.flags.showAgenda || (typesWithLinks.includes(item.type) && !purposesWithoutLinks.includes(item.purpose))) {
       if (item.flags.agenda) {
+        // -> Meeting Materials
         links.push({
           id: `lnk-${item.id}-tar`,
           label: 'Download meeting materials as .tar archive',
@@ -305,7 +306,18 @@ const meetingEvents = computed(() => {
           color: 'red'
         })
       }
-      if (agendaStore.usesNotes) {
+      // -> Point to Wiki for Hackathon sessions, HedgeDocs otherwise
+      if (item.name.toLowerCase().includes('hackathon')) {
+        links.push({
+          id: `lnk-${item.id}-wiki`,
+          label: 'Wiki',
+          icon: 'book',
+          href: getUrl('hackathonWiki', {
+            meetingNumber: agendaStore.meeting.number
+          }),
+          color: 'blue'
+        })
+      } else if (agendaStore.usesNotes) {
         links.push({
           id: `lnk-${item.id}-note`,
           label: 'Notepad for note-takers',
diff --git a/client/shared/urls.json b/client/shared/urls.json
index 285caa07d2..15410d68df 100644
--- a/client/shared/urls.json
+++ b/client/shared/urls.json
@@ -1,5 +1,6 @@
 {
   "bofDefinition": "https://www.ietf.org/how/bofs/",
+  "hackathonWiki": "https://wiki.ietf.org/meeting/{meetingNumber}/hackathon",
   "meetingCalIcs": "/meeting/{meetingNumber}/agenda.ics",
   "meetingDetails": "/meeting/{meetingNumber}/session/{eventAcronym}/",
   "meetingMaterialsPdf": "/meeting/{meetingNumber}/agenda/{eventAcronym}-drafts.pdf",
diff --git a/ietf/templates/meeting/agenda.html b/ietf/templates/meeting/agenda.html
index f3228aa5ad..089141bde8 100644
--- a/ietf/templates/meeting/agenda.html
+++ b/ietf/templates/meeting/agenda.html
@@ -101,7 +101,7 @@
     <div id="app"></div>
     <div id="app-loading">
         <div id="app-loading-footer">
-            <a class="btn btn-light text-body-secondary mb-3" href="/meeting/{{ meetingData.meetingNumber }}/agenda.txt"><small>Switch to text-only version &#11166;</small></a>
+            <a class="btn btn-light text-body-secondary mb-3" href="/meeting/{{ meetingData.meetingNumber }}/agenda.txt"><small>Switch to text-only version &raquo;</small></a>
         </div>
     </div>
 {% endblock %}
diff --git a/playwright/tests/meeting/agenda.spec.js b/playwright/tests/meeting/agenda.spec.js
index 109e1b3b81..d31dbd5c2c 100644
--- a/playwright/tests/meeting/agenda.spec.js
+++ b/playwright/tests/meeting/agenda.spec.js
@@ -286,10 +286,17 @@ test.describe('past - desktop', () => {
               // No meeting materials yet warning badge
               await expect(eventButtons.locator('.no-meeting-materials')).toBeVisible()
             }
-            // Notepad button
-            const hedgeDocLink = `https://notes.ietf.org/notes-ietf-${meetingData.meeting.number}-${event.type === 'plenary' ? 'plenary' : event.acronym}`
-            await expect(eventButtons.locator(`#btn-lnk-${event.id}-note`)).toHaveAttribute('href', hedgeDocLink)
-            await expect(eventButtons.locator(`#btn-lnk-${event.id}-note > i.bi`)).toBeVisible()
+            if (event.name.toLowerCase().includes('hackathon')) {
+              // Hackathon Wiki button
+              const hackathonWikiLink = `https://wiki.ietf.org/meeting/${meetingData.meeting.number}/hackathon`
+              await expect(eventButtons.locator(`#btn-lnk-${event.id}-wiki`)).toHaveAttribute('href', hackathonWikiLink)
+              await expect(eventButtons.locator(`#btn-lnk-${event.id}-wiki > i.bi`)).toBeVisible()
+            } else {
+              // Notepad button
+              const hedgeDocLink = `https://notes.ietf.org/notes-ietf-${meetingData.meeting.number}-${event.type === 'plenary' ? 'plenary' : event.acronym}`
+              await expect(eventButtons.locator(`#btn-lnk-${event.id}-note`)).toHaveAttribute('href', hedgeDocLink)
+              await expect(eventButtons.locator(`#btn-lnk-${event.id}-note > i.bi`)).toBeVisible()
+            }
             // Chat logs
             await expect(eventButtons.locator(`#btn-lnk-${event.id}-logs`)).toHaveAttribute('href', event.links.chatArchive)
             await expect(eventButtons.locator(`#btn-lnk-${event.id}-logs > i.bi`)).toBeVisible()
@@ -1162,10 +1169,17 @@ test.describe('future - desktop', () => {
             // No meeting materials yet warning badge
             await expect(eventButtons.locator('.no-meeting-materials')).toBeVisible()
           }
-          // Notepad button
-          const hedgeDocLink = `https://notes.ietf.org/notes-ietf-${meetingData.meeting.number}-${event.type === 'plenary' ? 'plenary' : event.acronym}`
-          await expect(eventButtons.locator(`#btn-lnk-${event.id}-note`)).toHaveAttribute('href', hedgeDocLink)
-          await expect(eventButtons.locator(`#btn-lnk-${event.id}-note > i.bi`)).toBeVisible()
+          if (event.name.toLowerCase().includes('hackathon')) {
+            // Hackathon Wiki button
+            const hackathonWikiLink = `https://wiki.ietf.org/meeting/${meetingData.meeting.number}/hackathon`
+            await expect(eventButtons.locator(`#btn-lnk-${event.id}-wiki`)).toHaveAttribute('href', hackathonWikiLink)
+            await expect(eventButtons.locator(`#btn-lnk-${event.id}-wiki > i.bi`)).toBeVisible()
+          } else {
+            // Notepad button
+            const hedgeDocLink = `https://notes.ietf.org/notes-ietf-${meetingData.meeting.number}-${event.type === 'plenary' ? 'plenary' : event.acronym}`
+            await expect(eventButtons.locator(`#btn-lnk-${event.id}-note`)).toHaveAttribute('href', hedgeDocLink)
+            await expect(eventButtons.locator(`#btn-lnk-${event.id}-note > i.bi`)).toBeVisible()
+          }
           // Chat room
           await expect(eventButtons.locator(`#btn-lnk-${event.id}-room`)).toHaveAttribute('href', event.links.chat)
           await expect(eventButtons.locator(`#btn-lnk-${event.id}-room > i.bi`)).toBeVisible()

From 171e50b1cd2f57bd9c4b099e1d33e3444d127834 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 17 Dec 2024 11:08:35 -0500
Subject: [PATCH 214/656] fix: homepage logo width (#8338)

* fix: homepage logo width

* fix: make Robert happy
---
 ietf/static/css/ietf.scss | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss
index e2d5cb3959..f1f2b94a19 100644
--- a/ietf/static/css/ietf.scss
+++ b/ietf/static/css/ietf.scss
@@ -236,9 +236,13 @@ th,
 
 // Helper to constrain the size of the main logo
 .ietflogo {
-    width: 75%;
+    width: 100%;
     max-width: 300px;
 }
+.ietflogo > img {
+    min-width: 100px;
+    width: 100%;
+}
 
 // Make revision numbers pagination items fixed-width
 .revision-list {

From 427045d45687f242228ed830b51a5c21dc005eb4 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 17 Dec 2024 15:09:44 -0400
Subject: [PATCH 215/656] chore: remove unused method (#8341)

* chore: remove unused method

* chore: adjust another comment
---
 ietf/submit/utils.py | 69 ++------------------------------------------
 1 file changed, 2 insertions(+), 67 deletions(-)

diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index e6cbcb12f7..5906785048 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -770,70 +770,6 @@ def save_files(form):
         log.log("saved file %s" % name)
     return file_name
 
-def get_draft_meta(form, saved_files):
-    authors = []
-    file_name = saved_files
-
-    if form.cleaned_data['xml']:
-        # Some meta-information, such as the page-count, can only
-        # be retrieved from the generated text file.  Provide a
-        # parsed draft object to get at that kind of information.
-        file_name['txt'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (form.filename, form.revision))
-        file_size = os.stat(file_name['txt']).st_size
-        with io.open(file_name['txt']) as txt_file:
-            form.parsed_draft = PlaintextDraft(txt_file.read(), txt_file.name)
-    else:
-        file_size = form.cleaned_data['txt'].size
-
-    if form.authors:
-        authors = form.authors
-    else:
-        # If we don't have an xml file, try to extract the
-        # relevant information from the text file
-        for author in form.parsed_draft.get_author_list():
-            full_name, first_name, middle_initial, last_name, name_suffix, email, country, company = author
-
-            name = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip()
-
-            if email:
-                try:
-                    validate_email(email)
-                except ValidationError:
-                    email = ""
-
-            def turn_into_unicode(s):
-                if s is None:
-                    return ""
-
-                if isinstance(s, str):
-                    return s
-                else:
-                    try:
-                        return s.decode("utf-8")
-                    except UnicodeDecodeError:
-                        try:
-                            return s.decode("latin-1")
-                        except UnicodeDecodeError:
-                            return ""
-
-            name = turn_into_unicode(name)
-            email = turn_into_unicode(email)
-            company = turn_into_unicode(company)
-
-            authors.append({
-                "name": name,
-                "email": email,
-                "affiliation": company,
-                "country": country
-            })
-
-    if form.abstract:
-        abstract = form.abstract
-    else:
-        abstract = form.parsed_draft.get_abstract()
-
-    return authors, abstract, file_name, file_size
-
 
 def get_submission(form):
     # See if there is a Submission in state waiting-for-draft
@@ -1272,8 +1208,7 @@ def process_submission_xml(filename, revision):
 def _turn_into_unicode(s: Optional[Union[str, bytes]]):
     """Decode a possibly null string-like item as a string
     
-    Copied from ietf.submit.utils.get_draft_meta(), would be nice to
-    ditch this.
+    Would be nice to ditch this.
     """
     if s is None:
         return ""
@@ -1317,7 +1252,7 @@ def process_submission_text(filename, revision):
     if title:
         title = _normalize_title(title)
 
-    # Drops \r, \n, <, >. Based on get_draft_meta() behavior
+    # Translation taable drops \r, \n, <, >.
     trans_table = str.maketrans("", "", "\r\n<>")
     authors = [
         {

From fdd378390f60ee666d47815940a7cede38d23dc1 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 19 Dec 2024 12:06:38 -0600
Subject: [PATCH 216/656] feat: copy the 1wg files to so... many... places...
 (#8347)

---
 ietf/group/tasks.py      | 35 ++++++++++++++++++++---------------
 ietf/group/tests_info.py |  2 ++
 ietf/settings.py         |  2 ++
 3 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/ietf/group/tasks.py b/ietf/group/tasks.py
index f19246fb55..8b4c994ba1 100644
--- a/ietf/group/tasks.py
+++ b/ietf/group/tasks.py
@@ -43,23 +43,28 @@ def generate_wg_charters_files_task():
         encoding="utf8",
     )
 
-    charter_copy_dest = getattr(settings, "CHARTER_COPY_PATH", None)
-    if charter_copy_dest is not None:
-        if not Path(charter_copy_dest).is_dir():
-            log.log(
-                f"Error copying 1wg-charter files to {charter_copy_dest}: it does not exist or is not a directory"
-            )
-        else:
-            try:
-                shutil.copy2(charters_file, charter_copy_dest)
-            except IOError as err:
-                log.log(f"Error copying {charters_file} to {charter_copy_dest}: {err}")
-            try:
-                shutil.copy2(charters_by_acronym_file, charter_copy_dest)
-            except IOError as err:
+    charter_copy_dests = [
+        getattr(settings, "CHARTER_COPY_PATH", None), 
+        getattr(settings, "CHARTER_COPY_OTHER_PATH", None),
+        getattr(settings, "CHARTER_COPY_THIRD_PATH", None),
+    ]
+    for charter_copy_dest in charter_copy_dests:
+        if charter_copy_dest is not None:
+            if not Path(charter_copy_dest).is_dir():
                 log.log(
-                    f"Error copying {charters_by_acronym_file} to {charter_copy_dest}: {err}"
+                    f"Error copying 1wg-charter files to {charter_copy_dest}: it does not exist or is not a directory"
                 )
+            else:
+                try:
+                    shutil.copy2(charters_file, charter_copy_dest)
+                except IOError as err:
+                    log.log(f"Error copying {charters_file} to {charter_copy_dest}: {err}")
+                try:
+                    shutil.copy2(charters_by_acronym_file, charter_copy_dest)
+                except IOError as err:
+                    log.log(
+                        f"Error copying {charters_by_acronym_file} to {charter_copy_dest}: {err}"
+                    )
 
 
 @shared_task
diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py
index 35c8b2b0b7..32d919c779 100644
--- a/ietf/group/tests_info.py
+++ b/ietf/group/tests_info.py
@@ -62,6 +62,8 @@ class GroupPagesTests(TestCase):
     settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [
         "CHARTER_PATH",
         "CHARTER_COPY_PATH",
+        "CHARTER_COPY_OTHER_PATH", # Note: not explicitly testing use of 
+        "CHARTER_COPY_THIRD_PATH", #       either of these settings
         "GROUP_SUMMARY_PATH",
     ]
 
diff --git a/ietf/settings.py b/ietf/settings.py
index 4e678b3808..efd04c6068 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -744,6 +744,8 @@ def skip_unreadable_post(record):
 RFC_PATH = '/a/www/ietf-ftp/rfc/'
 CHARTER_PATH = '/a/ietfdata/doc/charter/'
 CHARTER_COPY_PATH = '/a/www/ietf-ftp/ietf'  # copy 1wg-charters files here if set
+CHARTER_COPY_OTHER_PATH = '/a/www/ftp/ietf'
+CHARTER_COPY_THIRD_PATH = '/a/www/ftp/charter'
 GROUP_SUMMARY_PATH = '/a/www/ietf-ftp/ietf'
 BOFREQ_PATH = '/a/ietfdata/doc/bofreq/'
 CONFLICT_REVIEW_PATH = '/a/ietfdata/doc/conflict-review'

From d25af5e71bcb75b4b4ba61d41c3e7655b9975a57 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 19 Dec 2024 18:01:33 -0600
Subject: [PATCH 217/656] fix: clean ftp copy of repository when cleaning
 repository (#8354)

---
 ietf/doc/expire.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py
index 63955d091a..57af2ad91f 100644
--- a/ietf/doc/expire.py
+++ b/ietf/doc/expire.py
@@ -213,11 +213,11 @@ def splitext(fn):
 
         def move_file_to(subdir):
             # Similar to move_draft_files_to_archive
-            # ghostlinkd would keep this in the combined all archive since it would
-            # be sourced from a different place. But when ghostlinkd is removed, nothing
-            # new is needed here - the file will already exist in the combined archive
             shutil.move(path,
                         os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, subdir, basename))
+            mark = Path(settings.FTP_DIR) / "internet-drafts" / basename
+            if mark.exists():
+                mark.unlink()
 
         try:
             doc = Document.objects.get(name=filename, rev=revision)

From 8cadc6988f1bb815c2d35578b146ddc2c1daea6a Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 20 Dec 2024 08:50:53 -0600
Subject: [PATCH 218/656] chore: remove obviated manual INSTALL instructions
 (#8355)

---
 dev/INSTALL | 157 ----------------------------------------------------
 1 file changed, 157 deletions(-)
 delete mode 100644 dev/INSTALL

diff --git a/dev/INSTALL b/dev/INSTALL
deleted file mode 100644
index 9a8004010d..0000000000
--- a/dev/INSTALL
+++ /dev/null
@@ -1,157 +0,0 @@
-==============================================================================
-			       IETF Datatracker
-==============================================================================
-
-------------------------------------------------------------------------------
-     Installation Instructions
-------------------------------------------------------------------------------
-
-General Instructions for Deployment of a New Release
-====================================================
-
- 0. Prepare to hold different roles at different stages of the instructions below.
-    You will need to be root, wwwrun, and some user in group docker. 
-    Consider using separate shells for the wwwrun and other roles. These instructions 
-    are written assuming you will only use one shell.
-
- 1. Make a directory to hold the new release as wwwrun::
-      sudo su - -s /bin/bash wwwrun
-      mkdir /a/www/ietf-datatracker/${releasenumber}
-      cd /a/www/ietf-datatracker/${releasenumber}
-
- 2. Fetch the release tarball from github 
-    (see https://github.com/ietf-tools/datatracker/releases)::
-
-      wget https://github.com/ietf-tools/datatracker/releases/download/${releasenumber}/release.tar.gz
-      tar xzvf release.tar.gz
-  
- 3. Copy ietf/settings_local.py from previous release::
-
-      cp ../web/ietf/settings_local.py ietf/
-
- 4. Setup a new virtual environment and install requirements::
-
-      python3.9 -mvenv env
-      source env/bin/activate
-      pip install -r requirements.txt
-      pip freeze > frozen-requirements.txt
-
-    (The pip freeze command records the exact versions of the Python libraries that pip installed.
-     This is used by the celery docker container to ensure it uses the same library versions as
-     the datatracker service.)
-
- 5. Move static files into place for CDN (/a/www/www6s/lib/dt):
-
-      ietf/manage.py collectstatic
-
- 6. Run system checks (which patches the just installed modules)::
-
-      ietf/manage.py check
-
- 7. Switch to the docker directory and update images as a user in group docker:
-
-      exit
-      cd /a/docker/datatracker
-      docker image tag ghcr.io/ietf-tools/datatracker-celery:latest datatracker-celery-fallback
-      docker image tag ghcr.io/ietf-tools/datatracker-mq:latest datatracker-mq-fallback
-      docker-compose pull
-
- 8. Stop and remove the async task containers:
-    Wait for this to finish cleanly. Usually this will only be a few seconds, but it may take up 
-    to about 10 minutes for the 'down' command to complete if a long-running task is in progress.
-
-      docker-compose down
-
- 9. Stop the datatracker and remove the web link so cron or other applications 
-    don't run code in the older deployment. 
-
-      sudo systemctl stop datatracker.socket datatracker.service
-      rm /a/www/ietf-datatracker/web
-
- 10. Return to the release directory and run migrations as wwwrun:
-
-      sudo su - -s /bin/bash wwwrun
-      cd /a/www/ietf-datatracker/${releasenumber}
-      ietf/manage.py migrate
-
-      Take note if any migrations were executed.
- 
- 11. Back out one directory level, then re-point the 'web' symlink::
-
-      cd ..
-      ln -s ${releasenumber} web
-
- 12. Start the datatracker service (it is no longer necessary to restart apache) ::
-
-      exit 
-      sudo systemctl start datatracker.service datatracker.socket
-
- 13. Start async task worker and message broker:
-
-      cd /a/docker/datatracker
-      bash startcommand
-
- 14. Verify operation:
-
-      http://datatracker.ietf.org/
-
- 15. If install failed and there were no migrations at step 9, revert web symlink and docker update and repeat the
-     restart in steps 11 and 12. To revert the docker update:
-
-          cd /a/docker/datatracker
-          docker-compose down
-          docker image rm ghcr.io/ietf-tools/datatracker-celery:latest ghcr.io/ietf-tools/datatracker-mq:latest
-          docker image tag datatracker-celery-fallback ghcr.io/ietf-tools/datatracker-celery:latest
-          docker image tag datatracker-mq-fallback ghcr.io/ietf-tools/datatracker-mq:latest
-          cd -
-
-     If there were migrations at step 10, they will need to be reversed before the restart at step 12.
-     If it's not obvious what to do to reverse the migrations, contact the dev team.
-
- 
-Patching a Production Release
-=============================
-
-Sometimes it can prove necessary to patch an existing release.
-The following process should be used:
-
- 1. Code and test the patch on an copy of the release with any
-    previously applied patches put in place.
-
- 2. Produce a patch file, named with date and subject::
-
-      $ git diff > 2013-03-25-ballot-calculation.patch
-
- 3. Move the patch file to the production server, and place it in 
-    '/a/www/ietf-datatracker/patches/'
-
- 4. Make a recursive copy of the production code to a new directory, named with a patch number.  
-
-      /a/www/ietf-datatracker $ rsync -a web/ ${releasenumber}.p1/
-
- 5. Apply the patch::
-
-      /a/www/ietf-datatracker $ cd ${releasenumber}.p1/      
-      /a/www/ietf-datatracker/${releasnumber}.p1 $ patch -p1 \
-			< ../patches/2013-03-25-ballot-calculation.patch
-
-    This must not produce any messages about failing to apply any chunks;
-    if it does, go back to 1. and figure out why.
-
- 6. Edit ``.../ietf/__init__.py`` in the new patched release to indicate the patch
-    version in the ``__patch__`` string.
-
- 7. Stop the async task container (this may take a few minutes if tasks are in progress):
-
-      cd /a/docker/datatracker
-      docker-compose down
-
- 8. Change the 'web' symlink, reload etc. as described in
-    `General Instructions for Deployment of a New Release`_.
-
- 9. Start async task worker:
-
-      cd /a/docker/datatracker
-      bash startcommand
-
-

From 8b7fa1199a72133fd3dd87d148a48cf08ad033e8 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 20 Dec 2024 08:53:12 -0600
Subject: [PATCH 219/656] chore: remove single-use repair_dead_on_expire task
 (#8348)

* chore: remove single-use repair_dead_on_expire task

* fix: remove repair_dead_on_expire

* chore: remove abandoned imports
---
 ietf/doc/expire.py      | 42 ++--------------------
 ietf/doc/tasks.py       |  6 ----
 ietf/doc/tests_draft.py | 79 +++--------------------------------------
 ietf/doc/tests_tasks.py |  5 ---
 4 files changed, 6 insertions(+), 126 deletions(-)

diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py
index 57af2ad91f..98554bae0e 100644
--- a/ietf/doc/expire.py
+++ b/ietf/doc/expire.py
@@ -13,10 +13,10 @@
 
 from typing import List, Optional      # pyflakes:ignore
 
-from ietf.doc.utils import new_state_change_event, update_action_holders
+from ietf.doc.utils import update_action_holders
 from ietf.utils import log
 from ietf.utils.mail import send_mail
-from ietf.doc.models import Document, DocEvent, State, StateDocEvent
+from ietf.doc.models import Document, DocEvent, State
 from ietf.person.models import Person 
 from ietf.meeting.models import Meeting
 from ietf.mailtrigger.utils import gather_address_lists
@@ -235,41 +235,3 @@ def move_file_to(subdir):
             # All uses of this past 2014 seem related to major system failures.
             move_file_to("unknown_ids")
 
-
-def repair_dead_on_expire():
-    by = Person.objects.get(name="(System)")
-    id_exists = State.objects.get(type="draft-iesg", slug="idexists")
-    dead = State.objects.get(type="draft-iesg", slug="dead")
-    dead_drafts = Document.objects.filter(
-        states__type="draft-iesg", states__slug="dead", type_id="draft"
-    )
-    for d in dead_drafts:
-        dead_event = d.latest_event(
-            StateDocEvent, state_type="draft-iesg", state__slug="dead"
-        )
-        if dead_event is not None:
-            if d.docevent_set.filter(type="expired_document").exists():
-                closest_expiry = min(
-                    [
-                        abs(e.time - dead_event.time)
-                        for e in d.docevent_set.filter(type="expired_document")
-                    ]
-                )
-                if closest_expiry.total_seconds() < 60:
-                    d.set_state(id_exists)
-                    events = []
-                    e = DocEvent(
-                        doc=d,
-                        rev=d.rev,
-                        type="added_comment",
-                        by=by,
-                        desc="IESG Dead state was set due only to document expiry - changing IESG state to ID-Exists",
-                    )
-                    e.skip_community_list_notification = True
-                    e.save()
-                    events.append(e)
-                    e = new_state_change_event(d, by, dead, id_exists)
-                    e.skip_community_list_notification = True
-                    e.save()
-                    events.append(e)
-                    d.save_with_history(events)
diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py
index b7f89e1f9b..f1de459dd8 100644
--- a/ietf/doc/tasks.py
+++ b/ietf/doc/tasks.py
@@ -18,7 +18,6 @@
     in_draft_expire_freeze,
     get_expired_drafts,
     expirable_drafts,
-    repair_dead_on_expire,
     send_expire_notice_for_draft,
     expire_draft,
     clean_up_draft_files,
@@ -62,11 +61,6 @@ def expire_ids_task():
         raise
 
 
-@shared_task
-def repair_dead_on_expire_task():
-    repair_dead_on_expire()
-
-
 @shared_task
 def notify_expirations_task(notify_days=14):
     for doc in get_soon_to_expire_drafts(notify_days):
diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py
index 84959625c9..2405806682 100644
--- a/ietf/doc/tests_draft.py
+++ b/ietf/doc/tests_draft.py
@@ -19,10 +19,10 @@
 
 import debug                            # pyflakes:ignore
 
-from ietf.doc.expire import expirable_drafts, get_expired_drafts, repair_dead_on_expire, send_expire_notice_for_draft, expire_draft
-from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, StateDocEventFactory, WgDraftFactory, RgDraftFactory, DocEventFactory
+from ietf.doc.expire import expirable_drafts, get_expired_drafts, send_expire_notice_for_draft, expire_draft
+from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, WgDraftFactory, RgDraftFactory, DocEventFactory
 from ietf.doc.models import ( Document, DocReminder, DocEvent,
-    ConsensusDocEvent, LastCallDocEvent, RelatedDocument, State, StateDocEvent, TelechatDocEvent, 
+    ConsensusDocEvent, LastCallDocEvent, RelatedDocument, State, TelechatDocEvent, 
     WriteupDocEvent, DocRelationshipName, IanaExpertDocEvent )
 from ietf.doc.utils import get_tags_for_stream_id, create_ballot_if_not_open
 from ietf.doc.views_draft import AdoptDraftForm
@@ -36,7 +36,7 @@
 from ietf.utils.test_utils import login_testing_unauthorized
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import TestCase
-from ietf.utils.timezone import date_today, datetime_today, datetime_from_date, DEADLINE_TZINFO
+from ietf.utils.timezone import date_today, datetime_from_date, DEADLINE_TZINFO
 
 
 class ChangeStateTests(TestCase):
@@ -845,77 +845,6 @@ def test_clean_up_draft_files(self):
         self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, txt)))
         self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, txt)))
 
-    @mock.patch("ietf.community.signals.notify_of_event")
-    def test_repair_dead_on_expire(self, mock_notify):
-
-        # Create a draft in iesg idexists - ensure it doesn't get new docevents.
-        # Create a draft in iesg dead with no expires within the window - ensure it doesn't get new docevents and its state doesn't change.
-        # Create a draft in iesg dead with an expiry in the window - ensure it gets the right doc events, iesg state changes, draft state doesn't change.
-        last_year = datetime_today() - datetime.timedelta(days=365)
-
-        not_dead = WgDraftFactory(name="draft-not-dead")
-        not_dead_event_count = not_dead.docevent_set.count()
-
-        dead_not_from_expires = WgDraftFactory(name="draft-dead-not-from-expiring")
-        dead_not_from_expires.set_state(
-            State.objects.get(type="draft-iesg", slug="dead")
-        )
-        StateDocEventFactory(
-            doc=dead_not_from_expires, state=("draft-iesg", "dead"), time=last_year
-        )
-        DocEventFactory(
-            doc=dead_not_from_expires,
-            type="expired_document",
-            time=last_year + datetime.timedelta(days=1),
-        )
-        dead_not_from_expires_event_count = dead_not_from_expires.docevent_set.count()
-
-        dead_from_expires = []
-        dead_from_expires_event_count = dict()
-        for delta in [-5, 5]:
-            d = WgDraftFactory(
-                name=f"draft-dead-from-expiring-just-{'before' if delta<0 else 'after'}"
-            )
-            d.set_state(State.objects.get(type="draft-iesg", slug="dead"))
-            StateDocEventFactory(doc=d, state=("draft-iesg", "dead"), time=last_year)
-            DocEventFactory(
-                doc=d,
-                type="expired_document",
-                time=last_year + datetime.timedelta(seconds=delta),
-            )
-            dead_from_expires.append(d)
-            dead_from_expires_event_count[d] = d.docevent_set.count()
-
-        notified_during_factory_work = mock_notify.call_count
-        for call_args in mock_notify.call_args_list:
-            e = call_args.args[0]
-            self.assertTrue(isinstance(e,DocEvent))
-            self.assertFalse(hasattr(e,"skip_community_list_notification"))
-
-        repair_dead_on_expire()
-
-        self.assertEqual(not_dead.docevent_set.count(), not_dead_event_count)
-        self.assertEqual(
-            dead_not_from_expires.docevent_set.count(),
-            dead_not_from_expires_event_count,
-        )
-        for d in dead_from_expires:
-            self.assertEqual(
-                d.docevent_set.count(), dead_from_expires_event_count[d] + 2
-            )
-            self.assertIn(
-                "due only to document expiry", d.latest_event(type="added_comment").desc
-            )
-            self.assertEqual(
-                d.latest_event(StateDocEvent).desc,
-                "IESG state changed to <b>I-D Exists</b> from Dead",
-            )
-        self.assertEqual(mock_notify.call_count, 4+notified_during_factory_work)
-        for call_args in mock_notify.call_args_list[-4:]:
-            e = call_args.args[0]
-            self.assertTrue(isinstance(e,DocEvent))
-            self.assertTrue(hasattr(e,"skip_community_list_notification"))
-            self.assertTrue(e.skip_community_list_notification)
 
 class ExpireLastCallTests(TestCase):
     def test_expire_last_call(self):
diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py
index 135b52f604..67997acd85 100644
--- a/ietf/doc/tests_tasks.py
+++ b/ietf/doc/tests_tasks.py
@@ -21,7 +21,6 @@
     generate_idnits2_rfcs_obsoleted_task,
     generate_idnits2_rfc_status_task,
     notify_expirations_task,
-    repair_dead_on_expire_task,
 )
 
 class TaskTests(TestCase):
@@ -99,10 +98,6 @@ def test_expire_last_calls_task(self, mock_get_expired, mock_expire):
         self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1]))
         self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2]))
 
-    @mock.patch("ietf.doc.tasks.repair_dead_on_expire")
-    def test_repair_dead_on_expire_task(self, mock_repair):
-        repair_dead_on_expire_task()
-        self.assertEqual(mock_repair.call_count, 1)
 
 class Idnits2SupportTests(TestCase):
     settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR']

From 7d464fcc558a0e03a535e23bb0dd01f6e7a8dabd Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 9 Jan 2025 11:23:20 -0600
Subject: [PATCH 220/656] fix: keep the ftp dir in sync when the datatracker
 creates artifacts (#8401)

* chore: fix typo in log message

* fix: write new conflict reviews to the ftp directory

* fix: log exception description

* chore: avoid reusing a name for mypy

* fix: write non-meeting-related slides to ftp/slides

* fix: write status changes to ftp dir

* fix: copy draft yang modules to the ftp dir

the last commit above (see #8401) has no tests :(. There aren't tests of this yang scaffolding to add to. Adding a set of tests should be an independent larger effort.
---
 ietf/doc/tests_conflict_review.py | 14 +++++++++++---
 ietf/doc/tests_material.py        | 13 +++++++++++--
 ietf/doc/tests_status_change.py   | 17 +++++++++++++----
 ietf/doc/utils_charter.py         |  6 +++---
 ietf/doc/views_conflict_review.py | 14 ++++++++++++--
 ietf/doc/views_material.py        | 16 ++++++++++++++--
 ietf/doc/views_status_change.py   | 23 +++++++++++++++++------
 ietf/submit/checkers.py           | 10 ++++++++++
 ietf/submit/utils.py              |  4 ++++
 9 files changed, 95 insertions(+), 22 deletions(-)

diff --git a/ietf/doc/tests_conflict_review.py b/ietf/doc/tests_conflict_review.py
index 485f5655eb..d2f94922b2 100644
--- a/ietf/doc/tests_conflict_review.py
+++ b/ietf/doc/tests_conflict_review.py
@@ -4,6 +4,7 @@
 
 import io
 import os
+from pathlib import Path
 
 from pyquery import PyQuery
 from textwrap import wrap
@@ -387,7 +388,7 @@ def setUp(self):
 
 
 class ConflictReviewSubmitTests(TestCase):
-    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CONFLICT_REVIEW_PATH']
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CONFLICT_REVIEW_PATH','FTP_PATH']
     def test_initial_submission(self):
         doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
         url = urlreverse('ietf.doc.views_conflict_review.submit',kwargs=dict(name=doc.name))
@@ -403,9 +404,15 @@ def test_initial_submission(self):
         # Right now, nothing to test - we let people put whatever the web browser will let them put into that textbox
 
         # sane post using textbox
-        path = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (doc.name, doc.rev))
+        basename = f"{doc.name}-{doc.rev}.txt"
+        path = Path(settings.CONFLICT_REVIEW_PATH) / basename
+        ftp_dir = Path(settings.FTP_DIR) / "conflict-reviews"
+        if not ftp_dir.exists():
+            ftp_dir.mkdir()
+        ftp_path = ftp_dir / basename
         self.assertEqual(doc.rev,'00')
-        self.assertFalse(os.path.exists(path))
+        self.assertFalse(path.exists())
+        self.assertFalse(ftp_path.exists())
         r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1"))
         self.assertEqual(r.status_code,302)
         doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
@@ -413,6 +420,7 @@ def test_initial_submission(self):
         with io.open(path) as f:
             self.assertEqual(f.read(),"Some initial review text\n")
             f.close()
+        self.assertTrue(ftp_path.exists())
         self.assertTrue( "submission-00" in doc.latest_event(NewRevisionDocEvent).desc)
 
     def test_subsequent_submission(self):
diff --git a/ietf/doc/tests_material.py b/ietf/doc/tests_material.py
index 065ff09a98..aaea8fec3d 100644
--- a/ietf/doc/tests_material.py
+++ b/ietf/doc/tests_material.py
@@ -28,7 +28,7 @@
 
 
 class GroupMaterialTests(TestCase):
-    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH']
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH', 'FTP_DIR']
     def setUp(self):
         super().setUp()
         self.materials_dir = self.tempdir("materials")
@@ -37,6 +37,10 @@ def setUp(self):
             self.slides_dir.mkdir()
         self.saved_document_path_pattern = settings.DOCUMENT_PATH_PATTERN
         settings.DOCUMENT_PATH_PATTERN = self.materials_dir + "/{doc.type_id}/"
+        self.assertTrue(Path(settings.FTP_DIR).exists())
+        ftp_slides_dir = Path(settings.FTP_DIR) / "slides"
+        if not ftp_slides_dir.exists():
+            ftp_slides_dir.mkdir()
 
         self.meeting_slides_dir = Path(settings.AGENDA_PATH) / "42" / "slides"
         if not self.meeting_slides_dir.exists():
@@ -112,7 +116,12 @@ def test_upload_slides(self):
         self.assertEqual(doc.title, "Test File - with fancy title")
         self.assertEqual(doc.get_state_slug(), "active")
 
-        with io.open(os.path.join(self.materials_dir, "slides", doc.name + "-" + doc.rev + ".pdf")) as f:
+        basename=f"{doc.name}-{doc.rev}.pdf"
+        filepath=Path(self.materials_dir) / "slides" / basename
+        with filepath.open() as f:
+            self.assertEqual(f.read(), content)
+        ftp_filepath=Path(settings.FTP_DIR) / "slides" / basename
+        with ftp_filepath.open() as f:
             self.assertEqual(f.read(), content)
 
         # check that posting same name is prevented
diff --git a/ietf/doc/tests_status_change.py b/ietf/doc/tests_status_change.py
index bec48ed4ef..bd4da4c092 100644
--- a/ietf/doc/tests_status_change.py
+++ b/ietf/doc/tests_status_change.py
@@ -4,6 +4,7 @@
 
 import io
 import os
+from pathlib import Path
 
 import debug    # pyflakes:ignore
 
@@ -540,7 +541,7 @@ def setUp(self):
         DocumentFactory(type_id='statchg',name='status-change-imaginary-mid-review',notify='notify@example.org')
 
 class StatusChangeSubmitTests(TestCase):
-    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['STATUS_CHANGE_PATH']
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['STATUS_CHANGE_PATH', 'FTP_PATH']
     def test_initial_submission(self):
         doc = Document.objects.get(name='status-change-imaginary-mid-review')
         url = urlreverse('ietf.doc.views_status_change.submit',kwargs=dict(name=doc.name))
@@ -556,14 +557,19 @@ def test_initial_submission(self):
         # Right now, nothing to test - we let people put whatever the web browser will let them put into that textbox
 
         # sane post using textbox
-        path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.name, doc.rev))
         self.assertEqual(doc.rev,'00')
-        self.assertFalse(os.path.exists(path))
+        basename = f"{doc.name}-{doc.rev}.txt"
+        filepath = Path(settings.STATUS_CHANGE_PATH) / basename
+        ftp_filepath = Path(settings.FTP_DIR) / "status-changes" / basename
+        self.assertFalse(filepath.exists())
+        self.assertFalse(ftp_filepath.exists())
         r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1"))
         self.assertEqual(r.status_code,302)
         doc = Document.objects.get(name='status-change-imaginary-mid-review')
         self.assertEqual(doc.rev,'00')
-        with io.open(path) as f:
+        with filepath.open() as f:
+            self.assertEqual(f.read(),"Some initial review text\n")
+        with ftp_filepath.open() as f:
             self.assertEqual(f.read(),"Some initial review text\n")
         self.assertTrue( "mid-review-00" in doc.latest_event(NewRevisionDocEvent).desc)
 
@@ -628,3 +634,6 @@ def test_subsequent_submission(self):
     def setUp(self):
         super().setUp()
         DocumentFactory(type_id='statchg',name='status-change-imaginary-mid-review',notify='notify@example.org')
+        ftp_subdir=Path(settings.FTP_DIR)/"status-changes"
+        if not ftp_subdir.exists():
+            ftp_subdir.mkdir()
diff --git a/ietf/doc/utils_charter.py b/ietf/doc/utils_charter.py
index b29d1e303c..287ce8cece 100644
--- a/ietf/doc/utils_charter.py
+++ b/ietf/doc/utils_charter.py
@@ -112,10 +112,10 @@ def fix_charter_revision_after_approval(charter, by):
     )
     try:
         os.link(new, ftp_filepath)
-    except IOError:
+    except IOError as ex:
         log(
-            "There was an error creating a harlink at %s pointing to %s"
-            % (ftp_filepath, new)
+            "There was an error creating a hardlink at %s pointing to %s: %s"
+            % (ftp_filepath, new, ex)
         )
 
     events = []
diff --git a/ietf/doc/views_conflict_review.py b/ietf/doc/views_conflict_review.py
index ec5a18c7a2..e55661ccdf 100644
--- a/ietf/doc/views_conflict_review.py
+++ b/ietf/doc/views_conflict_review.py
@@ -5,6 +5,7 @@
 import datetime
 import io
 import os
+from pathlib import Path
 
 from django import forms
 from django.shortcuts import render, get_object_or_404, redirect
@@ -181,12 +182,21 @@ def clean_txt(self):
         return get_cleaned_text_file_content(self.cleaned_data["txt"])
 
     def save(self, review):
-        filename = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (review.name, review.rev))
-        with io.open(filename, 'w', encoding='utf-8') as destination:
+        basename = f"{review.name}-{review.rev}.txt"
+        filepath = Path(settings.CONFLICT_REVIEW_PATH) / basename
+        with filepath.open('w', encoding='utf-8') as destination:
             if self.cleaned_data['txt']:
                 destination.write(self.cleaned_data['txt'])
             else:
                 destination.write(self.cleaned_data['content'])
+        ftp_filepath = Path(settings.FTP_DIR) / "conflict-reviews" / basename
+        try:
+            os.link(filepath, ftp_filepath) # Path.hardlink_to is not available until 3.10
+        except IOError as e:
+            log.log(
+                "There was an error creating a hardlink at %s pointing to %s: %s"
+                % (ftp_filepath, filepath, e)
+            )
 
 #This is very close to submit on charter - can we get better reuse?
 @role_required('Area Director','Secretariat')
diff --git a/ietf/doc/views_material.py b/ietf/doc/views_material.py
index b646ecf2fc..361bf5f1e2 100644
--- a/ietf/doc/views_material.py
+++ b/ietf/doc/views_material.py
@@ -3,8 +3,8 @@
 
 
 # views for managing group materials (slides, ...)
-import io
 import os
+from pathlib import Path
 import re
 
 from django import forms
@@ -162,9 +162,21 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None):
                 f = form.cleaned_data["material"]
                 file_ext = os.path.splitext(f.name)[1]
 
-                with io.open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + file_ext), 'wb+') as dest:
+                basename = f"{doc.name}-{doc.rev}{file_ext}" # Note the lack of a . before file_ext - see os.path.splitext
+                filepath = Path(doc.get_file_path()) / basename
+                with filepath.open('wb+') as dest:
                     for chunk in f.chunks():
                         dest.write(chunk)
+                if not doc.meeting_related():
+                    log.assertion('doc.type_id == "slides"')
+                    ftp_filepath = Path(settings.FTP_DIR) / doc.type_id / basename
+                    try:
+                        os.link(filepath, ftp_filepath) # Path.hardlink is not available until 3.10
+                    except IOError as ex:
+                        log.log(
+                            "There was an error creating a hardlink at %s pointing to %s: %s"
+                            % (ftp_filepath, filepath, ex)
+                        )
 
             if prev_rev != doc.rev:
                 e = NewRevisionDocEvent(type="new_revision", doc=doc, rev=doc.rev)
diff --git a/ietf/doc/views_status_change.py b/ietf/doc/views_status_change.py
index 9034971ebf..33b822348a 100644
--- a/ietf/doc/views_status_change.py
+++ b/ietf/doc/views_status_change.py
@@ -5,6 +5,7 @@
 import datetime
 import io
 import os
+from pathlib import Path
 import re
 
 from typing import Dict             # pyflakes:ignore
@@ -33,6 +34,7 @@
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.name.models import DocRelationshipName, StdLevelName
 from ietf.person.models import Person
+from ietf.utils.log import log
 from ietf.utils.mail import send_mail_preformatted
 from ietf.utils.textupload import get_cleaned_text_file_content
 from ietf.utils.timezone import date_today, DEADLINE_TZINFO
@@ -154,12 +156,21 @@ def clean_txt(self):
         return get_cleaned_text_file_content(self.cleaned_data["txt"])
 
     def save(self, doc):
-       filename = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.name, doc.rev))
-       with io.open(filename, 'w', encoding='utf-8') as destination:
-           if self.cleaned_data['txt']:
-               destination.write(self.cleaned_data['txt'])
-           else:
-               destination.write(self.cleaned_data['content'])
+        basename = f"{doc.name}-{doc.rev}.txt"
+        filename = Path(settings.STATUS_CHANGE_PATH) / basename
+        with io.open(filename, 'w', encoding='utf-8') as destination:
+            if self.cleaned_data['txt']:
+                destination.write(self.cleaned_data['txt'])
+            else:
+                destination.write(self.cleaned_data['content'])
+        try:
+            ftp_filename = Path(settings.FTP_DIR) / "status-changes" / basename
+            os.link(filename, ftp_filename) # Path.hardlink is not available until 3.10
+        except IOError as ex:
+            log(
+                "There was an error creating a hardlink at %s pointing to %s: %s"
+                % (ftp_filename, filename, ex)
+            )
 
 #This is very close to submit on charter - can we get better reuse?
 @role_required('Area Director','Secretariat')
diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py
index d29e2a2355..89908748a7 100644
--- a/ietf/submit/checkers.py
+++ b/ietf/submit/checkers.py
@@ -4,6 +4,7 @@
 
 import io
 import os
+from pathlib import Path
 import re
 import shutil
 import sys
@@ -280,6 +281,15 @@ def check_file_txt(self, path):
 
             dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model)
             shutil.move(path, dest)
+            ftp_dest = Path(settings.FTP_DIR) / "yang" / "draftmod" / model
+            try:
+                os.link(dest, ftp_dest)
+            except IOError as ex:
+                log(
+                    "There was an error creating a hardlink at %s pointing to %s: %s"
+                    % (ftp_dest, dest, ex)
+                )
+
 
             # summary result
             results.append({
diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index 5906785048..16cccc9b59 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -8,6 +8,7 @@
 import os
 import pathlib
 import re
+import subprocess
 import sys
 import time
 import traceback
@@ -1596,3 +1597,6 @@ def active(dirent):
                         modfile.unlink()
         except UnicodeDecodeError as e:
             log.log(f"Error processing {item.name}: {e}")
+
+    ftp_moddir = Path(settings.FTP_DIR) / "yang" / "draftmod"
+    subprocess.call(("/usr/bin/rsync", "-aq", "--delete", moddir, ftp_moddir))

From e5c4a9f2983a8d47842ad3e7a76167364fd2081d Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 9 Jan 2025 13:07:51 -0600
Subject: [PATCH 221/656] feat: additional filesystem monitoring (#8405)

* feat: additional filesystem monitoring

* chore: rename setting for tmp directory

* fix: restructure path to new endpoint

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 dev/deploy-to-container/settings_local.py |  1 +
 dev/diff/settings_local.py                |  1 +
 dev/tests/settings_local.py               |  1 +
 docker/configs/settings_local.py          |  1 +
 docker/scripts/app-create-dirs.sh         |  1 +
 ietf/api/tests.py                         |  8 ++++++++
 ietf/api/urls.py                          |  2 ++
 ietf/api/views.py                         | 20 +++++++++++++++++++-
 ietf/settings.py                          |  1 +
 9 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py
index 07bf0a7511..0a991ae9fe 100644
--- a/dev/deploy-to-container/settings_local.py
+++ b/dev/deploy-to-container/settings_local.py
@@ -64,6 +64,7 @@
 BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml'
 IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
 FTP_DIR = '/assets/ftp'
+NFS_METRICS_TMP_DIR = '/assets/tmp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
 SLIDE_STAGING_PATH = '/test/staging/'
diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py
index 6bcee46b61..95d1e481c9 100644
--- a/dev/diff/settings_local.py
+++ b/dev/diff/settings_local.py
@@ -60,6 +60,7 @@
 INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/'
 BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml'
 FTP_DIR = '/assets/ftp'
+NFS_METRICS_TMP_DIR = '/assets/tmp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
 SLIDE_STAGING_PATH = 'test/staging/'
diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py
index afadb3760b..7b10bee06a 100644
--- a/dev/tests/settings_local.py
+++ b/dev/tests/settings_local.py
@@ -59,6 +59,7 @@
 INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/'
 BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml'
 FTP_DIR = '/assets/ftp'
+NFS_METRICS_TMP_DIR = '/assets/tmp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
 SLIDE_STAGING_PATH = 'test/staging/'
diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py
index a1c19c80cf..5df5d15e82 100644
--- a/docker/configs/settings_local.py
+++ b/docker/configs/settings_local.py
@@ -50,6 +50,7 @@
 BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml'
 IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
 FTP_DIR = '/assets/ftp'
+NFS_METRICS_TMP_DIR = '/assets/tmp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
 SLIDE_STAGING_PATH = '/assets/www6s/staging/'
diff --git a/docker/scripts/app-create-dirs.sh b/docker/scripts/app-create-dirs.sh
index 50431f4793..3eb328a280 100755
--- a/docker/scripts/app-create-dirs.sh
+++ b/docker/scripts/app-create-dirs.sh
@@ -29,6 +29,7 @@ for sub in \
     /assets/www6/iesg \
     /assets/www6/iesg/evaluation \
     /assets/media/photo \
+    /assets/tmp \
     /assets/ftp \
     /assets/ftp/charter \
     /assets/ftp/internet-drafts \
diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index a8d6ac4e57..6e7fb374fe 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -970,6 +970,14 @@ def test_api_appauth(self):
             self.assertEqual(jsondata['success'], True)
             self.client.logout()
 
+    @override_settings(APP_API_TOKENS={"ietf.api.views.nfs_metrics": ["valid-token"]})
+    def test_api_nfs_metrics(self):
+        url = urlreverse("ietf.api.views.nfs_metrics")
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 403)
+        r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
+        self.assertContains(r, 'nfs_latency_seconds{operation="write"}')
+
     def test_api_get_session_matherials_no_agenda_meeting_url(self):
         meeting = MeetingFactory(type_id='ietf')
         session = SessionFactory(meeting=meeting)
diff --git a/ietf/api/urls.py b/ietf/api/urls.py
index a9aaaf5805..b0dbaf91ce 100644
--- a/ietf/api/urls.py
+++ b/ietf/api/urls.py
@@ -82,6 +82,8 @@
     url(r'^version/?$', api_views.version),
     # Application authentication API key
     url(r'^appauth/(?P<app>authortools|bibxml)$', api_views.app_auth),
+    # NFS metrics endpoint
+    url(r'^metrics/nfs/?$', api_views.nfs_metrics),
     # latest versions
     url(r'^rfcdiff-latest-json/%(name)s(?:-%(rev)s)?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, api_views.rfcdiff_latest_json),
     url(r'^rfcdiff-latest-json/(?P<name>[Rr][Ff][Cc] [0-9]+?)(\.txt|\.html)?/?$', api_views.rfcdiff_latest_json),
diff --git a/ietf/api/views.py b/ietf/api/views.py
index 3e56757528..2fd9d2730f 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -3,7 +3,10 @@
 
 import base64
 import binascii
+import datetime
 import json
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 import jsonschema
 import pytz
 import re
@@ -264,7 +267,22 @@ def app_auth(request, app: Literal["authortools", "bibxml"]):
             json.dumps({'success': True}),
             content_type='application/json')
 
-
+@requires_api_token
+@csrf_exempt
+def nfs_metrics(request):
+    with NamedTemporaryFile(dir=settings.NFS_METRICS_TMP_DIR,delete=False) as fp:
+        fp.close()
+        mark = datetime.datetime.now()
+        with open(fp.name, mode="w") as f:
+            f.write("whyioughta"*1024)
+        write_latency = (datetime.datetime.now() - mark).total_seconds()
+        mark = datetime.datetime.now()
+        with open(fp.name, "r") as f:
+            _=f.read()
+        read_latency = (datetime.datetime.now() - mark).total_seconds()
+        Path(f.name).unlink()
+    response=f'nfs_latency_seconds{{operation="write"}} {write_latency}\nnfs_latency_seconds{{operation="read"}} {read_latency}\n'
+    return HttpResponse(response)
 
 def find_doc_for_rfcdiff(name, rev):
     """rfcdiff lookup heuristics
diff --git a/ietf/settings.py b/ietf/settings.py
index efd04c6068..b452864be6 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -761,6 +761,7 @@ def skip_unreadable_post(record):
 DERIVED_DIR = '/a/ietfdata/derived'
 FTP_DIR = '/a/ftp'
 ALL_ID_DOWNLOAD_DIR = '/a/www/www6s/download'
+NFS_METRICS_TMP_DIR = '/a/tmp'
 
 DOCUMENT_FORMAT_ALLOWLIST = ["txt", "ps", "pdf", "xml", "html", ]
 

From e108a3ca2c81b31c23775bba2f443bd1d2d15d23 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 10 Jan 2025 15:05:44 -0500
Subject: [PATCH 222/656] chore(k8s): add inter-pod affinity

---
 k8s/auth.yaml      | 10 ++++++++++
 k8s/beat.yaml      | 10 ++++++++++
 k8s/celery.yaml    | 10 ++++++++++
 k8s/memcached.yaml | 10 ++++++++++
 k8s/rabbitmq.yaml  | 10 ++++++++++
 5 files changed, 50 insertions(+)

diff --git a/k8s/auth.yaml b/k8s/auth.yaml
index 2bdb064447..392e306b54 100644
--- a/k8s/auth.yaml
+++ b/k8s/auth.yaml
@@ -15,6 +15,16 @@ spec:
       labels:
         app: auth
     spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - datatracker
+              topologyKey: "kubernetes.io/hostname"
       securityContext:
         runAsNonRoot: true
       containers:
diff --git a/k8s/beat.yaml b/k8s/beat.yaml
index 72d74e11e4..9a8fe2f0a4 100644
--- a/k8s/beat.yaml
+++ b/k8s/beat.yaml
@@ -17,6 +17,16 @@ spec:
       labels:
         app: beat
     spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - datatracker
+              topologyKey: "kubernetes.io/hostname"
       securityContext:
         runAsNonRoot: true
       containers:
diff --git a/k8s/celery.yaml b/k8s/celery.yaml
index 10f58f0161..15f3bf0c7c 100644
--- a/k8s/celery.yaml
+++ b/k8s/celery.yaml
@@ -17,6 +17,16 @@ spec:
       labels:
         app: celery
     spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - datatracker
+              topologyKey: "kubernetes.io/hostname"
       securityContext:
         runAsNonRoot: true
       containers:
diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml
index 5a4c9f0aed..8f73f3d0d5 100644
--- a/k8s/memcached.yaml
+++ b/k8s/memcached.yaml
@@ -13,6 +13,16 @@ spec:
       labels:
         app: memcached
     spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - datatracker
+              topologyKey: "kubernetes.io/hostname"
       securityContext:
         runAsNonRoot: true
       containers:
diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml
index b016b3a5ab..3cab7ff565 100644
--- a/k8s/rabbitmq.yaml
+++ b/k8s/rabbitmq.yaml
@@ -13,6 +13,16 @@ spec:
       labels:
         app: rabbitmq
     spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - datatracker
+              topologyKey: "kubernetes.io/hostname"
       securityContext:
         runAsNonRoot: true
       containers:

From e1af5e7049f272556bcc5974c5f2d3c81fab9721 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 10 Jan 2025 19:02:47 -0400
Subject: [PATCH 223/656] chore: remove dumprelated.py and loadrelated.py
 (#8412)

* fix: correctly disconnect post_save signal

* chore: remove dumprelated.py and loadrelated.py

These have not been used in quite some time and would need
a careful review before trusting. Taking them out to avoid
accidents.
---
 ietf/utils/management/commands/dumprelated.py | 209 ------------------
 ietf/utils/management/commands/loadrelated.py | 129 -----------
 2 files changed, 338 deletions(-)
 delete mode 100644 ietf/utils/management/commands/dumprelated.py
 delete mode 100644 ietf/utils/management/commands/loadrelated.py

diff --git a/ietf/utils/management/commands/dumprelated.py b/ietf/utils/management/commands/dumprelated.py
deleted file mode 100644
index 66fbb33bf1..0000000000
--- a/ietf/utils/management/commands/dumprelated.py
+++ /dev/null
@@ -1,209 +0,0 @@
-# Copyright The IETF Trust 2018-2020, All Rights Reserved
-# -*- coding: utf-8 -*-
-
-
-import io
-import warnings
-from collections import OrderedDict
-
-from django.apps import apps
-from django.contrib.admin.utils import NestedObjects
-from django.core import serializers
-from django.core.management.base import BaseCommand, CommandError
-from django.core.management.utils import parse_apps_and_model_labels
-from django.db import DEFAULT_DB_ALIAS, router
-
-import debug                            # pyflakes:ignore
-debug.debug = True
-
-class ProxyModelWarning(Warning):
-    pass
-
-
-class Command(BaseCommand):
-    help = (
-        "Output a database object and its related objects as a fixture of the given format "
-    )
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            'args', metavar='app_label.ModelName', nargs=1,
-            help='Specifies the app_label.ModelName for which to dump objects given by --pks',
-        )
-        parser.add_argument(
-            '--format', default='json', dest='format',
-            help='Specifies the output serialization format for fixtures.',
-        )
-        parser.add_argument(
-            '--indent', default=None, dest='indent', type=int,
-            help='Specifies the indent level to use when pretty-printing output.',
-        )
-        parser.add_argument(
-            '--database', action='store', dest='database',
-            default=DEFAULT_DB_ALIAS,
-            help='Nominates a specific database to dump fixtures from. '
-                 'Defaults to the "default" database.',
-        )
-        parser.add_argument(
-            '-e', '--exclude', dest='exclude', action='append', default=[],
-            help='An app_label or app_label.ModelName to exclude '
-                 '(use multiple --exclude to exclude multiple apps/models).',
-        )
-        parser.add_argument(
-            '--natural-foreign', action='store_true', dest='use_natural_foreign_keys', default=False,
-            help='Use natural foreign keys if they are available.',
-        )
-        parser.add_argument(
-            '--natural-primary', action='store_true', dest='use_natural_primary_keys', default=False,
-            help='Use natural primary keys if they are available.',
-        )
-        parser.add_argument(
-            '-o', '--output', default=None, dest='output',
-            help='Specifies file to which the output is written.'
-        )
-        parser.add_argument(
-            '--pks', dest='primary_keys', required=True,
-            help="Only dump objects with given primary keys. Accepts a comma-separated "
-                 "list of keys. This option only works when you specify one model.",
-        )
-
-    def handle(self, *app_labels, **options):
-        format = options['format']
-        indent = options['indent']
-        using = options['database']
-        excludes = options['exclude']
-        output = options['output']
-        show_traceback = options['traceback']
-        use_natural_foreign_keys = options['use_natural_foreign_keys']
-        use_natural_primary_keys = options['use_natural_primary_keys']
-        pks = options['primary_keys']
-
-        if pks:
-            primary_keys = [pk.strip() for pk in pks.split(',')]
-        else:
-            primary_keys = []
-
-        excluded_models, excluded_apps = parse_apps_and_model_labels(excludes)
-
-        if len(app_labels) == 0:
-            if primary_keys:
-                raise CommandError("You can only use --pks option with one model")
-            app_list = OrderedDict(
-                (app_config, None) for app_config in apps.get_app_configs()
-                if app_config.models_module is not None and app_config not in excluded_apps
-            )
-        else:
-            if len(app_labels) > 1 and primary_keys:
-                raise CommandError("You can only use --pks option with one model")
-            app_list = OrderedDict()
-            for label in app_labels:
-                try:
-                    app_label, model_label = label.split('.')
-                    try:
-                        app_config = apps.get_app_config(app_label)
-                    except LookupError as e:
-                        raise CommandError(str(e))
-                    if app_config.models_module is None or app_config in excluded_apps:
-                        continue
-                    try:
-                        model = app_config.get_model(model_label)
-                    except LookupError:
-                        raise CommandError("Unknown model: %s.%s" % (app_label, model_label))
-
-                    app_list_value = app_list.setdefault(app_config, [])
-
-                    # We may have previously seen a "all-models" request for
-                    # this app (no model qualifier was given). In this case
-                    # there is no need adding specific models to the list.
-                    if app_list_value is not None:
-                        if model not in app_list_value:
-                            app_list_value.append(model)
-                except ValueError:
-                    if primary_keys:
-                        raise CommandError("You can only use --pks option with one model")
-                    # This is just an app - no model qualifier
-                    app_label = label
-                    try:
-                        app_config = apps.get_app_config(app_label)
-                    except LookupError as e:
-                        raise CommandError(str(e))
-                    if app_config.models_module is None or app_config in excluded_apps:
-                        continue
-                    app_list[app_config] = None
-
-        # Check that the serialization format exists; this is a shortcut to
-        # avoid collating all the objects and _then_ failing.
-        if format not in serializers.get_public_serializer_formats():
-            try:
-                serializers.get_serializer(format)
-            except serializers.SerializerDoesNotExist:
-                pass
-
-            raise CommandError("Unknown serialization format: %s" % format)
-
-        def flatten(l):
-            if isinstance(l, list):
-                for el in l:
-                    if isinstance(el, list):
-                        for sub in flatten(el):
-                            yield sub
-                    else:
-                        yield el
-            else:
-                yield l
-
-        def get_objects(count_only=False):
-            """
-            Collate the objects to be serialized. If count_only is True, just
-            count the number of objects to be serialized.
-            """
-            models = serializers.sort_dependencies(list(app_list.items()))
-            for model in models:
-                if model in excluded_models:
-                    continue
-                if model._meta.proxy and model._meta.proxy_for_model not in models:
-                    warnings.warn(
-                        "%s is a proxy model and won't be serialized." % model._meta.label,
-                        category=ProxyModelWarning,
-                    )
-                if not model._meta.proxy and router.allow_migrate_model(using, model):
-                    objects = model._default_manager
-
-                    queryset = objects.using(using).order_by(model._meta.pk.name)
-                    if primary_keys:
-                        queryset = queryset.filter(pk__in=primary_keys)
-                    if count_only:
-                        yield queryset.order_by().count()
-                    else:
-                        for obj in queryset.iterator():
-                            collector = NestedObjects(using=using)
-                            collector.collect([obj,])
-                            object_list = list(flatten(collector.nested()))
-                            object_list.reverse()
-                            for o in object_list:
-                                yield o
-
-        try:
-            self.stdout.ending = None
-            progress_output = None
-            object_count = 0
-            # If dumpdata is outputting to stdout, there is no way to display progress
-            if (output and self.stdout.isatty() and options['verbosity'] > 0):
-                progress_output = self.stdout
-                object_count = sum(get_objects(count_only=True))
-            stream = io.open(output, 'w') if output else None
-            try:
-                serializers.serialize(
-                    format, get_objects(), indent=indent,
-                    use_natural_foreign_keys=use_natural_foreign_keys,
-                    use_natural_primary_keys=use_natural_primary_keys,
-                    stream=stream or self.stdout, progress_output=progress_output,
-                    object_count=object_count,
-                )
-            finally:
-                if stream:
-                    stream.close()
-        except Exception as e:
-            if show_traceback:
-                raise
-            raise CommandError("Unable to serialize database: %s" % e)
diff --git a/ietf/utils/management/commands/loadrelated.py b/ietf/utils/management/commands/loadrelated.py
deleted file mode 100644
index d8ae19dc77..0000000000
--- a/ietf/utils/management/commands/loadrelated.py
+++ /dev/null
@@ -1,129 +0,0 @@
-# Copyright The IETF Trust 2018-2020, All Rights Reserved
-# -*- coding: utf-8 -*-
-
-
-import gzip
-import os
-#import sys
-import tqdm
-import zipfile
-
-try:
-    import bz2
-    has_bz2 = True
-except ImportError:
-    has_bz2 = False
-
-from django.core.exceptions import ObjectDoesNotExist
-from django.core import serializers
-from django.db import DEFAULT_DB_ALIAS, DatabaseError, IntegrityError, connections
-from django.db.models.signals import post_save
-from django.utils.encoding import force_str
-import django.core.management.commands.loaddata as loaddata
-
-import debug                            # pyflakes:ignore
-
-from ietf.community.signals import notify_of_events_receiver
-
-class Command(loaddata.Command):
-    help = ("""
-
-        Load a fixture of related objects to the database.  The fixture is expected
-        to contain a set of related objects, created with the 'dumprelated' management
-        command.  It differs from the 'loaddata' command in that it silently ignores
-        attempts to load duplicate entries, and continues loading subsequent entries.
-
-        """)
-
-    def add_arguments(self, parser):
-        parser.add_argument('args', metavar='fixture', nargs='+', help='Fixture files.')
-        parser.add_argument(
-            '--database', action='store', dest='database', default=DEFAULT_DB_ALIAS,
-            help='Nominates a specific database to load fixtures into. Defaults to the "default" database.',
-        )
-        parser.add_argument(
-            '--ignorenonexistent', '-i', action='store_true', dest='ignore', default=False,
-            help='Ignores entries in the serialized data for fields that do not '
-                 'currently exist on the model.',
-        )
-
-    def handle(self, *args, **options):
-        self.ignore = options['ignore']
-        self.using = options['database']
-        self.verbosity = options['verbosity']
-        #
-        self.compression_formats = {
-            None: (open, 'rb'),
-            'gz': (gzip.GzipFile, 'rb'),
-            'zip': (SingleZipReader, 'r'),
-        }
-        if has_bz2:
-            self.compression_formats['bz2'] = (bz2.BZ2File, 'r')
-        #
-        self.serialization_formats = serializers.get_public_serializer_formats()
-        #
-        post_save.disconnect(notify_of_events_receiver())
-        #
-        connection = connections[self.using]
-        self.fixture_count = 0
-        self.loaded_object_count = 0
-        self.fixture_object_count = 0
-        #
-        for arg in args:
-            fixture_file = arg
-            self.stdout.write("Loading objects from %s" % fixture_file)
-            _, ser_fmt, cmp_fmt = self.parse_name(os.path.basename(fixture_file))
-            open_method, mode = self.compression_formats[cmp_fmt]
-            fixture = open_method(fixture_file, mode)
-            objects_in_fixture = 0
-            self.stdout.write("Getting object count...\b\b\b", ending='')
-            self.stdout.flush()
-            for o in serializers.deserialize(ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore,):
-                objects_in_fixture += 1
-            self.stdout.write(" %d" % objects_in_fixture)
-            #
-            fixture = open_method(fixture_file, mode)
-            self.fixture_count += 1
-            objects = serializers.deserialize(ser_fmt, fixture, using=self.using, ignorenonexistent=self.ignore,)
-            with connection.constraint_checks_disabled():
-                for obj in tqdm.tqdm(objects, total=objects_in_fixture):
-                    try:
-                        obj.save(using=self.using)
-                        self.loaded_object_count += 1
-                    except (DatabaseError, IntegrityError, ObjectDoesNotExist, AttributeError) as e:
-                        error_msg = force_str(e)
-                        if "Duplicate entry" in error_msg:
-                            pass
-                        else:
-                            self.stderr.write("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % {
-                                'app_label': obj.object._meta.app_label,
-                                'object_name': obj.object._meta.object_name,
-                                'pk': obj.object.pk,
-                                'error_msg': error_msg,
-                            }, )
-            self.fixture_object_count += objects_in_fixture
-
-        if self.verbosity >= 1:
-            if self.fixture_object_count == self.loaded_object_count:
-                self.stdout.write(
-                    "Installed %d object(s) from %d fixture(s)"
-                    % (self.loaded_object_count, self.fixture_count)
-                )
-            else:
-                self.stdout.write(
-                    "Installed %d object(s) (of %d) from %d fixture(s)"
-                    % (self.loaded_object_count, self.fixture_object_count, self.fixture_count)
-                )
-
-        
-class SingleZipReader(zipfile.ZipFile):
-
-    def __init__(self, *args, **kwargs):
-        zipfile.ZipFile.__init__(self, *args, **kwargs)
-        if len(self.namelist()) != 1:
-            raise ValueError("Zip-compressed fixtures must contain one file.")
-
-    def read(self):
-        return zipfile.ZipFile.read(self, self.namelist()[0])
-
-

From 56985d105651b7d0ca30345dcb8472c5975f787b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Jan 2025 15:01:32 -0500
Subject: [PATCH 224/656] chore(deps): bump ncipollo/release-action from 1.14.0
 to 1.15.0 (#8420)

Bumps [ncipollo/release-action](https://github.com/ncipollo/release-action) from 1.14.0 to 1.15.0.
- [Release notes](https://github.com/ncipollo/release-action/releases)
- [Commits](https://github.com/ncipollo/release-action/compare/v1.14.0...v1.15.0)

---
updated-dependencies:
- dependency-name: ncipollo/release-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 46cb150e05..4771adde97 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -97,7 +97,7 @@ jobs:
         echo "IS_RELEASE=true" >> $GITHUB_ENV
         
     - name: Create Draft Release
-      uses: ncipollo/release-action@v1.14.0
+      uses: ncipollo/release-action@v1.15.0
       if: ${{ github.ref_name == 'release' }}
       with:
         prerelease: true
@@ -311,7 +311,7 @@ jobs:
         histCoveragePath: historical-coverage.json
         
     - name: Create Release
-      uses: ncipollo/release-action@v1.14.0
+      uses: ncipollo/release-action@v1.15.0
       if: ${{ env.SHOULD_DEPLOY == 'true' }}
       with:
         allowUpdates: true
@@ -324,7 +324,7 @@ jobs:
         token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Update Baseline Coverage
-      uses: ncipollo/release-action@v1.14.0
+      uses: ncipollo/release-action@v1.15.0
       if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }}
       with:
         allowUpdates: true

From 135f759b3065e66c45862e9f6ad832084edb3c76 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Jan 2025 15:02:45 -0500
Subject: [PATCH 225/656] chore(deps): bump dockerode in
 /dev/deploy-to-container in the npm group (#8393)

Bumps the npm group in /dev/deploy-to-container with 1 update: [dockerode](https://github.com/apocas/dockerode).


Updates `dockerode` from 4.0.2 to 4.0.3
- [Release notes](https://github.com/apocas/dockerode/releases)
- [Commits](https://github.com/apocas/dockerode/compare/v4.0.2...v4.0.3)

---
updated-dependencies:
- dependency-name: dockerode
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/deploy-to-container/package-lock.json | 394 +++++++++++++++++++---
 dev/deploy-to-container/package.json      |   2 +-
 2 files changed, 340 insertions(+), 56 deletions(-)

diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json
index 08b6eb6c62..86cb1f732f 100644
--- a/dev/deploy-to-container/package-lock.json
+++ b/dev/deploy-to-container/package-lock.json
@@ -6,7 +6,7 @@
     "": {
       "name": "deploy-to-container",
       "dependencies": {
-        "dockerode": "^4.0.2",
+        "dockerode": "^4.0.3",
         "fs-extra": "^11.2.0",
         "nanoid": "5.0.9",
         "nanoid-dictionary": "5.0.0-beta.1",
@@ -23,6 +23,35 @@
       "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz",
       "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q=="
     },
+    "node_modules/@grpc/grpc-js": {
+      "version": "1.12.5",
+      "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.12.5.tgz",
+      "integrity": "sha512-d3iiHxdpg5+ZcJ6jnDSOT8Z0O0VMVGy34jAnYLUX8yd36b1qn8f1TwOA/Lc7TsOh03IkPJ38eGI5qD2EjNkoEA==",
+      "dependencies": {
+        "@grpc/proto-loader": "^0.7.13",
+        "@js-sdsl/ordered-map": "^4.4.2"
+      },
+      "engines": {
+        "node": ">=12.10.0"
+      }
+    },
+    "node_modules/@grpc/proto-loader": {
+      "version": "0.7.13",
+      "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.13.tgz",
+      "integrity": "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==",
+      "dependencies": {
+        "lodash.camelcase": "^4.3.0",
+        "long": "^5.0.0",
+        "protobufjs": "^7.2.5",
+        "yargs": "^17.7.2"
+      },
+      "bin": {
+        "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/@isaacs/cliui": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
@@ -123,6 +152,15 @@
         "node": ">=18.0.0"
       }
     },
+    "node_modules/@js-sdsl/ordered-map": {
+      "version": "4.4.2",
+      "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz",
+      "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==",
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/js-sdsl"
+      }
+    },
     "node_modules/@pkgjs/parseargs": {
       "version": "0.11.0",
       "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
@@ -132,6 +170,68 @@
         "node": ">=14"
       }
     },
+    "node_modules/@protobufjs/aspromise": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
+      "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="
+    },
+    "node_modules/@protobufjs/base64": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
+      "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="
+    },
+    "node_modules/@protobufjs/codegen": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
+      "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="
+    },
+    "node_modules/@protobufjs/eventemitter": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
+      "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="
+    },
+    "node_modules/@protobufjs/fetch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
+      "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.1",
+        "@protobufjs/inquire": "^1.1.0"
+      }
+    },
+    "node_modules/@protobufjs/float": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
+      "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="
+    },
+    "node_modules/@protobufjs/inquire": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
+      "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="
+    },
+    "node_modules/@protobufjs/path": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
+      "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="
+    },
+    "node_modules/@protobufjs/pool": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
+      "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="
+    },
+    "node_modules/@protobufjs/utf8": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
+      "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="
+    },
+    "node_modules/@types/node": {
+      "version": "22.10.5",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.5.tgz",
+      "integrity": "sha512-F8Q+SeGimwOo86fiovQh8qiXfFEh2/ocYv7tU5pJ3EXMSSxk1Joj5wefpFK2fHTf/N6HKGSxIDBT9f3gCxXPkQ==",
+      "dependencies": {
+        "undici-types": "~6.20.0"
+      }
+    },
     "node_modules/ansi-regex": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
@@ -246,14 +346,14 @@
       "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
     },
     "node_modules/cpu-features": {
-      "version": "0.0.9",
-      "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.9.tgz",
-      "integrity": "sha512-AKjgn2rP2yJyfbepsmLfiYcmtNn/2eUvocUyM/09yB0YDiz39HteK/5/T4Onf0pmdYDMgkBoGvRLvEguzyL7wQ==",
+      "version": "0.0.10",
+      "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz",
+      "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==",
       "hasInstallScript": true,
       "optional": true,
       "dependencies": {
         "buildcheck": "~0.0.6",
-        "nan": "^2.17.0"
+        "nan": "^2.19.0"
       },
       "engines": {
         "node": ">=10.0.0"
@@ -273,11 +373,11 @@
       }
     },
     "node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
+      "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
       "dependencies": {
-        "ms": "2.1.2"
+        "ms": "^2.1.3"
       },
       "engines": {
         "node": ">=6.0"
@@ -289,9 +389,9 @@
       }
     },
     "node_modules/docker-modem": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.3.tgz",
-      "integrity": "sha512-89zhop5YVhcPEt5FpUFGr3cDyceGhq/F9J+ZndQ4KfqNvfbJpPMfgeixFgUj5OjCYAboElqODxY5Z1EBsSa6sg==",
+      "version": "5.0.5",
+      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.5.tgz",
+      "integrity": "sha512-Cxw8uEcvNTRmsQuGqzzfiCnfGgf96tVJItLh8taOX0miTcIBALKH5TckCSuZbpbjP7uhAl81dOL9sxfa6HgCIg==",
       "dependencies": {
         "debug": "^4.1.1",
         "readable-stream": "^3.5.0",
@@ -303,13 +403,17 @@
       }
     },
     "node_modules/dockerode": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.2.tgz",
-      "integrity": "sha512-9wM1BVpVMFr2Pw3eJNXrYYt6DT9k0xMcsSCjtPvyQ+xa1iPg/Mo3T/gUcwI0B2cczqCeCYRPF8yFYDwtFXT0+w==",
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.3.tgz",
+      "integrity": "sha512-QSXJFcBQNaGZO6U3qWW4B7p8yRIJn/dWmvL2AQWfO/bjptBBO6QYdVkYSYFz9qoivP2jsOHZfmXMAfrK0BMKyg==",
       "dependencies": {
         "@balena/dockerignore": "^1.0.2",
-        "docker-modem": "^5.0.3",
-        "tar-fs": "~2.0.1"
+        "@grpc/grpc-js": "^1.11.1",
+        "@grpc/proto-loader": "^0.7.13",
+        "docker-modem": "^5.0.5",
+        "protobufjs": "^7.3.2",
+        "tar-fs": "~2.0.1",
+        "uuid": "^10.0.0"
       },
       "engines": {
         "node": ">= 8.0"
@@ -473,6 +577,16 @@
         "graceful-fs": "^4.1.6"
       }
     },
+    "node_modules/lodash.camelcase": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
+      "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
+    },
+    "node_modules/long": {
+      "version": "5.2.4",
+      "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz",
+      "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg=="
+    },
     "node_modules/lru-cache": {
       "version": "10.2.2",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz",
@@ -535,14 +649,14 @@
       "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
     },
     "node_modules/ms": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
     "node_modules/nan": {
-      "version": "2.18.0",
-      "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz",
-      "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==",
+      "version": "2.22.0",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz",
+      "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==",
       "optional": true
     },
     "node_modules/nanoid": {
@@ -598,6 +712,29 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/protobufjs": {
+      "version": "7.4.0",
+      "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz",
+      "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==",
+      "hasInstallScript": true,
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.2",
+        "@protobufjs/base64": "^1.1.2",
+        "@protobufjs/codegen": "^2.0.4",
+        "@protobufjs/eventemitter": "^1.1.0",
+        "@protobufjs/fetch": "^1.1.0",
+        "@protobufjs/float": "^1.0.2",
+        "@protobufjs/inquire": "^1.1.0",
+        "@protobufjs/path": "^1.1.2",
+        "@protobufjs/pool": "^1.1.0",
+        "@protobufjs/utf8": "^1.1.0",
+        "@types/node": ">=13.7.0",
+        "long": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/pump": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
@@ -713,9 +850,9 @@
       "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ=="
     },
     "node_modules/ssh2": {
-      "version": "1.15.0",
-      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.15.0.tgz",
-      "integrity": "sha512-C0PHgX4h6lBxYx7hcXwu3QWdh4tg6tZZsTfXcdvc5caW/EMxaB4H9dWsl7qk+F7LAW762hp8VbXOX7x4xUYvEw==",
+      "version": "1.16.0",
+      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz",
+      "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==",
       "hasInstallScript": true,
       "dependencies": {
         "asn1": "^0.2.6",
@@ -725,8 +862,8 @@
         "node": ">=10.16.0"
       },
       "optionalDependencies": {
-        "cpu-features": "~0.0.9",
-        "nan": "^2.18.0"
+        "cpu-features": "~0.0.10",
+        "nan": "^2.20.0"
       }
     },
     "node_modules/string_decoder": {
@@ -875,6 +1012,11 @@
       "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
       "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA=="
     },
+    "node_modules/undici-types": {
+      "version": "6.20.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
+      "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="
+    },
     "node_modules/universalify": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
@@ -888,6 +1030,18 @@
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
       "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
     },
+    "node_modules/uuid": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
+      "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",
+      "funding": [
+        "https://github.com/sponsors/broofa",
+        "https://github.com/sponsors/ctavan"
+      ],
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
     "node_modules/which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@@ -988,6 +1142,26 @@
       "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz",
       "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q=="
     },
+    "@grpc/grpc-js": {
+      "version": "1.12.5",
+      "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.12.5.tgz",
+      "integrity": "sha512-d3iiHxdpg5+ZcJ6jnDSOT8Z0O0VMVGy34jAnYLUX8yd36b1qn8f1TwOA/Lc7TsOh03IkPJ38eGI5qD2EjNkoEA==",
+      "requires": {
+        "@grpc/proto-loader": "^0.7.13",
+        "@js-sdsl/ordered-map": "^4.4.2"
+      }
+    },
+    "@grpc/proto-loader": {
+      "version": "0.7.13",
+      "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.13.tgz",
+      "integrity": "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==",
+      "requires": {
+        "lodash.camelcase": "^4.3.0",
+        "long": "^5.0.0",
+        "protobufjs": "^7.2.5",
+        "yargs": "^17.7.2"
+      }
+    },
     "@isaacs/cliui": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
@@ -1054,12 +1228,79 @@
         "minipass": "^7.0.4"
       }
     },
+    "@js-sdsl/ordered-map": {
+      "version": "4.4.2",
+      "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz",
+      "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw=="
+    },
     "@pkgjs/parseargs": {
       "version": "0.11.0",
       "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
       "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
       "optional": true
     },
+    "@protobufjs/aspromise": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
+      "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="
+    },
+    "@protobufjs/base64": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
+      "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="
+    },
+    "@protobufjs/codegen": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
+      "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="
+    },
+    "@protobufjs/eventemitter": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
+      "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="
+    },
+    "@protobufjs/fetch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
+      "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
+      "requires": {
+        "@protobufjs/aspromise": "^1.1.1",
+        "@protobufjs/inquire": "^1.1.0"
+      }
+    },
+    "@protobufjs/float": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
+      "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="
+    },
+    "@protobufjs/inquire": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
+      "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="
+    },
+    "@protobufjs/path": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
+      "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="
+    },
+    "@protobufjs/pool": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
+      "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="
+    },
+    "@protobufjs/utf8": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
+      "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="
+    },
+    "@types/node": {
+      "version": "22.10.5",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.5.tgz",
+      "integrity": "sha512-F8Q+SeGimwOo86fiovQh8qiXfFEh2/ocYv7tU5pJ3EXMSSxk1Joj5wefpFK2fHTf/N6HKGSxIDBT9f3gCxXPkQ==",
+      "requires": {
+        "undici-types": "~6.20.0"
+      }
+    },
     "ansi-regex": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
@@ -1142,13 +1383,13 @@
       "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
     },
     "cpu-features": {
-      "version": "0.0.9",
-      "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.9.tgz",
-      "integrity": "sha512-AKjgn2rP2yJyfbepsmLfiYcmtNn/2eUvocUyM/09yB0YDiz39HteK/5/T4Onf0pmdYDMgkBoGvRLvEguzyL7wQ==",
+      "version": "0.0.10",
+      "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz",
+      "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==",
       "optional": true,
       "requires": {
         "buildcheck": "~0.0.6",
-        "nan": "^2.17.0"
+        "nan": "^2.19.0"
       }
     },
     "cross-spawn": {
@@ -1162,17 +1403,17 @@
       }
     },
     "debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
+      "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
       "requires": {
-        "ms": "2.1.2"
+        "ms": "^2.1.3"
       }
     },
     "docker-modem": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.3.tgz",
-      "integrity": "sha512-89zhop5YVhcPEt5FpUFGr3cDyceGhq/F9J+ZndQ4KfqNvfbJpPMfgeixFgUj5OjCYAboElqODxY5Z1EBsSa6sg==",
+      "version": "5.0.5",
+      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.5.tgz",
+      "integrity": "sha512-Cxw8uEcvNTRmsQuGqzzfiCnfGgf96tVJItLh8taOX0miTcIBALKH5TckCSuZbpbjP7uhAl81dOL9sxfa6HgCIg==",
       "requires": {
         "debug": "^4.1.1",
         "readable-stream": "^3.5.0",
@@ -1181,13 +1422,17 @@
       }
     },
     "dockerode": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.2.tgz",
-      "integrity": "sha512-9wM1BVpVMFr2Pw3eJNXrYYt6DT9k0xMcsSCjtPvyQ+xa1iPg/Mo3T/gUcwI0B2cczqCeCYRPF8yFYDwtFXT0+w==",
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.3.tgz",
+      "integrity": "sha512-QSXJFcBQNaGZO6U3qWW4B7p8yRIJn/dWmvL2AQWfO/bjptBBO6QYdVkYSYFz9qoivP2jsOHZfmXMAfrK0BMKyg==",
       "requires": {
         "@balena/dockerignore": "^1.0.2",
-        "docker-modem": "^5.0.3",
-        "tar-fs": "~2.0.1"
+        "@grpc/grpc-js": "^1.11.1",
+        "@grpc/proto-loader": "^0.7.13",
+        "docker-modem": "^5.0.5",
+        "protobufjs": "^7.3.2",
+        "tar-fs": "~2.0.1",
+        "uuid": "^10.0.0"
       }
     },
     "eastasianwidth": {
@@ -1297,6 +1542,16 @@
         "universalify": "^2.0.0"
       }
     },
+    "lodash.camelcase": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
+      "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
+    },
+    "long": {
+      "version": "5.2.4",
+      "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz",
+      "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg=="
+    },
     "lru-cache": {
       "version": "10.2.2",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz",
@@ -1335,14 +1590,14 @@
       "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
     },
     "ms": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
     "nan": {
-      "version": "2.18.0",
-      "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz",
-      "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==",
+      "version": "2.22.0",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz",
+      "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==",
       "optional": true
     },
     "nanoid": {
@@ -1377,6 +1632,25 @@
         "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0"
       }
     },
+    "protobufjs": {
+      "version": "7.4.0",
+      "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz",
+      "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==",
+      "requires": {
+        "@protobufjs/aspromise": "^1.1.2",
+        "@protobufjs/base64": "^1.1.2",
+        "@protobufjs/codegen": "^2.0.4",
+        "@protobufjs/eventemitter": "^1.1.0",
+        "@protobufjs/fetch": "^1.1.0",
+        "@protobufjs/float": "^1.0.2",
+        "@protobufjs/inquire": "^1.1.0",
+        "@protobufjs/path": "^1.1.2",
+        "@protobufjs/pool": "^1.1.0",
+        "@protobufjs/utf8": "^1.1.0",
+        "@types/node": ">=13.7.0",
+        "long": "^5.0.0"
+      }
+    },
     "pump": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
@@ -1448,14 +1722,14 @@
       "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ=="
     },
     "ssh2": {
-      "version": "1.15.0",
-      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.15.0.tgz",
-      "integrity": "sha512-C0PHgX4h6lBxYx7hcXwu3QWdh4tg6tZZsTfXcdvc5caW/EMxaB4H9dWsl7qk+F7LAW762hp8VbXOX7x4xUYvEw==",
+      "version": "1.16.0",
+      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz",
+      "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==",
       "requires": {
         "asn1": "^0.2.6",
         "bcrypt-pbkdf": "^1.0.2",
-        "cpu-features": "~0.0.9",
-        "nan": "^2.18.0"
+        "cpu-features": "~0.0.10",
+        "nan": "^2.20.0"
       }
     },
     "string_decoder": {
@@ -1571,6 +1845,11 @@
       "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
       "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA=="
     },
+    "undici-types": {
+      "version": "6.20.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
+      "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="
+    },
     "universalify": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
@@ -1581,6 +1860,11 @@
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
       "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
     },
+    "uuid": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
+      "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ=="
+    },
     "which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json
index 8f16279cea..53a362b9d0 100644
--- a/dev/deploy-to-container/package.json
+++ b/dev/deploy-to-container/package.json
@@ -2,7 +2,7 @@
   "name": "deploy-to-container",
   "type": "module",
   "dependencies": {
-    "dockerode": "^4.0.2",
+    "dockerode": "^4.0.3",
     "fs-extra": "^11.2.0",
     "nanoid": "5.0.9",
     "nanoid-dictionary": "5.0.0-beta.1",

From 1ffd205ef065ec5cd5fd66ddea55e2a702650579 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 14 Jan 2025 08:48:55 -0600
Subject: [PATCH 226/656] fix: rsync to the correct dest (#8421)

---
 ietf/submit/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index 16cccc9b59..49f28c4196 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -1598,5 +1598,7 @@ def active(dirent):
         except UnicodeDecodeError as e:
             log.log(f"Error processing {item.name}: {e}")
 
-    ftp_moddir = Path(settings.FTP_DIR) / "yang" / "draftmod"
+    ftp_moddir = Path(settings.FTP_DIR) / "yang" / "draftmod/"
+    if not moddir.endswith("/"):
+        moddir += "/"
     subprocess.call(("/usr/bin/rsync", "-aq", "--delete", moddir, ftp_moddir))

From d1d33e7ad50d6c7625541c0966023c2cdfe77c16 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Thu, 16 Jan 2025 20:44:55 +0000
Subject: [PATCH 227/656] ci: update base image target version to 20250116T2033

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index a923bf693f..2f35125564 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20241212T1741
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250116T2033
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index b5d33714f2..fd2d539a9e 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20241212T1741
+20250116T2033

From df27ba9934c15b02ad93ec3c6eb02ee0705730b4 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 16 Jan 2025 16:21:42 -0500
Subject: [PATCH 228/656] ci: Update ci-run-tests.yml

---
 .github/workflows/ci-run-tests.yml | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml
index 9121bf8aea..278bd8af2f 100644
--- a/.github/workflows/ci-run-tests.yml
+++ b/.github/workflows/ci-run-tests.yml
@@ -13,7 +13,34 @@ on:
       - 'package.json'
 
 jobs:
+  # -----------------------------------------------------------------
+  # PREPARE
+  # -----------------------------------------------------------------
+  prepare:
+    name: Prepare
+    runs-on: ubuntu-latest
+    outputs:
+      base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }}
+
+    steps:   
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+        fetch-tags: false
+
+    - name: Get Base Image Target Version
+      id: baseimgversion
+      run: |
+        echo "base_image_version=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT
+
+  # -----------------------------------------------------------------
+  # TESTS
+  # -----------------------------------------------------------------
   tests:
+    name: Run Tests
     uses: ./.github/workflows/tests.yml
+    needs: [prepare]
     with:
-      ignoreLowerCoverage: false
\ No newline at end of file
+      ignoreLowerCoverage: false
+      skipSelenium: true
+      targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }}

From c848a5a00bebef2662d0e189fe0f230333a71229 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 17 Jan 2025 11:16:15 -0400
Subject: [PATCH 229/656] feat: async investigate_fragment task; celery results
 backend (#8428)

* feat: investigate docs asynchronously

* refactor: move script to its own js file

* fix: adjust polling interval/duration

* test: test new task

* fix: extra tag/fix whitespace

* style: restore whitespace (I hope)

* style: black/standard styling

* test: fix test of investigate view

* test: improve/delint tests
---
 ietf/doc/forms.py                   |   1 +
 ietf/doc/tasks.py                   |   9 ++
 ietf/doc/tests.py                   | 134 +++++++++++++++--
 ietf/doc/tests_tasks.py             |  13 ++
 ietf/doc/views_doc.py               |  65 ++++++++-
 ietf/settings.py                    |  15 +-
 ietf/static/js/investigate.js       |  53 +++++++
 ietf/templates/doc/investigate.html | 218 +++++++++++++++-------------
 package.json                        |   1 +
 requirements.txt                    |   1 +
 10 files changed, 386 insertions(+), 124 deletions(-)
 create mode 100644 ietf/static/js/investigate.js

diff --git a/ietf/doc/forms.py b/ietf/doc/forms.py
index f77b218318..8a1e9ecb98 100644
--- a/ietf/doc/forms.py
+++ b/ietf/doc/forms.py
@@ -276,6 +276,7 @@ class InvestigateForm(forms.Form):
         ),
         min_length=8,
     )
+    task_id = forms.CharField(required=False, widget=forms.HiddenInput)
 
     def clean_name_fragment(self):
         disallowed_characters = ["%", "/", "\\", "*"]
diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py
index f1de459dd8..6eb901e6c7 100644
--- a/ietf/doc/tasks.py
+++ b/ietf/doc/tasks.py
@@ -31,6 +31,7 @@
     generate_idnits2_rfcs_obsoleted,
     update_or_create_draft_bibxml_file,
     ensure_draft_bibxml_path_exists,
+    investigate_fragment,
 )
 
 
@@ -119,3 +120,11 @@ def generate_draft_bibxml_files_task(days=7, process_all=False):
             update_or_create_draft_bibxml_file(event.doc, event.rev)
         except Exception as err:
             log.log(f"Error generating bibxml for {event.doc.name}-{event.rev}: {err}")
+
+
+@shared_task(ignore_result=False)
+def investigate_fragment_task(name_fragment: str):
+    return {
+        "name_fragment": name_fragment,
+        "results": investigate_fragment(name_fragment),
+    }
diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index 0630fcd8d4..f5af7bb48b 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -3280,7 +3280,8 @@ def test_investigate_fragment(self):
             "draft-this-should-not-be-possible-00.txt",
         )
 
-    def test_investigate(self):
+    def test_investigate_get(self):
+        """GET with no querystring should retrieve the investigate UI"""
         url = urlreverse("ietf.doc.views_doc.investigate")
         login_testing_unauthorized(self, "secretary", url)
         r = self.client.get(url)
@@ -3288,36 +3289,143 @@ def test_investigate(self):
         q = PyQuery(r.content)
         self.assertEqual(len(q("form#investigate")), 1)
         self.assertEqual(len(q("div#results")), 0)
-        r = self.client.post(url, dict(name_fragment="this-is-not-found"))
+
+    @mock.patch("ietf.doc.views_doc.AsyncResult")
+    def test_investgate_get_task_id(self, mock_asyncresult):
+        """GET with querystring should lookup task status"""
+        url = urlreverse("ietf.doc.views_doc.investigate")
+        login_testing_unauthorized(self, "secretary", url)
+        mock_asyncresult.return_value.ready.return_value = True
+        r = self.client.get(url + "?id=a-task-id")
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.json(), {"status": "ready"})
+        self.assertTrue(mock_asyncresult.called)
+        self.assertEqual(mock_asyncresult.call_args, mock.call("a-task-id"))
+        mock_asyncresult.reset_mock()
+
+        mock_asyncresult.return_value.ready.return_value = False
+        r = self.client.get(url + "?id=a-task-id")
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.json(), {"status": "notready"})
+        self.assertTrue(mock_asyncresult.called)
+        self.assertEqual(mock_asyncresult.call_args, mock.call("a-task-id"))
+
+    @mock.patch("ietf.doc.views_doc.investigate_fragment_task")
+    def test_investigate_post(self, mock_investigate_fragment_task):
+        """POST with a name_fragment and no task_id should start a celery task"""
+        url = urlreverse("ietf.doc.views_doc.investigate")
+        login_testing_unauthorized(self, "secretary", url)
+
+        # test some invalid cases
+        r = self.client.post(url, {"name_fragment": "short"})  # limit is >= 8 characters
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
+        self.assertEqual(len(q("#id_name_fragment.is-invalid")), 1)
+        self.assertFalse(mock_investigate_fragment_task.delay.called)
+        for char in ["*", "%", "/", "\\"]:
+            r = self.client.post(url, {"name_fragment": f"bad{char}character"})
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            self.assertEqual(len(q("#id_name_fragment.is-invalid")), 1)
+            self.assertFalse(mock_investigate_fragment_task.delay.called)
+        
+        # now a valid one
+        mock_investigate_fragment_task.delay.return_value.id = "a-task-id"
+        r = self.client.post(url, {"name_fragment": "this-is-a-valid-fragment"})
+        self.assertEqual(r.status_code, 200)
+        self.assertTrue(mock_investigate_fragment_task.delay.called)
+        self.assertEqual(mock_investigate_fragment_task.delay.call_args, mock.call("this-is-a-valid-fragment"))
+        self.assertEqual(r.json(), {"id": "a-task-id"})
+
+    @mock.patch("ietf.doc.views_doc.AsyncResult")
+    def test_investigate_post_task_id(self, mock_asyncresult):
+        """POST with name_fragment and task_id should retrieve results"""
+        url = urlreverse("ietf.doc.views_doc.investigate")
+        login_testing_unauthorized(self, "secretary", url)
+
+        # First, test a non-successful result - this could be a failure or non-existent task id
+        mock_result = mock_asyncresult.return_value
+        mock_result.successful.return_value = False
+        r = self.client.post(url, {"name_fragment": "some-fragment", "task_id": "a-task-id"})
+        self.assertContains(r, "The investigation task failed.", status_code=200)
+        self.assertTrue(mock_asyncresult.called)
+        self.assertEqual(mock_asyncresult.call_args, mock.call("a-task-id"))
+        self.assertFalse(mock_result.get.called)
+        mock_asyncresult.reset_mock()
+        q = PyQuery(r.content)
+        self.assertEqual(q("#id_name_fragment").val(), "some-fragment")
+        self.assertEqual(q("#id_task_id").val(), "a-task-id")
+
+        # now the various successful result mixes
+        mock_result = mock_asyncresult.return_value
+        mock_result.successful.return_value = True
+        mock_result.get.return_value = {
+            "name_fragment": "different-fragment",
+            "results": {
+                "can_verify": set(),
+                "unverifiable_collections": set(),
+                "unexpected": set(),
+            }
+        }
+        r = self.client.post(url, {"name_fragment": "some-fragment", "task_id": "a-task-id"})
+        self.assertEqual(r.status_code, 200)
+        self.assertTrue(mock_asyncresult.called)
+        self.assertEqual(mock_asyncresult.call_args, mock.call("a-task-id"))
+        mock_asyncresult.reset_mock()
+        q = PyQuery(r.content)
+        self.assertEqual(q("#id_name_fragment").val(), "different-fragment", "name_fragment should be reset")
+        self.assertEqual(q("#id_task_id").val(), "", "task_id should be cleared")
         self.assertEqual(len(q("div#results")), 1)
         self.assertEqual(len(q("table#authenticated")), 0)
         self.assertEqual(len(q("table#unverifiable")), 0)
         self.assertEqual(len(q("table#unexpected")), 0)
-        r = self.client.post(url, dict(name_fragment="mixed-provenance"))
+
+        # This file was created in setUp. It allows the view to render properly
+        # but its location / content don't matter for this test otherwise.
+        a_file_that_exists = Path(settings.INTERNET_DRAFT_PATH) / "draft-this-is-active-00.txt"
+
+        mock_result.get.return_value = {
+            "name_fragment": "different-fragment",
+            "results": {
+                "can_verify": {a_file_that_exists},
+                "unverifiable_collections": {a_file_that_exists},
+                "unexpected": set(),
+            }
+        }
+        r = self.client.post(url, {"name_fragment": "some-fragment", "task_id": "a-task-id"})
         self.assertEqual(r.status_code, 200)
+        self.assertTrue(mock_asyncresult.called)
+        self.assertEqual(mock_asyncresult.call_args, mock.call("a-task-id"))
+        mock_asyncresult.reset_mock()
         q = PyQuery(r.content)
+        self.assertEqual(q("#id_name_fragment").val(), "different-fragment", "name_fragment should be reset")
+        self.assertEqual(q("#id_task_id").val(), "", "task_id should be cleared")
         self.assertEqual(len(q("div#results")), 1)
         self.assertEqual(len(q("table#authenticated")), 1)
         self.assertEqual(len(q("table#unverifiable")), 1)
         self.assertEqual(len(q("table#unexpected")), 0)
-        r = self.client.post(url, dict(name_fragment="not-be-possible"))
+
+        mock_result.get.return_value = {
+            "name_fragment": "different-fragment",
+            "results": {
+                "can_verify": set(),
+                "unverifiable_collections": set(),
+                "unexpected": {a_file_that_exists},
+            }
+        }
+        r = self.client.post(url, {"name_fragment": "some-fragment", "task_id": "a-task-id"})
         self.assertEqual(r.status_code, 200)
+        self.assertTrue(mock_asyncresult.called)
+        self.assertEqual(mock_asyncresult.call_args, mock.call("a-task-id"))
+        mock_asyncresult.reset_mock()
         q = PyQuery(r.content)
+        self.assertEqual(q("#id_name_fragment").val(), "different-fragment", "name_fragment should be reset")
+        self.assertEqual(q("#id_task_id").val(), "", "task_id should be cleared")
         self.assertEqual(len(q("div#results")), 1)
         self.assertEqual(len(q("table#authenticated")), 0)
         self.assertEqual(len(q("table#unverifiable")), 0)
         self.assertEqual(len(q("table#unexpected")), 1)
-        r = self.client.post(url, dict(name_fragment="short"))
-        self.assertEqual(r.status_code, 200)
-        q = PyQuery(r.content)
-        self.assertEqual(len(q("#id_name_fragment.is-invalid")), 1)
-        for char in ["*", "%", "/", "\\"]:
-            r = self.client.post(url, dict(name_fragment=f"bad{char}character"))
-            self.assertEqual(r.status_code, 200)
-            q = PyQuery(r.content)
-            self.assertEqual(len(q("#id_name_fragment.is-invalid")), 1)
+
 
 class LogIOErrorTests(TestCase):
 
diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py
index 67997acd85..8a6ffa8be1 100644
--- a/ietf/doc/tests_tasks.py
+++ b/ietf/doc/tests_tasks.py
@@ -20,6 +20,7 @@
     generate_draft_bibxml_files_task,
     generate_idnits2_rfcs_obsoleted_task,
     generate_idnits2_rfc_status_task,
+    investigate_fragment_task,
     notify_expirations_task,
 )
 
@@ -98,6 +99,18 @@ def test_expire_last_calls_task(self, mock_get_expired, mock_expire):
         self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1]))
         self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2]))
 
+    def test_investigate_fragment_task(self):
+        investigation_results = object()  # singleton
+        with mock.patch(
+            "ietf.doc.tasks.investigate_fragment", return_value=investigation_results
+        ) as mock_inv:
+            retval = investigate_fragment_task("some fragment")
+        self.assertTrue(mock_inv.called)
+        self.assertEqual(mock_inv.call_args, mock.call("some fragment"))
+        self.assertEqual(
+            retval, {"name_fragment": "some fragment", "results": investigation_results}
+        )
+
 
 class Idnits2SupportTests(TestCase):
     settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR']
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 9f7cf12bcb..591a72d907 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -41,10 +41,11 @@
 
 from pathlib import Path
 
+from celery.result import AsyncResult
 from django.core.cache import caches
 from django.core.exceptions import PermissionDenied
 from django.db.models import Max
-from django.http import HttpResponse, Http404, HttpResponseBadRequest
+from django.http import HttpResponse, Http404, HttpResponseBadRequest, JsonResponse
 from django.shortcuts import render, get_object_or_404, redirect
 from django.template.loader import render_to_string
 from django.urls import reverse as urlreverse
@@ -59,8 +60,9 @@
     ConsensusDocEvent, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent,
     IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS, DocumentActionHolder, DocumentAuthor,
     RelatedDocument, RelatedDocHistory)
+from ietf.doc.tasks import investigate_fragment_task
 from ietf.doc.utils import (augment_events_with_revision,
-    can_adopt_draft, can_unadopt_draft, get_chartering_type, get_tags_for_stream_id, investigate_fragment,
+    can_adopt_draft, can_unadopt_draft, get_chartering_type, get_tags_for_stream_id,
     needed_ballot_positions, nice_consensus, update_telechat, has_same_ballot,
     get_initial_notify, make_notify_changed_event, make_rev_history, default_consensus,
     add_events_message_info, get_unicode_document_content,
@@ -2275,16 +2277,67 @@ def idnits2_state(request, name, rev=None):
         content_type="text/plain;charset=utf-8",
     )
 
+
 @role_required("Secretariat")
 def investigate(request):
+    """Investigate a fragment
+
+    A plain GET with no querystring returns the UI page.
+
+    POST with the task_id field empty starts an async task and returns a JSON response with
+    the ID needed to monitor the task for results.
+
+    GET with a querystring parameter "id" will poll the status of the async task and return "ready"
+    or "notready".
+
+    POST with the task_id field set to the id of a "ready" task will return its results or an error
+    if the task failed or the id is invalid (expired, never exited, etc).
+    """
     results = None
+    # Start an investigation or retrieve a result on a POST
     if request.method == "POST":
         form = InvestigateForm(request.POST)
         if form.is_valid():
-            name_fragment = form.cleaned_data["name_fragment"]
-            results = investigate_fragment(name_fragment)
+            task_id = form.cleaned_data["task_id"]
+            if task_id:
+                # Ignore the rest of the form and retrieve the result
+                task_result = AsyncResult(task_id)
+                if task_result.successful():
+                    retval = task_result.get()
+                    results = retval["results"]
+                    form.data = form.data.copy()
+                    form.data["name_fragment"] = retval[
+                        "name_fragment"
+                    ]  # ensure consistency
+                    del form.data["task_id"]  # do not request the task result again
+                else:
+                    form.add_error(
+                        None,
+                        "The investigation task failed. Please try again and ask for help if this recurs.",
+                    )
+                # Falls through to the render at the end!
+            else:
+                name_fragment = form.cleaned_data["name_fragment"]
+                task_result = investigate_fragment_task.delay(name_fragment)
+                return JsonResponse({"id": task_result.id})
     else:
-        form = InvestigateForm()
+        task_id = request.GET.get("id", None)
+        if task_id is not None:
+            # Check status if we got the "id" parameter
+            task_result = AsyncResult(task_id)
+            return JsonResponse(
+                {"status": "ready" if task_result.ready() else "notready"}
+            )
+        else:
+            # Serve up an empty form
+            form = InvestigateForm()
+
+    # If we get here, it is just a plain GET - serve the UI
     return render(
-        request, "doc/investigate.html", context=dict(form=form, results=results)
+        request,
+        "doc/investigate.html",
+        context={
+            "form": form,
+            "results": results,
+        },
     )
diff --git a/ietf/settings.py b/ietf/settings.py
index b452864be6..a1dc9ffe21 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -452,6 +452,7 @@ def skip_unreadable_post(record):
     'django_vite',
     'django_bootstrap5',
     'django_celery_beat',
+    'django_celery_results',
     'corsheaders',
     'django_markup',
     'oidc_provider',
@@ -1226,7 +1227,9 @@ def skip_unreadable_post(record):
 # https://docs.celeryq.dev/en/stable/userguide/tasks.html#rpc-result-backend-rabbitmq-qpid
 # Results can be retrieved only once and only by the caller of the task. Results will be
 # lost if the message broker restarts.
-CELERY_RESULT_BACKEND = 'rpc://'  # sends a msg via the msg broker
+CELERY_RESULT_BACKEND = 'django-cache'  # use a Django cache for results
+CELERY_CACHE_BACKEND = 'celery-results'  # which Django cache to use
+CELERY_RESULT_EXPIRES = datetime.timedelta(minutes=5)  # how long are results valid? (Default is 1 day)
 CELERY_TASK_IGNORE_RESULT = True  # ignore results unless specifically enabled for a task
 
 # Meetecho API setup: Uncomment this and provide real credentials to enable
@@ -1309,6 +1312,11 @@ def skip_unreadable_post(record):
                     "MAX_ENTRIES": 5000,
                 },
             },
+            "celery-results": {
+                "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                "KEY_PREFIX": "ietf:celery",
+            },
         }
     else:
         CACHES = {
@@ -1347,6 +1355,11 @@ def skip_unreadable_post(record):
                     "MAX_ENTRIES": 5000,
                 },
             },
+            "celery-results": {
+                "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache",
+                "LOCATION": "app:11211",
+                "KEY_PREFIX": "ietf:celery",
+            },
         }
 
 PUBLISH_IPR_STATES = ['posted', 'removed', 'removed_objfalse']
diff --git a/ietf/static/js/investigate.js b/ietf/static/js/investigate.js
new file mode 100644
index 0000000000..b22e099b1e
--- /dev/null
+++ b/ietf/static/js/investigate.js
@@ -0,0 +1,53 @@
+// Copyright The IETF Trust 2025, All Rights Reserved
+document.addEventListener('DOMContentLoaded', () => {
+    const investigateForm = document.forms['investigate']
+    investigateForm.addEventListener('submit', (event) => {
+        // Intercept submission unless we've filled in the task_id field
+        if (!investigateForm.elements['id_task_id'].value) {
+            event.preventDefault()
+            runInvestigation()
+        }
+    })
+
+    const runInvestigation = async () => {
+        // Submit the request
+        const response = await fetch('', {
+            method: investigateForm.method, body: new FormData(investigateForm)
+        })
+        if (!response.ok) {
+            loadResultsFromTask('bogus-task-id') // bad task id will generate an error from Django
+        }
+        const taskId = (await response.json()).id
+        // Poll for completion of the investigation up to 18*10 = 180 seconds 
+        waitForResults(taskId, 18)
+    }
+
+    const waitForResults = async (taskId, retries) => {
+        // indicate that investigation is in progress
+        document.getElementById('spinner').classList.remove('d-none')
+        document.getElementById('investigate-button').disabled = true
+        investigateForm.elements['id_name_fragment'].disabled = true
+
+        const response = await fetch('?' + new URLSearchParams({ id: taskId }))
+        if (!response.ok) {
+            loadResultsFromTask('bogus-task-id') // bad task id will generate an error from Django
+        }
+        const result = await response.json()
+        if (result.status !== 'ready' && retries > 0) {
+            // 10 seconds per retry
+            setTimeout(waitForResults, 10000, taskId, retries - 1)
+        } else {
+            /* Either the response is ready or we timed out waiting. In either case, submit
+               the task_id via POST and let Django display an error if it's not ready. Before
+               submitting, re-enable the form fields so the POST is valid. Other in-progress
+               indicators will be reset when the POST response is loaded. */
+            loadResultsFromTask(taskId)
+        }
+    }
+
+    const loadResultsFromTask = (taskId) => {
+        investigateForm.elements['id_name_fragment'].disabled = false
+        investigateForm.elements['id_task_id'].value = taskId
+        investigateForm.submit()
+    }
+})
diff --git a/ietf/templates/doc/investigate.html b/ietf/templates/doc/investigate.html
index bdcf644406..436a8ce91a 100644
--- a/ietf/templates/doc/investigate.html
+++ b/ietf/templates/doc/investigate.html
@@ -6,112 +6,122 @@
     <link rel="stylesheet" href="{% static "ietf/css/list.css" %}">
 {% endblock %}
 {% block content %}
-  {% origin %}
-  <h1>Investigate</h1>
-  <form id="investigate" method="post">
-    {% csrf_token %}
-    {% bootstrap_form form %}
-    <button class="btn btn-primary" type="submit">Investigate</button>
-  </form>
-  {% if results %}
-    <div id="results">
-      {% if results.can_verify %}
-        <h2>These can be authenticated</h2>
-        <table id="authenticated" class="table table-sm table-striped tablesorter">
-          <thead>
-            <tr>
-              <th scope="col" data-sort="name">Name</th>
-              <th scope="col" data-sort="modified">Last Modified On</th>
-              <th scope="col" data-sort="link">Link</th>
-              <th scope="col" data-sort="source">Source</th>
-            </tr>
-          </thead>
-          <tbody>
-            {% for path in results.can_verify %}
-              {% with url=path|url_for_path %}
-                <tr>
-                    <td>{{path.name}}</td>
-                    <td>
-                        {% if path|mtime_is_epoch %}
-                          Timestamp has been lost (is Unix Epoch)
-                        {% else %}
-                          {{path|mtime|date:"DATETIME_FORMAT"}}
-                        {% endif %}
-                    </td>
-                    <td><a href="{{url}}">{{url}}</a></td>
-                    <td>{{path}}</td>
-                </tr>
-              {% endwith %}
-            {% endfor %}
-          </tbody>
-        </table>
-      {% else %}
-      <h2>Nothing with this name fragment can be authenticated</h2>
-      {% endif %}
-      <hr>
-      {% if results.unverifiable_collections %}
-        <h2>These are in the archive, but cannot be authenticated</h2>
-        <table id="unverifiable" class="table table-sm table-striped tablesorter">
-          <thead>
-            <tr>
-              <th scope="col" data-sort="name">Name</th>
-              <th scope="col" data-sort="modified">Last Modified On</th>
-              <th scope="col" data-sort="link">Link</th>
-              <th scope="col" data-sort="source">Source</th>
-            </tr>
-          </thead>
-          <tbody>
-            {% for path in results.unverifiable_collections %}
-              {% with url=path|url_for_path %}
-              <tr>
-                <td>{{path.name}}</td>
-                <td>
-                    {% if path|mtime_is_epoch %}
-                      Timestamp has been lost (is Unix Epoch)
-                    {% else %}
-                      {{path|mtime|date:"DATETIME_FORMAT"}}
-                    {% endif %}
-                </td>
-                <td><a href="{{url}}">{{url}}</a></td>
-                <td>{{path}}</td>
-              </tr>
-              {% endwith %}
-            {% endfor %}
-          </tbody>
-        </table>
-      {% endif %}
-      {% if results.unexpected %}
-        <h2>These are unexpected and we do not know what their origin is. These cannot be authenticated</h2>
-        <table id="unexpected" class="table table-sm table-striped tablesorter">
-          <thead>
-            <tr>
-              <th scope="col" data-sort="name">Name</th>
-              <th scope="col" data-sort="modified">Last Modified On</th>
-              <th scope="col" data-sort="link">Link</th>
-            </tr>
-          </thead>
-          <tbody>
-            {% for path in results.unexpected %}
-              {% with url=path|url_for_path %}
-              <tr>
-                <td>{{path.name}}</td>
-                <td>
-                    {% if path|mtime_is_epoch %}
-                      Timestamp has been lost (is Unix Epoch)
-                    {% else %}
-                      {{path|mtime|date:"DATETIME_FORMAT"}}
-                    {% endif %}
-                </td>
-                <td><a href="{{url}}">{{url}}</a></td>
-              </tr>
-              {% endwith %}
-            {% endfor %}
-          </tbody>
-        </table>
-      {% endif %}
+    {% origin %}
+    <h1>Investigate</h1>
+    <div class="mb-3">
+        <form id="investigate" method="post">
+            {% csrf_token %}
+            {% bootstrap_form form %}
+            <button class="btn btn-primary" type="submit" id="investigate-button">
+            <span id="spinner"
+                  class="spinner-border spinner-border-sm d-none"
+                  role="status"
+                  aria-hidden="true">
+            </span>
+                Investigate
+            </button>
+        </form>
     </div>
-  {% endif %}
+    {% if results %}
+        <div id="results">
+            {% if results.can_verify %}
+                <h2>These can be authenticated</h2>
+                <table id="authenticated" class="table table-sm table-striped tablesorter">
+                    <thead>
+                    <tr>
+                        <th scope="col" data-sort="name">Name</th>
+                        <th scope="col" data-sort="modified">Last Modified On</th>
+                        <th scope="col" data-sort="link">Link</th>
+                        <th scope="col" data-sort="source">Source</th>
+                    </tr>
+                    </thead>
+                    <tbody>
+                    {% for path in results.can_verify %}
+                        {% with url=path|url_for_path %}
+                            <tr>
+                                <td>{{ path.name }}</td>
+                                <td>
+                                    {% if path|mtime_is_epoch %}
+                                        Timestamp has been lost (is Unix Epoch)
+                                    {% else %}
+                                        {{ path|mtime|date:"DATETIME_FORMAT" }}
+                                    {% endif %}
+                                </td>
+                                <td><a href="{{ url }}">{{ url }}</a></td>
+                                <td>{{ path }}</td>
+                            </tr>
+                        {% endwith %}
+                    {% endfor %}
+                    </tbody>
+                </table>
+            {% else %}
+                <h2>Nothing with this name fragment can be authenticated</h2>
+            {% endif %}
+            <hr>
+            {% if results.unverifiable_collections %}
+                <h2>These are in the archive, but cannot be authenticated</h2>
+                <table id="unverifiable" class="table table-sm table-striped tablesorter">
+                    <thead>
+                    <tr>
+                        <th scope="col" data-sort="name">Name</th>
+                        <th scope="col" data-sort="modified">Last Modified On</th>
+                        <th scope="col" data-sort="link">Link</th>
+                        <th scope="col" data-sort="source">Source</th>
+                    </tr>
+                    </thead>
+                    <tbody>
+                    {% for path in results.unverifiable_collections %}
+                        {% with url=path|url_for_path %}
+                            <tr>
+                                <td>{{ path.name }}</td>
+                                <td>
+                                    {% if path|mtime_is_epoch %}
+                                        Timestamp has been lost (is Unix Epoch)
+                                    {% else %}
+                                        {{ path|mtime|date:"DATETIME_FORMAT" }}
+                                    {% endif %}
+                                </td>
+                                <td><a href="{{ url }}">{{ url }}</a></td>
+                                <td>{{ path }}</td>
+                            </tr>
+                        {% endwith %}
+                    {% endfor %}
+                    </tbody>
+                </table>
+            {% endif %}
+            {% if results.unexpected %}
+                <h2>These are unexpected and we do not know what their origin is. These cannot be authenticated</h2>
+                <table id="unexpected" class="table table-sm table-striped tablesorter">
+                    <thead>
+                    <tr>
+                        <th scope="col" data-sort="name">Name</th>
+                        <th scope="col" data-sort="modified">Last Modified On</th>
+                        <th scope="col" data-sort="link">Link</th>
+                    </tr>
+                    </thead>
+                    <tbody>
+                    {% for path in results.unexpected %}
+                        {% with url=path|url_for_path %}
+                            <tr>
+                                <td>{{ path.name }}</td>
+                                <td>
+                                    {% if path|mtime_is_epoch %}
+                                        Timestamp has been lost (is Unix Epoch)
+                                    {% else %}
+                                        {{ path|mtime|date:"DATETIME_FORMAT" }}
+                                    {% endif %}
+                                </td>
+                                <td><a href="{{ url }}">{{ url }}</a></td>
+                            </tr>
+                        {% endwith %}
+                    {% endfor %}
+                    </tbody>
+                </table>
+            {% endif %}
+        </div>
+    {% endif %}
 {% endblock %}
 {% block js %}
     <script src="{% static "ietf/js/list.js" %}"></script>
+    <script src="{% static "ietf/js/investigate.js" %}"></script>
 {% endblock %}
\ No newline at end of file
diff --git a/package.json b/package.json
index afcabd13ed..b3d36b349c 100644
--- a/package.json
+++ b/package.json
@@ -132,6 +132,7 @@
         "ietf/static/js/highcharts.js",
         "ietf/static/js/highstock.js",
         "ietf/static/js/ietf.js",
+        "ietf/static/js/investigate.js",
         "ietf/static/js/ipr-edit.js",
         "ietf/static/js/ipr-search.js",
         "ietf/static/js/js-cookie.js",
diff --git a/requirements.txt b/requirements.txt
index ec5fc60b5f..ae8b06faee 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,6 +13,7 @@ Django>4.2,<5
 django-analytical>=3.1.0
 django-bootstrap5>=21.3
 django-celery-beat>=2.3.0
+django-celery-results>=2.5.1
 django-csp>=3.7
 django-cors-headers>=3.11.0
 django-debug-toolbar>=3.2.4

From 03d6d598ed7fb9172a68f7dfd7925951a0e78cf3 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Fri, 17 Jan 2025 15:28:12 +0000
Subject: [PATCH 230/656] ci: update base image target version to 20250117T1516

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 2f35125564..59fbd0cc1b 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250116T2033
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250117T1516
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index fd2d539a9e..fb0f6385f8 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250116T2033
+20250117T1516

From ad56d42d9c6858457f7a36856e7c071d7d41f5e0 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 17 Jan 2025 12:35:26 -0600
Subject: [PATCH 231/656] ci: dev/test blobstore image (#8431)

* ci: dev/test blobstore image

* chore: removed QEMU step
---
 .github/workflows/build-devblobstore.yml | 47 ++++++++++++++++++++++++
 docker/devblobstore.Dockerfile           |  9 +++++
 2 files changed, 56 insertions(+)
 create mode 100644 .github/workflows/build-devblobstore.yml
 create mode 100644 docker/devblobstore.Dockerfile

diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml
new file mode 100644
index 0000000000..2d8214b448
--- /dev/null
+++ b/.github/workflows/build-devblobstore.yml
@@ -0,0 +1,47 @@
+name: Build Dev/Test Blobstore Docker Image
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - '.github/workflows/build-devblobstore.yml'
+
+  workflow_dispatch: 
+
+env:
+  MINIO_VERSION: latest
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Docker Build & Push
+      uses: docker/build-push-action@v6
+      env:
+        DOCKER_BUILD_NO_SUMMARY: true
+      with:
+        context: .
+        file: docker/devblobstore.Dockerfile
+        platforms: linux/amd64,linux/arm64
+        push: true
+        build-args: MINIO_VERSION=${{ env.MINIO_VERSION }}
+        tags: |
+          ghcr.io/ietf-tools/datatracker-devblobstore:${{ env.MINIO_VERSION }}
+          ghcr.io/ietf-tools/datatracker-devblobstore:latest
diff --git a/docker/devblobstore.Dockerfile b/docker/devblobstore.Dockerfile
new file mode 100644
index 0000000000..40bfbd0e96
--- /dev/null
+++ b/docker/devblobstore.Dockerfile
@@ -0,0 +1,9 @@
+ARG MINIO_VERSION=latest
+FROM quay.io/minio/minio:${MINIO_VERSION}
+LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
+
+ENV MINIO_ROOT_USER=minio_root
+ENV MINIO_ROOT_PASSWORD=minio_pass
+ENV MINIO_DEFAULT_BUCKETS=defaultbucket
+
+CMD ["server", "--console-address", ":9001", "/data"]

From 65a96948dc8545e3b15f791c9c855fc7c64d03bf Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 20 Jan 2025 11:59:18 -0400
Subject: [PATCH 232/656] fix: map "Editorial" -> StreamName (#8442)

---
 ietf/sync/rfceditor.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 6b3482f10d..889317cdcf 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -373,6 +373,7 @@ def update_docs_from_rfc_index(
         "INDEPENDENT": StreamName.objects.get(slug="ise"),
         "IRTF": StreamName.objects.get(slug="irtf"),
         "IAB": StreamName.objects.get(slug="iab"),
+        "Editorial": StreamName.objects.get(slug="editorial"),
         "Legacy": StreamName.objects.get(slug="legacy"),
     }
 

From 3ec94225e36d8caa6ee3c516499874455d51b088 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 21 Jan 2025 10:59:40 -0400
Subject: [PATCH 233/656] chore: remove deprecated_api_upload_bluesheet method
 (#8443)

* chore: remove deprecated_api_upload_bluesheet()

* test: remove test of removed method
---
 ietf/api/tests.py     | 92 -------------------------------------------
 ietf/meeting/views.py | 64 ------------------------------
 2 files changed, 156 deletions(-)

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 6e7fb374fe..d195f49b94 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -554,98 +554,6 @@ def test_api_upload_polls_and_chatlog(self):
             newdoccontent = get_unicode_document_content(newdoc.name, Path(session.meeting.get_materials_path()) / type_id / newdoc.uploaded_filename)
             self.assertEqual(json.loads(content), json.loads(newdoccontent))
 
-    def test_deprecated_api_upload_bluesheet(self):
-        url = urlreverse('ietf.meeting.views.api_upload_bluesheet')
-        recmanrole = RoleFactory(group__type_id='ietf', name_id='recman')
-        recman = recmanrole.person
-        meeting = MeetingFactory(type_id='ietf')
-        session = SessionFactory(group__type_id='wg', meeting=meeting)
-        group = session.group
-        apikey = PersonalApiKeyFactory(endpoint=url, person=recman)
-
-        people = [
-            {"name": "Andrea Andreotti", "affiliation": "Azienda"},
-            {"name": "Bosse Bernadotte", "affiliation": "Bolag"},
-            {"name": "Charles Charlemagne", "affiliation": "Compagnie"},
-        ]
-        for i in range(3):
-            faker = random_faker()
-            people.append(dict(name=faker.name(), affiliation=faker.company()))
-        bluesheet = json.dumps(people)
-
-        # error cases
-        r = self.client.post(url, {})
-        self.assertContains(r, "Missing apikey parameter", status_code=400)
-
-        badrole = RoleFactory(group__type_id='ietf', name_id='ad')
-        badapikey = PersonalApiKeyFactory(endpoint=url, person=badrole.person)
-        badrole.person.user.last_login = timezone.now()
-        badrole.person.user.save()
-        r = self.client.post(url, {'apikey': badapikey.hash()})
-        self.assertContains(r, "Restricted to roles: Recording Manager, Secretariat", status_code=403)
-
-        r = self.client.post(url, {'apikey': apikey.hash()})
-        self.assertContains(r, "Too long since last regular login", status_code=400)
-        recman.user.last_login = timezone.now()
-        recman.user.save()
-
-        r = self.client.get(url, {'apikey': apikey.hash()})
-        self.assertContains(r, "Method not allowed", status_code=405)
-
-        r = self.client.post(url, {'apikey': apikey.hash(), 'group': group.acronym})
-        self.assertContains(r, "Missing meeting parameter", status_code=400)
-
-        r = self.client.post(url, {'apikey': apikey.hash(), 'meeting': meeting.number, })
-        self.assertContains(r, "Missing group parameter", status_code=400)
-
-        r = self.client.post(url, {'apikey': apikey.hash(), 'meeting': meeting.number, 'group': group.acronym})
-        self.assertContains(r, "Missing item parameter", status_code=400)
-
-        r = self.client.post(url,
-                             {'apikey': apikey.hash(), 'meeting': meeting.number, 'group': group.acronym, 'item': '1'})
-        self.assertContains(r, "Missing bluesheet parameter", status_code=400)
-
-        r = self.client.post(url, {'apikey': apikey.hash(), 'meeting': '1', 'group': group.acronym,
-                                   'item': '1', 'bluesheet': bluesheet, })
-        self.assertContains(r, "No sessions found for meeting", status_code=400)
-
-        r = self.client.post(url, {'apikey': apikey.hash(), 'meeting': meeting.number, 'group': 'bogous',
-                                   'item': '1', 'bluesheet': bluesheet, })
-        self.assertContains(r, "No sessions found in meeting '%s' for group 'bogous'" % meeting.number, status_code=400)
-
-        r = self.client.post(url, {'apikey': apikey.hash(), 'meeting': meeting.number, 'group': group.acronym,
-                                   'item': '1', 'bluesheet': "foobar", })
-        self.assertContains(r, "Invalid json value: 'foobar'", status_code=400)
-
-        r = self.client.post(url, {'apikey': apikey.hash(), 'meeting': meeting.number, 'group': group.acronym,
-                                   'item': '5', 'bluesheet': bluesheet, })
-        self.assertContains(r, "No item '5' found in list of sessions for group", status_code=400)
-
-        r = self.client.post(url, {'apikey': apikey.hash(), 'meeting': meeting.number, 'group': group.acronym,
-                                   'item': 'foo', 'bluesheet': bluesheet, })
-        self.assertContains(r, "Expected a numeric value for 'item', found 'foo'", status_code=400)
-
-        r = self.client.post(url, {'apikey': apikey.hash(), 'meeting': meeting.number, 'group': group.acronym,
-                                   'item': '1', 'bluesheet': bluesheet, })
-        self.assertContains(r, "Done", status_code=200)
-
-        # Submit again, with slightly different content, as an updated version
-        people[1]['affiliation'] = 'Bolaget AB'
-        bluesheet = json.dumps(people)
-        r = self.client.post(url, {'apikey': apikey.hash(), 'meeting': meeting.number, 'group': group.acronym,
-                                   'item': '1', 'bluesheet': bluesheet, })
-        self.assertContains(r, "Done", status_code=200)
-
-        bluesheet = session.presentations.filter(document__type__slug='bluesheets').first().document
-        # We've submitted an update; check that the rev is right
-        self.assertEqual(bluesheet.rev, '01')
-        # Check the content
-        with open(bluesheet.get_file_name()) as file:
-            text = file.read()
-            for p in people:
-                self.assertIn(p['name'], html.unescape(text))
-                self.assertIn(p['affiliation'], html.unescape(text))
-
     def test_api_upload_bluesheet(self):
         url = urlreverse("ietf.meeting.views.api_upload_bluesheet")
         recmanrole = RoleFactory(group__type_id="ietf", name_id="recman")
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index f386f8932f..feb14f730b 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -4666,11 +4666,6 @@ def err(code, text):
             content="Method not allowed", content_type="text/plain", permitted_methods=('POST',)
         )
 
-    # Temporary: fall back to deprecated interface if we have old-style parameters.
-    # Do away with this once meetecho is using the new pk-based interface.
-    if any(k in request.POST for k in ['meeting', 'group', 'item']):
-        return deprecated_api_upload_bluesheet(request)
-
     session_id = request.POST.get('session_id', None)
     if session_id is None:
         return err(400, 'Missing session_id parameter')
@@ -4707,65 +4702,6 @@ def err(code, text):
     return HttpResponse("Done", status=200, content_type='text/plain')
 
 
-def deprecated_api_upload_bluesheet(request):
-    def err(code, text):
-        return HttpResponse(text, status=code, content_type='text/plain')
-    if request.method == 'POST':
-        # parameters:
-        #   apikey: the poster's personal API key
-        #   meeting: number as string, i.e., '101', or 'interim-2018-quic-02'
-        #   group: acronym or special, i.e., 'quic' or 'plenary'
-        #   item: '1', '2', '3' (the group's first, second, third etc.
-        #                           session during the week)
-        #   bluesheet: json blob with
-        #       [{'name': 'Name', 'affiliation': 'Organization', }, ...]
-        for item in ['meeting', 'group', 'item', 'bluesheet',]:
-            value = request.POST.get(item)
-            if not value:
-                return err(400, "Missing %s parameter" % item)
-        number = request.POST.get('meeting')
-        sessions = Session.objects.filter(meeting__number=number)
-        if not sessions.exists():
-            return err(400, "No sessions found for meeting '%s'" % (number, ))
-        acronym = request.POST.get('group')
-        sessions = sessions.filter(group__acronym=acronym)
-        if not sessions.exists():
-            return err(400, "No sessions found in meeting '%s' for group '%s'" % (number, acronym))
-        session_times = [ (s.official_timeslotassignment().timeslot.time, s.id, s) for s in sessions if s.official_timeslotassignment() ]
-        session_times.sort()
-        item = request.POST.get('item')
-        if not item.isdigit():
-            return err(400, "Expected a numeric value for 'item', found '%s'" % (item, ))
-        n = int(item)-1              # change 1-based to 0-based
-        try:
-            time, __, session = session_times[n]
-        except IndexError:
-            return err(400, "No item '%s' found in list of sessions for group" % (item, ))
-        bjson = request.POST.get('bluesheet')
-        try:
-            data = json.loads(bjson)
-        except json.decoder.JSONDecodeError:
-            return err(400, "Invalid json value: '%s'" % (bjson, ))
-
-        text = render_to_string('meeting/bluesheet.txt', {
-                'data': data,
-                'session': session,
-            })
-
-        fd, name = tempfile.mkstemp(suffix=".txt", text=True)
-        os.close(fd)
-        with open(name, "w") as file:
-            file.write(text)
-        with open(name, "br") as file:
-            save_err = save_bluesheet(request, session, file)
-        if save_err:
-            return err(400, save_err)
-    else:
-        return err(405, "Method not allowed")
-
-    return HttpResponse("Done", status=200, content_type='text/plain')
-
-
 def important_dates(request, num=None, output_format=None):
     assert num is None or num.isdigit()
     preview_roles = ['Area Director', 'Secretariat', 'IETF Chair', 'IAD', ]

From ba1ba6c6258f3e9150898b1090cffc6d5ee0bed9 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 21 Jan 2025 12:17:41 -0400
Subject: [PATCH 234/656] test: fix test_api_upload_bluesheet (#8446)

* test: fix test_api_upload_bluesheet

* chore: delint
---
 ietf/api/tests.py | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index d195f49b94..d9af457e95 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -560,7 +560,6 @@ def test_api_upload_bluesheet(self):
         recman = recmanrole.person
         meeting = MeetingFactory(type_id="ietf")
         session = SessionFactory(group__type_id="wg", meeting=meeting)
-        group = session.group
         apikey = PersonalApiKeyFactory(endpoint=url, person=recman)
 
         people = [
@@ -600,18 +599,6 @@ def test_api_upload_bluesheet(self):
         r = self.client.post(url, {"apikey": apikey.hash(), "session_id": session.pk})
         self.assertContains(r, "Missing bluesheet parameter", status_code=400)
 
-        r = self.client.post(
-            url,
-            {
-                "apikey": apikey.hash(),
-                "meeting": meeting.number,
-                "group": group.acronym,
-                "item": "1",
-                "bluesheet": "foobar",
-            },
-        )
-        self.assertContains(r, "Invalid json value: 'foobar'", status_code=400)
-
         bad_session_pk = int(Session.objects.order_by("-pk").first().pk) + 1
         r = self.client.post(
             url,
@@ -650,9 +637,7 @@ def test_api_upload_bluesheet(self):
             url,
             {
                 "apikey": apikey.hash(),
-                "meeting": meeting.number,
-                "group": group.acronym,
-                "item": "1",
+                "session_id": session.pk,
                 "bluesheet": bluesheet,
             },
         )

From 233e23f40ab7cb863d5992c411345e4ca9c10416 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 21 Jan 2025 10:19:15 -0600
Subject: [PATCH 235/656] fix: correct ftp dir path (#8433)

---
 ietf/settings.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/settings.py b/ietf/settings.py
index a1dc9ffe21..7fd4308bb0 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -745,8 +745,8 @@ def skip_unreadable_post(record):
 RFC_PATH = '/a/www/ietf-ftp/rfc/'
 CHARTER_PATH = '/a/ietfdata/doc/charter/'
 CHARTER_COPY_PATH = '/a/www/ietf-ftp/ietf'  # copy 1wg-charters files here if set
-CHARTER_COPY_OTHER_PATH = '/a/www/ftp/ietf'
-CHARTER_COPY_THIRD_PATH = '/a/www/ftp/charter'
+CHARTER_COPY_OTHER_PATH = '/a/ftp/ietf'
+CHARTER_COPY_THIRD_PATH = '/a/ftp/charter'
 GROUP_SUMMARY_PATH = '/a/www/ietf-ftp/ietf'
 BOFREQ_PATH = '/a/ietfdata/doc/bofreq/'
 CONFLICT_REVIEW_PATH = '/a/ietfdata/doc/conflict-review'

From 1c4820db46f89fbc31535906ca3da2d56360738f Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 21 Jan 2025 14:42:57 -0400
Subject: [PATCH 236/656] ci: celery-results cache for k8s (#8450)

---
 k8s/settings_local.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index a63a02e536..f266ffcd62 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -272,6 +272,11 @@ def _multiline_to_list(s):
             "MAX_ENTRIES": 5000,
         },
     },
+    "celery-results": {
+        "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache",
+        "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+        "KEY_PREFIX": "ietf:celery",
+    },
 }
 
 _csrf_trusted_origins_str = os.environ.get("DATATRACKER_CSRF_TRUSTED_ORIGINS")

From 9a72cc0d6ab6e951eb14f077da36b46d7dca9b6a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 22 Jan 2025 13:52:09 -0400
Subject: [PATCH 237/656] fix: increase investigate time; more feedback (#8457)

---
 ietf/static/js/investigate.js       |  6 +++---
 ietf/templates/doc/investigate.html | 12 +++++++-----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/ietf/static/js/investigate.js b/ietf/static/js/investigate.js
index b22e099b1e..ee1c545147 100644
--- a/ietf/static/js/investigate.js
+++ b/ietf/static/js/investigate.js
@@ -18,13 +18,13 @@ document.addEventListener('DOMContentLoaded', () => {
             loadResultsFromTask('bogus-task-id') // bad task id will generate an error from Django
         }
         const taskId = (await response.json()).id
-        // Poll for completion of the investigation up to 18*10 = 180 seconds 
-        waitForResults(taskId, 18)
+        // Poll for completion of the investigation up to 60*10 = 600 seconds 
+        waitForResults(taskId, 60)
     }
 
     const waitForResults = async (taskId, retries) => {
         // indicate that investigation is in progress
-        document.getElementById('spinner').classList.remove('d-none')
+        document.querySelectorAll('.investigation-indicator').forEach(elt => elt.classList.remove('d-none'))
         document.getElementById('investigate-button').disabled = true
         investigateForm.elements['id_name_fragment'].disabled = true
 
diff --git a/ietf/templates/doc/investigate.html b/ietf/templates/doc/investigate.html
index 436a8ce91a..8b613117d5 100644
--- a/ietf/templates/doc/investigate.html
+++ b/ietf/templates/doc/investigate.html
@@ -13,13 +13,15 @@ <h1>Investigate</h1>
             {% csrf_token %}
             {% bootstrap_form form %}
             <button class="btn btn-primary" type="submit" id="investigate-button">
-            <span id="spinner"
-                  class="spinner-border spinner-border-sm d-none"
-                  role="status"
-                  aria-hidden="true">
-            </span>
+                <span class="spinner-border spinner-border-sm investigation-indicator d-none"
+                      role="status"
+                      aria-hidden="true">
+                </span>
                 Investigate
             </button>
+            <div class="alert alert-info mt-3 d-none investigation-indicator">
+                Please be patient, processing may take several minutes.
+            </div>
         </form>
     </div>
     {% if results %}

From 266d5bed3c2e0d2004b12e9d02dbc209f8295cad Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 22 Jan 2025 14:19:21 -0400
Subject: [PATCH 238/656] feat: cache file investigation results (#8459)

* feat: cache result of investigate_fragment

* test: test caching
---
 ietf/doc/tests.py | 37 +++++++++++++++++++++++++++
 ietf/doc/utils.py | 65 +++++++++++++++++++++++++++++++----------------
 2 files changed, 80 insertions(+), 22 deletions(-)

diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index f5af7bb48b..d74688f3f6 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -5,6 +5,8 @@
 import os
 import datetime
 import io
+from hashlib import sha384
+
 from django.http import HttpRequest
 import lxml
 import bibtexparser
@@ -3280,6 +3282,41 @@ def test_investigate_fragment(self):
             "draft-this-should-not-be-possible-00.txt",
         )
 
+    @mock.patch("ietf.doc.utils.caches")
+    def test_investigate_fragment_cache(self, mock_caches):
+        """investigate_fragment should cache its result"""
+        mock_default_cache = mock_caches["default"]
+        mock_default_cache.get.return_value = None  # disable cache
+        result = investigate_fragment("this-is-active")
+        self.assertEqual(len(result["can_verify"]), 1)
+        self.assertEqual(len(result["unverifiable_collections"]), 0)
+        self.assertEqual(len(result["unexpected"]), 0)
+        self.assertEqual(
+            list(result["can_verify"])[0].name, "draft-this-is-active-00.txt"
+        )
+        self.assertTrue(mock_default_cache.get.called)
+        self.assertTrue(mock_default_cache.set.called)
+        expected_key = f"investigate_fragment:{sha384(b'this-is-active').hexdigest()}"
+        self.assertEqual(mock_default_cache.set.call_args.kwargs["key"], expected_key)
+        cached_value = mock_default_cache.set.call_args.kwargs["value"]  # hang on to this
+        mock_default_cache.reset_mock()
+
+        # Check that a cached value is used
+        mock_default_cache.get.return_value = cached_value
+        with mock.patch("ietf.doc.utils.Path") as mock_path:
+            result = investigate_fragment("this-is-active")
+        # Check that we got the same results
+        self.assertEqual(len(result["can_verify"]), 1)
+        self.assertEqual(len(result["unverifiable_collections"]), 0)
+        self.assertEqual(len(result["unexpected"]), 0)
+        self.assertEqual(
+            list(result["can_verify"])[0].name, "draft-this-is-active-00.txt"
+        )
+        # And that we used the cache
+        self.assertFalse(mock_path.called)  # a proxy for "did the method do any real work"
+        self.assertTrue(mock_default_cache.get.called)
+        self.assertEqual(mock_default_cache.get.call_args, mock.call(expected_key))
+
     def test_investigate_get(self):
         """GET with no querystring should retrieve the investigate UI"""
         url = urlreverse("ietf.doc.views_doc.investigate")
diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index ff19dfbde6..53307526a1 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -11,12 +11,14 @@
 
 from collections import defaultdict, namedtuple, Counter
 from dataclasses import dataclass
+from hashlib import sha384
 from pathlib import Path
 from typing import Iterator, Optional, Union
 from zoneinfo import ZoneInfo
 
 from django.conf import settings
 from django.contrib import messages
+from django.core.cache import caches
 from django.db.models import OuterRef
 from django.forms import ValidationError
 from django.http import Http404
@@ -1459,29 +1461,48 @@ def get_doc_email_aliases(name: Optional[str] = None):
     return sorted(aliases, key=lambda a: (a["doc_name"]))
 
 
-def investigate_fragment(name_fragment):
-    can_verify = set()
-    for root in [settings.INTERNET_DRAFT_PATH, settings.INTERNET_DRAFT_ARCHIVE_DIR]:
-        can_verify.update(list(Path(root).glob(f"*{name_fragment}*")))
-    archive_verifiable_names = set([p.name for p in can_verify])
-    # Can also verify drafts in proceedings directories
-    can_verify.update(list(Path(settings.AGENDA_PATH).glob(f"**/*{name_fragment}*")))
-
-    # N.B. This reflects the assumption that the internet draft archive dir is in the
-    # a directory with other collections (at /a/ietfdata/draft/collections as this is written)
-    unverifiable_collections = set([
-        p for p in
-        Path(settings.INTERNET_DRAFT_ARCHIVE_DIR).parent.glob(f"**/*{name_fragment}*")
-        if p.name not in archive_verifiable_names
-    ])
+def investigate_fragment(name_fragment: str):
+    cache = caches["default"]
+    # Ensure name_fragment does not interact badly with the cache key handling
+    name_digest = sha384(name_fragment.encode("utf8")).hexdigest()
+    cache_key = f"investigate_fragment:{name_digest}"
+    cached_result = cache.get(cache_key)
+    if cached_result is not None:
+        can_verify = cached_result["can_verify"]
+        unverifiable_collections = cached_result["unverifiable_collections"]
+        unexpected = cached_result["unexpected"]
+    else:
+        can_verify = set()
+        for root in [settings.INTERNET_DRAFT_PATH, settings.INTERNET_DRAFT_ARCHIVE_DIR]:
+            can_verify.update(list(Path(root).glob(f"*{name_fragment}*")))
+        archive_verifiable_names = set([p.name for p in can_verify])
+        # Can also verify drafts in proceedings directories
+        can_verify.update(list(Path(settings.AGENDA_PATH).glob(f"**/*{name_fragment}*")))
     
-    unverifiable_collections.difference_update(can_verify)
-
-    expected_names = set([p.name for p in can_verify.union(unverifiable_collections)])
-    maybe_unexpected = list(
-        Path(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR).glob(f"*{name_fragment}*")
-    )
-    unexpected = [p for p in maybe_unexpected if p.name not in expected_names]
+        # N.B. This reflects the assumption that the internet draft archive dir is in the
+        # a directory with other collections (at /a/ietfdata/draft/collections as this is written)
+        unverifiable_collections = set([
+            p for p in
+            Path(settings.INTERNET_DRAFT_ARCHIVE_DIR).parent.glob(f"**/*{name_fragment}*")
+            if p.name not in archive_verifiable_names
+        ])
+        
+        unverifiable_collections.difference_update(can_verify)
+    
+        expected_names = set([p.name for p in can_verify.union(unverifiable_collections)])
+        maybe_unexpected = list(
+            Path(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR).glob(f"*{name_fragment}*")
+        )
+        unexpected = [p for p in maybe_unexpected if p.name not in expected_names]
+        cache.set(
+            key=cache_key,
+            timeout=3600,  # 1 hour
+            value={
+                "can_verify": can_verify,
+                "unverifiable_collections": unverifiable_collections,
+                "unexpected": unexpected,
+            }
+        )
 
     return dict(
         can_verify=can_verify,

From d202ebbb91e206985a776c317d54c72e3aacf402 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 22 Jan 2025 15:07:06 -0400
Subject: [PATCH 239/656] refactor: single def of dict structure (#8460)

---
 ietf/doc/utils.py | 29 +++++++++--------------------
 1 file changed, 9 insertions(+), 20 deletions(-)

diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index 53307526a1..10fe9ff2d7 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -1466,12 +1466,8 @@ def investigate_fragment(name_fragment: str):
     # Ensure name_fragment does not interact badly with the cache key handling
     name_digest = sha384(name_fragment.encode("utf8")).hexdigest()
     cache_key = f"investigate_fragment:{name_digest}"
-    cached_result = cache.get(cache_key)
-    if cached_result is not None:
-        can_verify = cached_result["can_verify"]
-        unverifiable_collections = cached_result["unverifiable_collections"]
-        unexpected = cached_result["unexpected"]
-    else:
+    result = cache.get(cache_key)
+    if result is None:
         can_verify = set()
         for root in [settings.INTERNET_DRAFT_PATH, settings.INTERNET_DRAFT_ARCHIVE_DIR]:
             can_verify.update(list(Path(root).glob(f"*{name_fragment}*")))
@@ -1494,21 +1490,14 @@ def investigate_fragment(name_fragment: str):
             Path(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR).glob(f"*{name_fragment}*")
         )
         unexpected = [p for p in maybe_unexpected if p.name not in expected_names]
-        cache.set(
-            key=cache_key,
-            timeout=3600,  # 1 hour
-            value={
-                "can_verify": can_verify,
-                "unverifiable_collections": unverifiable_collections,
-                "unexpected": unexpected,
-            }
+        result = dict(
+            can_verify=can_verify,
+            unverifiable_collections=unverifiable_collections,
+            unexpected=unexpected,
         )
-
-    return dict(
-        can_verify=can_verify,
-        unverifiable_collections=unverifiable_collections,
-        unexpected=unexpected,
-    )
+        # 1 hour caching
+        cache.set(key=cache_key, timeout=3600, value=result)
+    return result
 
 
 def update_or_create_draft_bibxml_file(doc, rev):

From ac88f73439fa1d240fbee9d7856eb3a49c0f9201 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 24 Jan 2025 10:30:02 -0600
Subject: [PATCH 240/656] chore: remove unreachable code (#8464)

---
 ietf/meeting/utils.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index b68a311f5d..b17c84f313 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -738,15 +738,10 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N
 
     This function takes a _binary mode_ file object, a filename and a meeting object and subdir as string.
     It saves the file to the appropriate directory, get_materials_path() + subdir.
-    If the file is a zip file, it creates a new directory in 'slides', which is the basename of the
-    zip file and unzips the file in the new directory.
     """
     filename = Path(filename)
-    is_zipfile = filename.suffix == '.zip'
 
     path = Path(meeting.get_materials_path()) / subdir
-    if is_zipfile:
-        path = path / filename.stem
     path.mkdir(parents=True, exist_ok=True)
 
     # agendas and minutes can only have one file instance so delete file if it already exists
@@ -799,10 +794,6 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N
             for chunk in chunks:
                 destination.write(chunk)
 
-    # unzip zipfile
-    if is_zipfile:
-        subprocess.call(['unzip', filename], cwd=path)
-
     return None
 
 def new_doc_for_session(type_id, session):

From 56f723a3bcd27adaf66e04c93ea53a58e8227197 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Sat, 25 Jan 2025 15:05:01 -0600
Subject: [PATCH 241/656] chore: remove unreachable code (#8465)

---
 ietf/meeting/utils.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index b17c84f313..cfe7adfae7 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -744,14 +744,6 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N
     path = Path(meeting.get_materials_path()) / subdir
     path.mkdir(parents=True, exist_ok=True)
 
-    # agendas and minutes can only have one file instance so delete file if it already exists
-    if subdir in ('agenda', 'minutes'):
-        for f in path.glob(f'{filename.stem}.*'):
-            try:
-                f.unlink()
-            except FileNotFoundError:
-                pass  # if the file is already gone, so be it
-
     with (path / filename).open('wb+') as destination:
         # prep file for reading
         if hasattr(file, "chunks"):

From e91bda7e5e21fc7cbd7de500e8d66141bb6900c6 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 28 Jan 2025 13:28:19 -0400
Subject: [PATCH 242/656] feat: consolidate HTML sanitizing (#8471)

* refactor: isolate bleach code

* refactor: move html fns to html.py

* refactor: lose the bleach.py module; refactor

* refactor: sanitize_document -> clean_html

Drops <meta charset="utf-8"> addition after cleaning.

* fix: disambiguate import

* feat: restore <meta charset="utf-8"> tag

* chore: comments

* chore(deps): drop lxml_html_clean package

* refactor: on second thought, no meta charset

* refactor: sanitize_fragment -> clean_html

* test: remove check for charset

* chore: fix lint
---
 ietf/doc/templatetags/ietf_filters.py  |  15 ++--
 ietf/meeting/tests_views.py            |   3 +-
 ietf/meeting/utils.py                  |   6 +-
 ietf/utils/html.py                     | 115 ++++++++++++++-----------
 ietf/utils/markdown.py                 |  12 +--
 ietf/utils/templatetags/textfilters.py |   4 +-
 ietf/utils/text.py                     |  66 ++++++--------
 requirements.txt                       |   3 +-
 8 files changed, 110 insertions(+), 114 deletions(-)

diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py
index 5a8afd9956..e1a80a26b2 100644
--- a/ietf/doc/templatetags/ietf_filters.py
+++ b/ietf/doc/templatetags/ietf_filters.py
@@ -26,10 +26,11 @@
 from ietf.doc.models import BallotDocEvent, Document
 from ietf.doc.models import ConsensusDocEvent
 from ietf.ietfauth.utils import can_request_rfc_publication as utils_can_request_rfc_publication
-from ietf.utils.html import sanitize_fragment
 from ietf.utils import log
 from ietf.doc.utils import prettify_std_name
-from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped, bleach_linker, bleach_cleaner, validate_url
+from ietf.utils.html import clean_html
+from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped, linkify
+from ietf.utils.validators import validate_url
 
 register = template.Library()
 
@@ -98,7 +99,7 @@ def sanitize(value):
     attributes to those deemed acceptable.  See ietf/utils/html.py
     for the details.
     """
-    return mark_safe(sanitize_fragment(value))
+    return mark_safe(clean_html(value))
 
 
 # For use with ballot view
@@ -446,16 +447,16 @@ def ad_area(user):
 @register.filter
 def format_history_text(text, trunc_words=25):
     """Run history text through some cleaning and add ellipsis if it's too long."""
-    full = mark_safe(bleach_cleaner.clean(text))
-    full = bleach_linker.linkify(urlize_ietf_docs(full))
+    full = mark_safe(clean_html(text))
+    full = linkify(urlize_ietf_docs(full))
 
     return format_snippet(full, trunc_words)
 
 @register.filter
 def format_snippet(text, trunc_words=25): 
     # urlize if there aren't already links present
-    text = bleach_linker.linkify(text)
-    full = keep_spacing(collapsebr(linebreaksbr(mark_safe(sanitize_fragment(text)))))
+    text = linkify(text)
+    full = keep_spacing(collapsebr(linebreaksbr(mark_safe(clean_html(text)))))
     snippet = truncatewords_html(full, trunc_words)
     if snippet != full:
         return mark_safe('<div class="snippet">%s<button type="button" aria-label="Expand" class="btn btn-sm btn-primary show-all"><i class="bi bi-caret-down"></i></button></div><div class="d-none full">%s</div>' % (snippet, full))
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 0647da52ab..b52f8de9b2 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -6423,8 +6423,7 @@ def test_upload_minutes_agenda(self):
             text = doc.text()
             self.assertIn('Some text', text)
             self.assertNotIn('<section>', text)
-            self.assertIn('charset="utf-8"', text)
-
+        
             # txt upload
             test_file = BytesIO(b'This is some text for a test, with the word\nvirtual at the beginning of a line.')
             test_file.name = "some.txt"
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index cfe7adfae7..4f800980c3 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -30,7 +30,7 @@
 from ietf.name.models import SessionStatusName, ConstraintName, DocTypeName
 from ietf.person.models import Person
 from ietf.stats.models import MeetingRegistration
-from ietf.utils.html import sanitize_document
+from ietf.utils.html import clean_html
 from ietf.utils.log import log
 from ietf.utils.timezone import date_today
 
@@ -773,8 +773,8 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N
                     return "Failure trying to save '%s'. Hint: Try to upload as UTF-8: %s..." % (filename, str(e)[:120])
             # Whole file sanitization; add back what's missing from a complete
             # document (sanitize will remove these).
-            clean = sanitize_document(text)
-            destination.write(clean.encode('utf8'))
+            clean = clean_html(text)
+            destination.write(clean.encode("utf8"))
             if request and clean != text:
                 messages.warning(request,
                                  (
diff --git a/ietf/utils/html.py b/ietf/utils/html.py
index 9d0cd7c84f..3f3efe2f37 100644
--- a/ietf/utils/html.py
+++ b/ietf/utils/html.py
@@ -5,11 +5,7 @@
 
 
 import bleach
-import copy
 import html2text
-import lxml.etree
-import lxml.html
-import lxml.html.clean
 
 import debug                            # pyflakes:ignore
 
@@ -17,62 +13,66 @@
 from django.utils.functional import keep_lazy
 
 from ietf.utils.mime import get_mime_type
-from ietf.utils.text import bleach_cleaner, tags as acceptable_tags
 
-acceptable_protocols = ['http', 'https', 'mailto', 'xmpp', ]
 
-def unescape(text):
-    """
-    Returns the given text with ampersands, quotes and angle brackets decoded
-    for use in URLs.
+# Allow the protocols/tags/attributes we specifically want, plus anything that bleach declares
+# to be safe. As of 2025-01-27, the explicit lists for protocols and tags are a strict superset
+# of bleach's defaults.
+acceptable_protocols = bleach.sanitizer.ALLOWED_PROTOCOLS.union(
+    {"http", "https", "mailto", "ftp", "xmpp"}
+)
+acceptable_tags = bleach.sanitizer.ALLOWED_TAGS.union(
+    {
+        # fmt: off
+        "a", "abbr", "acronym", "address", "b", "big",
+        "blockquote", "body", "br", "caption", "center", "cite", "code", "col",
+        "colgroup", "dd", "del", "dfn", "dir", "div", "dl", "dt", "em", "font",
+        "h1", "h2", "h3", "h4", "h5", "h6", "head", "hr", "html", "i", "ins", "kbd",
+        "li", "ol", "p", "pre", "q", "s", "samp", "small", "span", "strike", "style",
+        "strong", "sub", "sup", "table", "title", "tbody", "td", "tfoot", "th", "thead",
+        "tr", "tt", "u", "ul", "var"
+        # fmt: on
+    }
+)
+acceptable_attributes = bleach.sanitizer.ALLOWED_ATTRIBUTES | {
+    "*": ["id"],
+    "ol": ["start"],
+}
+
+
+# Instantiate sanitizer classes
+_bleach_cleaner = bleach.sanitizer.Cleaner(
+    tags=acceptable_tags,
+    attributes=acceptable_attributes,
+    protocols=acceptable_protocols,
+    strip=True,
+)
+
+
+_liberal_bleach_cleaner = bleach.sanitizer.Cleaner(
+    tags=acceptable_tags.union({"img", "figure", "figcaption"}),
+    attributes=acceptable_attributes | {"img": ["src", "alt"]},
+    protocols=acceptable_protocols,
+    strip=True,
+)
+
+
+def clean_html(text: str):
+    """Clean the HTML in a string"""
+    return _bleach_cleaner.clean(text)
+
+
+def liberal_clean_html(text: str):
+    """More permissively clean the HTML in a string"""
+    return _liberal_bleach_cleaner.clean(text)
 
-    This function undoes what django.utils.html.escape() does
-    """
-    return text.replace('&amp;', '&').replace('&#39;', "'").replace('&quot;', '"').replace('&gt;', '>').replace('&lt;', '<' )
 
 @keep_lazy(str)
 def remove_tags(html, tags):
     """Returns the given HTML sanitized, and with the given tags removed."""
-    allowed = set(acceptable_tags) - set([ t.lower() for t in tags ])
+    allowed = acceptable_tags - set(t.lower() for t in tags)
     return bleach.clean(html, tags=allowed, strip=True)
 
-# ----------------------------------------------------------------------
-# Html fragment cleaning
-
-def sanitize_fragment(html):
-    return bleach_cleaner.clean(html)
-
-# ----------------------------------------------------------------------
-# Page cleaning
-
-
-class Cleaner(lxml.html.clean.Cleaner):
-    charset = 'utf-8'
-    def __init__(self, charset='utf-8', **kw):
-        self.charset = charset
-        super(Cleaner, self).__init__(**kw)
-        
-    # Copied from lxml 4.2.0 and modified to insert charset meta:
-    def clean_html(self, html):
-        result_type = type(html)
-        if isinstance(html, (str, bytes)):
-            doc = lxml.html.fromstring(html)
-        else:
-            doc = copy.deepcopy(html)
-        self(doc)
-        head = doc.find('head')
-        if head != None:
-            meta = lxml.etree.Element('meta', charset=self.charset)
-            meta.tail = '\n'
-            head.insert(0, meta)
-        return lxml.html._transform_result(result_type, doc)
-
-# We will be saving as utf-8 later, so set that in the meta tag.
-lxml_cleaner = Cleaner(allow_tags=acceptable_tags, remove_unknown_tags=None, style=False, page_structure=False, charset='utf-8')
-
-def sanitize_document(html):
-    return lxml_cleaner.clean_html(html)
-
 
 # ----------------------------------------------------------------------
 # Text field cleaning
@@ -86,4 +86,15 @@ def clean_text_field(text):
     else:
         raise forms.ValidationError("Unexpected text field mime type: %s" % mime_type)
     return text
-    
+
+
+def unescape(text):
+    """
+    Returns the given text with ampersands, quotes and angle brackets decoded
+    for use in URLs.
+
+    This function undoes what django.utils.html.escape() does
+    """
+    return text.replace('&amp;', '&').replace('&#39;', "'").replace('&quot;', '"').replace('&gt;', '>').replace('&lt;', '<' )
+
+
diff --git a/ietf/utils/markdown.py b/ietf/utils/markdown.py
index 446d348959..0b522685b2 100644
--- a/ietf/utils/markdown.py
+++ b/ietf/utils/markdown.py
@@ -12,13 +12,15 @@
 from django.utils.safestring import mark_safe
 
 from ietf.doc.templatetags.ietf_filters import urlize_ietf_docs
-from ietf.utils.text import bleach_cleaner, liberal_bleach_cleaner, bleach_linker
+from .html import clean_html, liberal_clean_html
+from .text import linkify
+
 
 
 class LinkifyExtension(Extension):
     """
     Simple Markdown extension inspired by https://github.com/daGrevis/mdx_linkify,
-    but using our bleach_linker directly. Doing the linkification on the converted
+    but using our own linker directly. Doing the linkification on the converted
     Markdown output introduces artifacts.
     """
 
@@ -31,12 +33,12 @@ def extendMarkdown(self, md):
 
 class LinkifyPostprocessor(Postprocessor):
     def run(self, text):
-        return urlize_ietf_docs(bleach_linker.linkify(text))
+        return urlize_ietf_docs(linkify(text))
 
 
 def markdown(text):
     return mark_safe(
-        bleach_cleaner.clean(
+        clean_html(
             python_markdown.markdown(
                 text,
                 extensions=[
@@ -52,7 +54,7 @@ def markdown(text):
 
 def liberal_markdown(text):
     return mark_safe(
-        liberal_bleach_cleaner.clean(
+        liberal_clean_html(
             python_markdown.markdown(
                 text,
                 extensions=[
diff --git a/ietf/utils/templatetags/textfilters.py b/ietf/utils/templatetags/textfilters.py
index 70b94cf673..3b240740e0 100644
--- a/ietf/utils/templatetags/textfilters.py
+++ b/ietf/utils/templatetags/textfilters.py
@@ -11,7 +11,7 @@
 
 import debug                            # pyflakes:ignore
 
-from ietf.utils.text import xslugify as _xslugify, texescape, bleach_linker
+from ietf.utils.text import linkify as _linkify, xslugify as _xslugify, texescape
 
 register = template.Library()
 
@@ -74,7 +74,7 @@ def texescape_filter(value):
 @register.filter
 @stringfilter
 def linkify(value):
-    text = mark_safe(bleach_linker.linkify(value))
+    text = mark_safe(_linkify(value))
     return text
 
 @register.filter
diff --git a/ietf/utils/text.py b/ietf/utils/text.py
index 2fba113d01..4e5d5b6cd5 100644
--- a/ietf/utils/text.py
+++ b/ietf/utils/text.py
@@ -1,17 +1,15 @@
 # Copyright The IETF Trust 2016-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-
-import bleach  # type: ignore
-import copy
+import bleach
 import email
 import re
 import textwrap
 import tlds
 import unicodedata
 
-from django.core.validators import URLValidator
 from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
 from django.utils.functional import keep_lazy
 from django.utils.safestring import mark_safe
 
@@ -19,66 +17,52 @@
 
 from .texescape import init as texescape_init, tex_escape_map
 
-tlds_sorted = sorted(tlds.tld_set, key=len, reverse=True)
-protocols = set(bleach.sanitizer.ALLOWED_PROTOCOLS)
-protocols.add("ftp")  # we still have some ftp links
-protocols.add("xmpp")  # we still have some xmpp links
-
-tags = set(bleach.sanitizer.ALLOWED_TAGS).union(
-    {
-        # fmt: off
-        'a', 'abbr', 'acronym', 'address', 'b', 'big',
-        'blockquote', 'body', 'br', 'caption', 'center', 'cite', 'code', 'col',
-        'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt', 'em', 'font',
-        'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'head', 'hr', 'html', 'i', 'ins', 'kbd',
-        'li', 'ol', 'p', 'pre', 'q', 's', 'samp', 'small', 'span', 'strike', 'style',
-        'strong', 'sub', 'sup', 'table', 'title', 'tbody', 'td', 'tfoot', 'th', 'thead',
-        'tr', 'tt', 'u', 'ul', 'var'
-        # fmt: on
-    }
-)
+# Sort in reverse so substrings are considered later - e.g., so ".co" comes after ".com".
+tlds_sorted = sorted(tlds.tld_set, reverse=True)
 
-attributes = copy.copy(bleach.sanitizer.ALLOWED_ATTRIBUTES)
-attributes["*"] = ["id"]
-attributes["ol"] = ["start"]
+# Protocols we're interested in auto-linking. See also ietf.utils.html.acceptable_protocols,
+# which is protocols we allow people to include explicitly  in sanitized html. 
+linkable_protocols = ["http", "https", "mailto", "ftp", "xmpp"]
 
-bleach_cleaner = bleach.sanitizer.Cleaner(
-    tags=tags, attributes=attributes, protocols=protocols, strip=True
-)
-
-liberal_tags = copy.copy(tags)
-liberal_attributes = copy.copy(attributes)
-liberal_tags.update(["img","figure","figcaption"])
-liberal_attributes["img"] = ["src","alt"]
 
-liberal_bleach_cleaner = bleach.sanitizer.Cleaner(
-    tags=liberal_tags, attributes=liberal_attributes, protocols=protocols, strip=True
-)
-
-validate_url = URLValidator()
+_validate_url = URLValidator() 
 
 
 def check_url_validity(attrs, new=False):
+    """Callback for bleach linkify
+    
+    :param attrs: dict of attributes of the <a> tag
+    :param new: boolean - True if the link is new; False if <a> was found in text
+    :return: new dict of attributes for the link, or None to block link creation 
+
+    Attributes are namespaced, so normally look like `(None, "SomeAttribute")`.
+    This includes as the keys in the `attrs` argument, so `attrs[(None, "href")]`
+    would be the value of the href attribute.  
+    """
     if (None, "href") not in attrs:
         # rfc2html creates a tags without href
         return attrs
     url = attrs[(None, "href")]
     try:
         if url.startswith("http"):
-            validate_url(url)
+            _validate_url(url)
     except ValidationError:
         return None
     return attrs
 
 
-bleach_linker = bleach.Linker(
+_bleach_linker = bleach.Linker(
     callbacks=[check_url_validity],
-    url_re=bleach.linkifier.build_url_re(tlds=tlds_sorted, protocols=protocols),
+    url_re=bleach.linkifier.build_url_re(tlds=tlds_sorted, protocols=linkable_protocols),
     email_re=bleach.linkifier.build_email_re(tlds=tlds_sorted),  # type: ignore
     parse_email=True,
 )
 
 
+def linkify(text):
+    return _bleach_linker.linkify(text)
+
+
 @keep_lazy(str)
 def xslugify(value):
     """
diff --git a/requirements.txt b/requirements.txt
index ae8b06faee..073a6bfa0a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -43,8 +43,7 @@ jsonfield>=3.1.0    # for SubmissionCheck.  This is https://github.com/bradjaspe
 jsonschema[format]>=4.2.1
 jwcrypto>=1.2    # for signed notifications - this is aspirational, and is not really used.
 logging_tree>=1.9    # Used only by the showloggers management command
-lxml>=5.3.0    # lxml[html_clean] fails on some architectures
-lxml_html_clean>=0.4.1
+lxml>=5.3.0
 markdown>=3.3.6
 types-markdown>=3.3.6
 mock>=4.0.3    # Used only by tests, of course

From 826f4d87478c41d7909bfb1eb1558455e14e37b9 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Tue, 28 Jan 2025 17:39:50 +0000
Subject: [PATCH 243/656] ci: update base image target version to 20250128T1728

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 59fbd0cc1b..852ba43c80 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250117T1516
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250128T1728
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index fb0f6385f8..fbc9426744 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250117T1516
+20250128T1728

From d3bf0a37dccfbd7ca3aa8c5190ceaf96c33813fb Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 29 Jan 2025 15:11:12 -0400
Subject: [PATCH 244/656] refactor: don't template inside <script> (#8480)

---
 ietf/static/js/complete-review.js              |  4 +++-
 ietf/templates/doc/review/complete_review.html | 16 +++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ietf/static/js/complete-review.js b/ietf/static/js/complete-review.js
index a359dac237..3a58ba9700 100644
--- a/ietf/static/js/complete-review.js
+++ b/ietf/static/js/complete-review.js
@@ -24,6 +24,8 @@ $(document)
             .before(mailArchiveSearchTemplate);
 
         var mailArchiveSearch = form.find(".mail-archive-search");
+        const isReviewer = mailArchiveSearch.data('isReviewer');
+        const searchMailArchiveUrl = mailArchiveSearch.data('searchMailArchiveUrl');
 
         var retrievingData = null;
 
@@ -190,4 +192,4 @@ $(document)
             form.find("[name=review_submission][value=link]")
                 .trigger("click");
         }
-    });
\ No newline at end of file
+    });
diff --git a/ietf/templates/doc/review/complete_review.html b/ietf/templates/doc/review/complete_review.html
index f30c641e46..091f30532e 100644
--- a/ietf/templates/doc/review/complete_review.html
+++ b/ietf/templates/doc/review/complete_review.html
@@ -94,7 +94,13 @@ <h1>
         {% endif %}
         <div class="template d-none">
             {% if mail_archive_query_urls %}
-                <div class="mail-archive-search">
+                <div class="mail-archive-search"
+                    {% if assignment %}
+                     data-search-mail-archive-url="{% url "ietf.doc.views_review.search_mail_archive" name=doc.name assignment_id=assignment.pk %}"
+                    {% else %}
+                     data-search-mail-archive-url="{% url "ietf.doc.views_review.search_mail_archive" name=doc.name acronym=team.acronym %}"
+                    {% endif %}
+                     data-is-reviewer="{{ is_reviewer|yesno:"true,false" }}">
                     <div class="offset-md-2 col-md-10">
                         <label for="mail-archive-subjects" class="form-label">Search {{team.list_email}} mail archive subjects for:</label>
                         <div class="input-group mb-3">
@@ -144,13 +150,5 @@ <h1>
 {% endblock %}
 {% block js %}
     <script src="{% static 'ietf/js/datepicker.js' %}"></script>
-    <script>
-        {% if assignment %}
-            var searchMailArchiveUrl = "{% url "ietf.doc.views_review.search_mail_archive" name=doc.name assignment_id=assignment.pk %}";
-        {% else %}
-            var searchMailArchiveUrl = "{% url "ietf.doc.views_review.search_mail_archive" name=doc.name acronym=team.acronym %}";
-        {% endif %}
-        var isReviewer = {{ is_reviewer|yesno:'true,false' }};
-    </script>
     <script src="{% static 'ietf/js/complete-review.js' %}"></script>
 {% endblock %}
\ No newline at end of file

From da858725fd43b42255fa867fc2aa2abbec429299 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 30 Jan 2025 12:21:15 -0400
Subject: [PATCH 245/656] fix: guard against KeyError (#8485)

---
 ietf/iesg/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/iesg/utils.py b/ietf/iesg/utils.py
index a56fa72cee..56571dc753 100644
--- a/ietf/iesg/utils.py
+++ b/ietf/iesg/utils.py
@@ -33,7 +33,7 @@ def telechat_page_count(date=None, docs=None, ad=None):
             ballot = draft.active_ballot()
             if ballot:
                 positions = ballot.active_balloter_positions()
-                ad_position = positions[ad]
+                ad_position = positions.get(ad, None)
                 if ad_position is None or ad_position.pos_id == "norecord":
                     ad_pages_left_to_ballot_on += draft.pages or 0
 

From a1c3a940e82d205a7c6e9e22b7974df39039dfca Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 30 Jan 2025 12:22:42 -0500
Subject: [PATCH 246/656] ci: pin aws cli version because AWS broke things

---
 .github/workflows/build.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4771adde97..a14ea73e9b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -178,6 +178,11 @@ jobs:
       with:
         python-version: "3.x"
 
+    - name: Setup AWS CLI
+      uses: unfor19/install-aws-cli-action@v1
+      with:
+        version: 2.22.35
+
     - name: Download a Coverage Results
       if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
       uses: actions/download-artifact@v4.1.8

From db7d3074da79734707db9e5475295bc3d65c810d Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@holloway.co.nz>
Date: Sat, 1 Feb 2025 05:28:39 +1300
Subject: [PATCH 247/656] feat: Add session recordings (#8218)

* feat: add session recordings

* feat: add session recordings

* feat: deleting recordings

* feat: deleting recordings and initial form values

* feat: use meeting date rather than today for initial title field. Fix delete recording

* feat: confirm delete recordings modal. fix server utils delete recording

* fix: removing debug console.log

* feat: change button name from 'Ok' to 'Delete' for confirm deletion to be clearer

* feat: UTC time in string and delete modal text

* fix: django html validation tests

* fix: django html validation tests

* fix: django html validation tests

* refactor: Work with SessionPresentations

* fix: better ordering

* chore: drop rev, hide table when empty

* test: test delete_recordings method

* fix: debug delete_recordings

* test: test add_session_recordings view

* fix: better permissions handling

* fix: only delete recordings for selected session

* refactor: inline script -> js module

* chore: remove accidental import

*shakes fist at pycharm*

* fix: consistent timestamp format

plus slight rephrase

* style: Black

* chore: remove comment

* test: update test to match

* fix: reversible url pattern for materials

Tests were perturbed in a way that led to a test
getting an interim instead of an IETF meeting.
This exposed a bug reversing the URL for the
materials_document() view. This splits it into
two patterns that are equivalent to the original.

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 ietf/meeting/tests_views.py                   | 156 +++++++++++++++++-
 ietf/meeting/urls.py                          |   4 +-
 ietf/meeting/utils.py                         |  22 ++-
 ietf/meeting/views.py                         |  88 +++++++++-
 ietf/settings.py                              |   3 +
 ietf/static/js/add_session_recordings.js      |  30 ++++
 .../meeting/add_session_recordings.html       |  78 +++++++++
 .../meeting/session_details_panel.html        |   8 +
 package.json                                  |   1 +
 9 files changed, 385 insertions(+), 5 deletions(-)
 create mode 100644 ietf/static/js/add_session_recordings.js
 create mode 100644 ietf/templates/meeting/add_session_recordings.html

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index b52f8de9b2..3e948f8386 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -48,7 +48,7 @@
 from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting, make_interim_test_data
 from ietf.meeting.utils import condition_slide_order
 from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting
-from ietf.meeting.utils import create_recording, get_next_sequence, bluesheet_data
+from ietf.meeting.utils import create_recording, delete_recording, get_next_sequence, bluesheet_data
 from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule
 from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose, generate_agenda_data
 from ietf.name.models import SessionStatusName, ImportantDateName, RoleName, ProceedingsMaterialTypeName
@@ -441,6 +441,48 @@ def test_session_recordings_via_factories(self):
         self.assertIn(new_recording_title, links[0].text_content())
         #debug.show("q(f'#notes_and_recordings_{session_pk}')")
 
+    def test_delete_recordings(self):
+        # No user specified, active recording state
+        sp = SessionPresentationFactory(
+            document__type_id="recording",
+            document__external_url="https://example.com/some-recording",
+            document__states=[("recording", "active")],
+        )
+        doc = sp.document
+        doc.docevent_set.all().delete()  # clear this out
+        delete_recording(sp)
+        self.assertFalse(SessionPresentation.objects.filter(pk=sp.pk).exists())
+        self.assertEqual(doc.get_state("recording").slug, "deleted", "recording state updated")
+        self.assertEqual(doc.docevent_set.count(), 1, "one event added")
+        event = doc.docevent_set.first()
+        self.assertEqual(event.type, "changed_state", "event is a changed_state event")
+        self.assertEqual(event.by.name, "(System)", "system user is responsible")
+
+        # Specified user, no recording state
+        sp = SessionPresentationFactory(
+            document__type_id="recording",
+            document__external_url="https://example.com/some-recording",
+            document__states=[],
+        )
+        doc = sp.document
+        doc.docevent_set.all().delete()  # clear this out
+        user = PersonFactory()  # naming matches the methods - user is a Person, not a User
+        delete_recording(sp, user=user)
+        self.assertFalse(SessionPresentation.objects.filter(pk=sp.pk).exists())
+        self.assertEqual(doc.get_state("recording").slug, "deleted", "recording state updated")
+        self.assertEqual(doc.docevent_set.count(), 1, "one event added")
+        event = doc.docevent_set.first()
+        self.assertEqual(event.type, "changed_state", "event is a changed_state event")
+        self.assertEqual(event.by, user, "user is responsible")
+
+        # Document is not a recording
+        sp = SessionPresentationFactory(
+            document__type_id="draft",
+            document__external_url="https://example.com/some-recording",
+        )
+        with self.assertRaises(ValueError):
+            delete_recording(sp)
+
     def test_agenda_ical_next_meeting_type(self):
         # start with no upcoming IETF meetings, just an interim
         MeetingFactory(
@@ -7363,6 +7405,118 @@ def test_request_minutes(self):
         self.assertEqual(r.status_code,302)
         self.assertEqual(len(outbox),1)
 
+    @override_settings(YOUTUBE_DOMAINS=["youtube.com"])
+    def test_add_session_recordings(self):
+        session = SessionFactory(meeting__type_id="ietf")
+        url = urlreverse(
+            "ietf.meeting.views.add_session_recordings", 
+            kwargs={"session_id": session.pk, "num": session.meeting.number},
+        )
+        # does not fully validate authorization for non-secretariat users :-(
+        login_testing_unauthorized(self, "secretary", url)
+        r = self.client.get(url)
+        pq = PyQuery(r.content)
+        title_input = pq("input#id_title")
+        self.assertIsNotNone(title_input)
+        self.assertEqual(
+            title_input.attr.value,
+            "Video recording of {acro} for {timestamp}".format(
+                acro=session.group.acronym,
+                timestamp=session.official_timeslotassignment().timeslot.utc_start_time().strftime(
+                    "%Y-%m-%d %H:%M"
+                ),
+            ),
+        )
+        
+        with patch("ietf.meeting.views.create_recording") as mock_create:
+            r = self.client.post(
+                url,
+                data={
+                    "title": "This is my video title",
+                    "url": "",
+                }
+            )
+        self.assertFalse(mock_create.called)
+        
+        with patch("ietf.meeting.views.create_recording") as mock_create:
+            r = self.client.post(
+                url,
+                data={
+                    "title": "This is my video title",
+                    "url": "https://yubtub.com/this-is-not-a-youtube-video",
+                }
+            )
+        self.assertFalse(mock_create.called)
+
+        with patch("ietf.meeting.views.create_recording") as mock_create:
+            r = self.client.post(
+                url,
+                data={
+                    "title": "This is my video title",
+                    "url": "https://youtube.com/finally-a-video",
+                }
+            )
+        self.assertTrue(mock_create.called)
+        self.assertEqual(
+            mock_create.call_args, 
+            call(
+                session, 
+                "https://youtube.com/finally-a-video",
+                title="This is my video title",
+                user=Person.objects.get(user__username="secretary"),
+            ),
+        )
+
+        # CAN delete session presentation for this session
+        sp = SessionPresentationFactory(
+            session=session,
+            document__type_id="recording",
+            document__external_url="https://example.com/some-video",
+        )
+        with patch("ietf.meeting.views.delete_recording") as mock_delete:
+            r = self.client.post(
+                url,
+                data={
+                    "delete": str(sp.pk),
+                }
+            )
+        self.assertEqual(r.status_code, 200)
+        self.assertTrue(mock_delete.called)
+        self.assertEqual(mock_delete.call_args, call(sp))
+        
+        # ValueError message from delete_recording does not reach the user
+        sp = SessionPresentationFactory(
+            session=session,
+            document__type_id="recording",
+            document__external_url="https://example.com/some-video",
+        )
+        with patch("ietf.meeting.views.delete_recording", side_effect=ValueError("oh joy!")) as mock_delete:
+            r = self.client.post(
+                url,
+                data={
+                    "delete": str(sp.pk),
+                }
+            )
+        self.assertTrue(mock_delete.called)
+        self.assertNotContains(r, "oh joy!", status_code=200)
+        
+        # CANNOT delete session presentation for a different session
+        sp_for_other_session = SessionPresentationFactory(
+            document__type_id="recording",
+            document__external_url="https://example.com/some-other-video",
+        )
+        with patch("ietf.meeting.views.delete_recording") as mock_delete:
+            r = self.client.post(
+                url,
+                data={
+                    "delete": str(sp_for_other_session.pk),
+                }
+            )
+        self.assertEqual(r.status_code, 404)
+        self.assertFalse(mock_delete.called)
+
+
+
 class HasMeetingsTests(TestCase):
     settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH']
 
diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py
index 42a5de623c..5da24ddb6f 100644
--- a/ietf/meeting/urls.py
+++ b/ietf/meeting/urls.py
@@ -16,6 +16,7 @@ def get_redirect_url(self, *args, **kwargs):
 safe_for_all_meeting_types = [
     url(r'^session/(?P<acronym>[-a-z0-9]+)/?$',  views.session_details),
     url(r'^session/(?P<session_id>\d+)/drafts$',  views.add_session_drafts),
+    url(r'^session/(?P<session_id>\d+)/recordings$', views.add_session_recordings),
     url(r'^session/(?P<session_id>\d+)/attendance$', views.session_attendance),
     url(r'^session/(?P<session_id>\d+)/bluesheets$', views.upload_session_bluesheets),
     url(r'^session/(?P<session_id>\d+)/minutes$', views.upload_session_minutes),
@@ -63,7 +64,8 @@ def get_redirect_url(self, *args, **kwargs):
 type_interim_patterns = [
     url(r'^agenda/(?P<acronym>[A-Za-z0-9-]+)-drafts.pdf$', views.session_draft_pdf),
     url(r'^agenda/(?P<acronym>[A-Za-z0-9-]+)-drafts.tgz$', views.session_draft_tarfile),
-    url(r'^materials/%(document)s((?P<ext>\.[a-z0-9]+)|/)?$' % settings.URL_REGEXPS, views.materials_document),
+    url(r'^materials/%(document)s(?P<ext>\.[a-z0-9]+)$' % settings.URL_REGEXPS, views.materials_document),
+    url(r'^materials/%(document)s/?$' % settings.URL_REGEXPS, views.materials_document),
     url(r'^agenda.json$', views.agenda_json)
 ]
 
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index 4f800980c3..e231cbed87 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -23,7 +23,7 @@
 from ietf.dbtemplate.models import DBTemplate
 from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot,
     Constraint, SchedTimeSessAssignment, SessionPresentation, Attended)
-from ietf.doc.models import Document, State, NewRevisionDocEvent
+from ietf.doc.models import Document, State, NewRevisionDocEvent, StateDocEvent
 from ietf.doc.models import DocEvent
 from ietf.group.models import Group
 from ietf.group.utils import can_manage_materials
@@ -853,6 +853,26 @@ def create_recording(session, url, title=None, user=None):
 
     return doc
 
+def delete_recording(session_presentation, user=None):
+    """Delete a session recording"""
+    document = session_presentation.document
+    if document.type_id != "recording":
+        raise ValueError(f"Document {document.pk} is not a recording (type_id={document.type_id})")
+    recording_state = document.get_state("recording")
+    deleted_state = State.objects.get(type_id="recording", slug="deleted")
+    if recording_state != deleted_state:
+        # Update the recording state and create a history event 
+        document.set_state(deleted_state)
+        StateDocEvent.objects.create(
+            type="changed_state",
+            by=user or Person.objects.get(name="(System)"),
+            doc=document,
+            rev=document.rev,
+            state_type=deleted_state.type,
+            state=deleted_state,
+        )
+    session_presentation.delete()
+
 def get_next_sequence(group, meeting, type):
     '''
     Returns the next sequence number to use for a document of type = type.
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index feb14f730b..2f2464028b 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -20,7 +20,7 @@
 from functools import partialmethod
 import jsonschema
 from pathlib import Path
-from urllib.parse import parse_qs, unquote, urlencode, urlsplit, urlunsplit
+from urllib.parse import parse_qs, unquote, urlencode, urlsplit, urlunsplit, urlparse
 from tempfile import mkstemp
 from wsgiref.handlers import format_date_time
 
@@ -86,7 +86,7 @@
 from ietf.meeting.utils import swap_meeting_schedule_timeslot_assignments, bulk_create_timeslots
 from ietf.meeting.utils import preprocess_meeting_important_dates
 from ietf.meeting.utils import new_doc_for_session, write_doc_for_session
-from ietf.meeting.utils import get_activity_stats, post_process, create_recording
+from ietf.meeting.utils import get_activity_stats, post_process, create_recording, delete_recording
 from ietf.meeting.utils import participants_for_meeting, generate_bluesheet, bluesheet_data, save_bluesheet
 from ietf.message.utils import infer_message
 from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName
@@ -103,6 +103,7 @@
 from ietf.utils.response import permission_denied
 from ietf.utils.text import xslugify
 from ietf.utils.timezone import datetime_today, date_today
+from ietf.settings import YOUTUBE_DOMAINS
 
 from .forms import (InterimMeetingModelForm, InterimAnnounceForm, InterimSessionModelForm,
     InterimCancelForm, InterimSessionInlineFormSet, RequestMinutesForm,
@@ -2568,6 +2569,89 @@ def add_session_drafts(request, session_id, num):
                     'form': form,
                   })
 
+class SessionRecordingsForm(forms.Form):
+    title = forms.CharField(max_length=255)
+    url = forms.URLField(label="URL of the recording (YouTube only)")
+
+    def clean_url(self):
+        url = self.cleaned_data['url']
+        parsed_url = urlparse(url)
+        if parsed_url.hostname not in YOUTUBE_DOMAINS:
+            raise forms.ValidationError("Must be a YouTube URL")
+        return url
+
+
+def add_session_recordings(request, session_id, num):
+    # num is redundant, but we're dragging it along an artifact of where we are in the current URL structure
+    session = get_object_or_404(Session, pk=session_id)
+    if not session.can_manage_materials(request.user):
+        permission_denied(
+            request, "You don't have permission to manage recordings for this session."
+        )
+    if session.is_material_submission_cutoff() and not has_role(
+        request.user, "Secretariat"
+    ):
+        raise Http404
+
+    session_number = None
+    official_timeslotassignment = session.official_timeslotassignment()
+    assertion("official_timeslotassignment is not None")
+    initial = {
+        "title": "Video recording of {acronym} for {timestamp}".format(
+            acronym=session.group.acronym,
+            timestamp=official_timeslotassignment.timeslot.utc_start_time().strftime(
+                "%Y-%m-%d %H:%M"
+            ),
+        )
+    }
+
+    # find session number if WG has more than one session at the meeting
+    sessions = get_sessions(session.meeting.number, session.group.acronym)
+    if len(sessions) > 1:
+        session_number = 1 + sessions.index(session)
+
+    presentations = session.presentations.filter(
+        document__in=session.get_material("recording", only_one=False),
+    ).order_by("document__title", "document__external_url")
+
+    if request.method == "POST":
+        pk_to_delete = request.POST.get("delete", None)
+        if pk_to_delete is not None:
+            session_presentation = get_object_or_404(presentations, pk=pk_to_delete)
+            try:
+                delete_recording(session_presentation)
+            except ValueError as err:
+                log(f"Error deleting recording from session {session.pk}: {err}")
+                messages.error(
+                    request,
+                    "Unable to delete this recording. Please contact the secretariat for assistance.",
+                )
+            form = SessionRecordingsForm(initial=initial)
+        else:
+            form = SessionRecordingsForm(request.POST)
+            if form.is_valid():
+                title = form.cleaned_data["title"]
+                url = form.cleaned_data["url"]
+                create_recording(session, url, title=title, user=request.user.person)
+                return redirect(
+                    "ietf.meeting.views.session_details",
+                    num=session.meeting.number,
+                    acronym=session.group.acronym,
+                )
+    else:
+        form = SessionRecordingsForm(initial=initial)
+
+    return render(
+        request,
+        "meeting/add_session_recordings.html",
+        {
+            "session": session,
+            "session_number": session_number,
+            "already_linked": presentations,
+            "form": form,
+        },
+    )
+
 
 def session_attendance(request, session_id, num):
     """Session attendance view
diff --git a/ietf/settings.py b/ietf/settings.py
index 7fd4308bb0..125127ba15 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -1397,3 +1397,6 @@ def skip_unreadable_post(record):
     CSRF_TRUSTED_ORIGINS += ['http://localhost:8000', 'http://127.0.0.1:8000', 'http://[::1]:8000']
     SESSION_COOKIE_SECURE = False
     SESSION_COOKIE_SAMESITE = 'Lax'
+
+
+YOUTUBE_DOMAINS = ['www.youtube.com', 'youtube.com', 'youtu.be', 'm.youtube.com', 'youtube-nocookie.com', 'www.youtube-nocookie.com']
diff --git a/ietf/static/js/add_session_recordings.js b/ietf/static/js/add_session_recordings.js
new file mode 100644
index 0000000000..c1c5932a48
--- /dev/null
+++ b/ietf/static/js/add_session_recordings.js
@@ -0,0 +1,30 @@
+// Copyright The IETF Trust 2024-2025, All Rights Reserved
+document.addEventListener('DOMContentLoaded', () => {
+    const form = document.getElementById('delete_recordings_form')
+    const dialog = document.getElementById('delete_confirm_dialog')
+    const dialog_link = document.getElementById('delete_confirm_link')
+    const dialog_submit = document.getElementById('delete_confirm_submit')
+    const dialog_cancel = document.getElementById('delete_confirm_cancel')
+
+    dialog.style.maxWidth = '30vw'
+
+    form.addEventListener('submit', (e) => {
+        e.preventDefault()
+        dialog_submit.value = e.submitter.value
+        const recording_link = e.submitter.closest('tr').querySelector('a')
+        dialog_link.setAttribute('href', recording_link.getAttribute('href'))
+        dialog_link.textContent = recording_link.textContent
+        dialog.showModal()
+    })
+
+    dialog_cancel.addEventListener('click', (e) => {
+        e.preventDefault()
+        dialog.close()
+    })
+
+    document.addEventListener('keydown', (e) => {
+        if (dialog.open && e.key === 'Escape') {
+            dialog.close()
+        }
+    })
+})
diff --git a/ietf/templates/meeting/add_session_recordings.html b/ietf/templates/meeting/add_session_recordings.html
new file mode 100644
index 0000000000..4f21e8f4d0
--- /dev/null
+++ b/ietf/templates/meeting/add_session_recordings.html
@@ -0,0 +1,78 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{% load origin static django_bootstrap5 %}
+{% block title %}Add I-Ds to {{ session.meeting }} : {{ session.group.acronym }}{% endblock %}
+{% block pagehead %}{{ form.media.css }}{% endblock %}
+{% block content %}
+    {% origin %}
+    <h1>
+        Add Recordings to {{ session.meeting }}
+        {% if session_number %}: Session {{ session_number }}{% endif %}
+        <br>
+        <small class="text-body-secondary">{{ session.group.acronym }}
+            {% if session.name %}: {{ session.name }}{% endif %}
+        </small>
+    </h1>
+    {% if session.is_material_submission_cutoff %}
+        <div class="alert alert-warning my-3">
+            The deadline for submission corrections has passed. This may affect published proceedings.
+        </div>
+    {% endif %}
+    {% if already_linked|length > 0 %}
+        <h2 class="mt-5">Recordings already linked to this session</h2>
+        <form method="post" id="delete_recordings_form">
+            {% csrf_token %}
+            <table class="table table-sm table-striped">
+                <thead>
+                <tr>
+                    <th scope="col">Title</th>
+                    <th scope="col">URL</th>
+                    <th scope="col">Delete</th>
+                </tr>
+                </thead>
+                <tbody>
+                {% for sp in already_linked %}{% with recording_doc=sp.document %}
+                    <tr>
+                        <td>{{ recording_doc.title }}</td>
+                        <td><a href="{{ recording_doc.external_url }}">{{ recording_doc.external_url }}</a></td>
+                        <td>
+                            <button type="submit"
+                                    aria-label="Delete {{ recording_doc.title }}"
+                                    class="btn btn-danger"
+                                    name="delete"
+                                    value="{{ sp.pk }}">
+                                Delete
+                            </button>
+                        </td>
+                    </tr>
+                {% endwith %}{% endfor %}
+                </tbody>
+            </table>
+        </form>
+    {% endif %}
+    <dialog id="delete_confirm_dialog">
+        <p>Really delete the link to <a href="#" id="delete_confirm_link">(default)</a>?</p>
+        <form method="post" class="d-flex justify-content-between">
+            {% csrf_token %}
+            <button class="btn btn-secondary" type="button" id="delete_confirm_cancel">Cancel</button>
+            <button class="btn btn-danger" type="submit" name="delete" id="delete_confirm_submit">Delete</button>
+        </form>
+    </dialog>
+    <h2 class="mt-5">Add a recording to this session</h2>
+    <form method="post">
+        {% csrf_token %}
+        {% bootstrap_form form %}
+        <button class="btn btn-{% if session.is_material_submission_cutoff %}warning{% else %}primary{% endif %}"
+                type="submit">
+            Add recording
+        </button>
+        <a class="btn btn-secondary float-end"
+           href="{% url 'ietf.meeting.views.session_details' num=session.meeting.number acronym=session.group.acronym %}">
+            Back
+        </a>
+    </form>
+{% endblock %}
+{% block js %}
+    {{ form.media.js }}
+    <script src="{% static 'ietf/js/add_session_recordings.js' %}"></script>
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/meeting/session_details_panel.html b/ietf/templates/meeting/session_details_panel.html
index a0f5884b9d..9b7a192f05 100644
--- a/ietf/templates/meeting/session_details_panel.html
+++ b/ietf/templates/meeting/session_details_panel.html
@@ -368,5 +368,13 @@ <h3 class="mt-4">Notes and recordings</h3>
             </tbody>
         </table>
     {% endif %}
+
+    {% if can_manage_materials %}
+        <a class="btn btn-primary"
+            href="{% url 'ietf.meeting.views.add_session_recordings' session_id=session.pk num=session.meeting.number %}">
+            Link additional recordings to session
+        </a>
+    {% endif %}
+
 {% endwith %}{% endwith %}
 {% endfor %}
diff --git a/package.json b/package.json
index b3d36b349c..6d1591ca15 100644
--- a/package.json
+++ b/package.json
@@ -111,6 +111,7 @@
         "ietf/static/images/irtf-logo-card.png",
         "ietf/static/images/irtf-logo-white.svg",
         "ietf/static/images/irtf-logo.svg",
+        "ietf/static/js/add_session_recordings.js",
         "ietf/static/js/agenda_filter.js",
         "ietf/static/js/agenda_materials.js",
         "ietf/static/js/complete-review.js",

From 83448cd7f2a96587711f990e8a63e6cacc05e095 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 31 Jan 2025 10:36:15 -0600
Subject: [PATCH 248/656] chore: remove unused code (#8490)

---
 .../commands/generate_draft_recursive_txt.py  | 17 -----------
 ietf/ipr/tests.py                             |  6 ----
 ietf/ipr/urls.py                              |  2 --
 ietf/ipr/utils.py                             | 25 ----------------
 ietf/ipr/views.py                             | 29 -------------------
 5 files changed, 79 deletions(-)
 delete mode 100644 ietf/ipr/management/commands/generate_draft_recursive_txt.py

diff --git a/ietf/ipr/management/commands/generate_draft_recursive_txt.py b/ietf/ipr/management/commands/generate_draft_recursive_txt.py
deleted file mode 100644
index 2c61a7604f..0000000000
--- a/ietf/ipr/management/commands/generate_draft_recursive_txt.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Copyright The IETF Trust 2014-2021, All Rights Reserved
-# -*- coding: utf-8 -*-
-
-
-from django.core.management.base import BaseCommand, CommandError
-
-from ietf.ipr.utils import generate_draft_recursive_txt
-
-
-class Command(BaseCommand):
-    help = ("Generate machine-readable list of IPR disclosures by Internet-Draft name (recursive)")
-
-    def handle(self, *args, **options):
-        try:
-            generate_draft_recursive_txt()
-        except (ValueError, IOError) as e:
-            raise CommandError(e)
diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py
index 67be278dbb..beb4d7fc1c 100644
--- a/ietf/ipr/tests.py
+++ b/ietf/ipr/tests.py
@@ -161,12 +161,6 @@ def test_ipr_history(self):
         r = self.client.get(urlreverse("ietf.ipr.views.history", kwargs=dict(id=ipr.pk)))
         self.assertContains(r, ipr.title)
 
-    def test_iprs_for_drafts(self):
-        draft=WgDraftFactory()
-        ipr = HolderIprDisclosureFactory(docs=[draft,])
-        r = self.client.get(urlreverse("ietf.ipr.views.by_draft_txt"))
-        self.assertContains(r, draft.name)
-        self.assertContains(r, str(ipr.pk))
 
     def test_about(self):
         r = self.client.get(urlreverse("ietf.ipr.views.about"))
diff --git a/ietf/ipr/urls.py b/ietf/ipr/urls.py
index 2b6abee31c..84ed04a66b 100644
--- a/ietf/ipr/urls.py
+++ b/ietf/ipr/urls.py
@@ -12,8 +12,6 @@
     url(r'^admin/$', RedirectView.as_view(url=reverse_lazy('ietf.ipr.views.admin',kwargs={'state':'pending'}), permanent=True)),
     url(r'^admin/(?P<state>pending|removed|parked)/$', views.admin),
     url(r'^ajax/search/$', views.ajax_search),
-    url(r'^by-draft/$', views.by_draft_txt),
-    url(r'^by-draft-recursive/$', views.by_draft_recursive_txt),
     url(r'^(?P<id>\d+)/$', views.show),
     url(r'^(?P<id>\d+)/addcomment/$', views.add_comment),
     url(r'^(?P<id>\d+)/addemail/$', views.add_email),
diff --git a/ietf/ipr/utils.py b/ietf/ipr/utils.py
index 8f0b9cf3f2..7e569a1d1d 100644
--- a/ietf/ipr/utils.py
+++ b/ietf/ipr/utils.py
@@ -4,7 +4,6 @@
 from textwrap import dedent
 
 from ietf.ipr.mail import process_response_email, UndeliverableIprResponseError
-from ietf.ipr.models import IprDocRel
 
 import debug                            # pyflakes:ignore
 
@@ -64,30 +63,6 @@ def related_docs(doc, relationship=('replaces', 'obs'), reverse_relationship=("b
     return list(set(results))
 
 
-def generate_draft_recursive_txt():
-    docipr = {}
-
-    for o in IprDocRel.objects.filter(disclosure__state='posted').select_related('document'):
-        doc = o.document
-        name = doc.name
-        related_set = set(doc) | set(doc.all_related_that_doc(('obs', 'replaces')))
-        for related in related_set:
-            name = related.name
-            if name.startswith("rfc"):
-                name = name.upper()
-            if not name in docipr:
-                docipr[name] = []
-            docipr[name].append(o.disclosure_id)
-
-    lines = [ "# Machine-readable list of IPR disclosures by Internet-Draft name" ]
-    for name, iprs in docipr.items():
-        lines.append(name + "\t" + "\t".join(str(ipr_id) for ipr_id in sorted(iprs)))
-
-    data = '\n'.join(lines)
-    filename = '/a/ietfdata/derived/ipr_draft_recursive.txt'
-    with open(filename, 'w') as f:
-        f.write(data)
-
     
 def ingest_response_email(message: bytes):
     from ietf.api.views import EmailIngestionError  # avoid circular import
diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py
index 0347c4d78d..24453df2d2 100644
--- a/ietf/ipr/views.py
+++ b/ietf/ipr/views.py
@@ -445,35 +445,6 @@ def history(request, id):
         'selected_tab_entry':'history'
     })
 
-def by_draft_txt(request):
-    docipr = {}
-
-    for o in IprDocRel.objects.filter(disclosure__state='posted').select_related('document'):
-        name = o.document.name
-        if name.startswith("rfc"):
-            name = name.upper()
-
-        if not name in docipr:
-            docipr[name] = []
-
-        docipr[name].append(o.disclosure_id)
-
-    lines = [ "# Machine-readable list of IPR disclosures by draft name" ]
-    for name, iprs in docipr.items():
-        lines.append(name + "\t" + "\t".join(str(ipr_id) for ipr_id in sorted(iprs)))
-
-    return HttpResponse("\n".join(lines), content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET)
-
-def by_draft_recursive_txt(request):
-    """Returns machine-readable list of IPR disclosures by draft name, recursive.
-    NOTE: this view is expensive and should be removed _after_ tools.ietf.org is retired,
-    including util function and management commands that generate the content for
-    this view."""
-
-    with open('/a/ietfdata/derived/ipr_draft_recursive.txt') as f:
-        content = f.read()
-    return HttpResponse(content, content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET)
-
 
 def new(request, _type, updates=None):
     """Submit a new IPR Disclosure.  If the updates field != None, this disclosure

From f85878c2e8acf6643288cfdf71ce1d97e26c8a0e Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 31 Jan 2025 15:51:24 -0600
Subject: [PATCH 249/656] fix: avoid writing a file (#8491)

---
 ietf/meeting/utils.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index e231cbed87..6e681fdc31 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -5,13 +5,13 @@
 import os
 import pytz
 import subprocess
-import tempfile
 
 from collections import defaultdict
 from pathlib import Path
 
 from django.conf import settings
 from django.contrib import messages
+from django.core.files.base import ContentFile
 from django.db.models import OuterRef, Subquery, TextField, Q, Value
 from django.db.models.functions import Coalesce
 from django.template.loader import render_to_string
@@ -225,12 +225,7 @@ def generate_bluesheet(request, session):
             'session': session,
             'data': data,
         })
-    fd, name = tempfile.mkstemp(suffix=".txt", text=True)
-    os.close(fd)
-    with open(name, "w") as file:
-        file.write(text)
-    with open(name, "br") as file:
-        return save_bluesheet(request, session, file)
+    return save_bluesheet(request, session, ContentFile(text.encode("utf-8"), name="unusednamepartsothereisanextension.txt"))
 
 
 def finalize(request, meeting):

From e0b1fd29ee8cfdcb4825e8a5891ad9c598ea56f6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 2 Feb 2025 18:04:24 -0500
Subject: [PATCH 250/656] chore(deps): bump the /dev/deploy-to-container npm
 dependencies (#8440)

Bumps the npm group in /dev/deploy-to-container with 2 updates: [dockerode](https://github.com/apocas/dockerode) and [fs-extra](https://github.com/jprichardson/node-fs-extra).


Updates `dockerode` from 4.0.3 to 4.0.4
- [Release notes](https://github.com/apocas/dockerode/releases)
- [Commits](https://github.com/apocas/dockerode/compare/v4.0.3...v4.0.4)

Updates `fs-extra` from 11.2.0 to 11.3.0
- [Changelog](https://github.com/jprichardson/node-fs-extra/blob/master/CHANGELOG.md)
- [Commits](https://github.com/jprichardson/node-fs-extra/compare/11.2.0...11.3.0)

---
updated-dependencies:
- dependency-name: dockerode
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
- dependency-name: fs-extra
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/deploy-to-container/package-lock.json | 63 +++++++++++++----------
 dev/deploy-to-container/package.json      |  4 +-
 2 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json
index 86cb1f732f..03327083b1 100644
--- a/dev/deploy-to-container/package-lock.json
+++ b/dev/deploy-to-container/package-lock.json
@@ -6,8 +6,8 @@
     "": {
       "name": "deploy-to-container",
       "dependencies": {
-        "dockerode": "^4.0.3",
-        "fs-extra": "^11.2.0",
+        "dockerode": "^4.0.4",
+        "fs-extra": "^11.3.0",
         "nanoid": "5.0.9",
         "nanoid-dictionary": "5.0.0-beta.1",
         "slugify": "1.6.6",
@@ -258,6 +258,7 @@
       "version": "0.2.6",
       "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz",
       "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==",
+      "license": "MIT",
       "dependencies": {
         "safer-buffer": "~2.1.0"
       }
@@ -290,6 +291,7 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
       "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==",
+      "license": "BSD-3-Clause",
       "dependencies": {
         "tweetnacl": "^0.14.3"
       }
@@ -376,6 +378,7 @@
       "version": "4.4.0",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
       "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
+      "license": "MIT",
       "dependencies": {
         "ms": "^2.1.3"
       },
@@ -389,9 +392,10 @@
       }
     },
     "node_modules/docker-modem": {
-      "version": "5.0.5",
-      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.5.tgz",
-      "integrity": "sha512-Cxw8uEcvNTRmsQuGqzzfiCnfGgf96tVJItLh8taOX0miTcIBALKH5TckCSuZbpbjP7uhAl81dOL9sxfa6HgCIg==",
+      "version": "5.0.6",
+      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz",
+      "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==",
+      "license": "Apache-2.0",
       "dependencies": {
         "debug": "^4.1.1",
         "readable-stream": "^3.5.0",
@@ -403,14 +407,15 @@
       }
     },
     "node_modules/dockerode": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.3.tgz",
-      "integrity": "sha512-QSXJFcBQNaGZO6U3qWW4B7p8yRIJn/dWmvL2AQWfO/bjptBBO6QYdVkYSYFz9qoivP2jsOHZfmXMAfrK0BMKyg==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.4.tgz",
+      "integrity": "sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==",
+      "license": "Apache-2.0",
       "dependencies": {
         "@balena/dockerignore": "^1.0.2",
         "@grpc/grpc-js": "^1.11.1",
         "@grpc/proto-loader": "^0.7.13",
-        "docker-modem": "^5.0.5",
+        "docker-modem": "^5.0.6",
         "protobufjs": "^7.3.2",
         "tar-fs": "~2.0.1",
         "uuid": "^10.0.0"
@@ -466,9 +471,10 @@
       "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
     },
     "node_modules/fs-extra": {
-      "version": "11.2.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz",
-      "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==",
+      "version": "11.3.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz",
+      "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==",
+      "license": "MIT",
       "dependencies": {
         "graceful-fs": "^4.2.0",
         "jsonfile": "^6.0.1",
@@ -651,12 +657,14 @@
     "node_modules/ms": {
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
-      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+      "license": "MIT"
     },
     "node_modules/nan": {
       "version": "2.22.0",
       "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz",
       "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==",
+      "license": "MIT",
       "optional": true
     },
     "node_modules/nanoid": {
@@ -804,7 +812,8 @@
     "node_modules/safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
-      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
+      "license": "MIT"
     },
     "node_modules/shebang-command": {
       "version": "2.0.0",
@@ -847,7 +856,8 @@
     "node_modules/split-ca": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz",
-      "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ=="
+      "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==",
+      "license": "ISC"
     },
     "node_modules/ssh2": {
       "version": "1.16.0",
@@ -1010,7 +1020,8 @@
     "node_modules/tweetnacl": {
       "version": "0.14.5",
       "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
-      "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA=="
+      "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==",
+      "license": "Unlicense"
     },
     "node_modules/undici-types": {
       "version": "6.20.0",
@@ -1411,9 +1422,9 @@
       }
     },
     "docker-modem": {
-      "version": "5.0.5",
-      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.5.tgz",
-      "integrity": "sha512-Cxw8uEcvNTRmsQuGqzzfiCnfGgf96tVJItLh8taOX0miTcIBALKH5TckCSuZbpbjP7uhAl81dOL9sxfa6HgCIg==",
+      "version": "5.0.6",
+      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz",
+      "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==",
       "requires": {
         "debug": "^4.1.1",
         "readable-stream": "^3.5.0",
@@ -1422,14 +1433,14 @@
       }
     },
     "dockerode": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.3.tgz",
-      "integrity": "sha512-QSXJFcBQNaGZO6U3qWW4B7p8yRIJn/dWmvL2AQWfO/bjptBBO6QYdVkYSYFz9qoivP2jsOHZfmXMAfrK0BMKyg==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.4.tgz",
+      "integrity": "sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==",
       "requires": {
         "@balena/dockerignore": "^1.0.2",
         "@grpc/grpc-js": "^1.11.1",
         "@grpc/proto-loader": "^0.7.13",
-        "docker-modem": "^5.0.5",
+        "docker-modem": "^5.0.6",
         "protobufjs": "^7.3.2",
         "tar-fs": "~2.0.1",
         "uuid": "^10.0.0"
@@ -1473,9 +1484,9 @@
       "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
     },
     "fs-extra": {
-      "version": "11.2.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz",
-      "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==",
+      "version": "11.3.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz",
+      "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==",
       "requires": {
         "graceful-fs": "^4.2.0",
         "jsonfile": "^6.0.1",
diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json
index 53a362b9d0..1f54745ebf 100644
--- a/dev/deploy-to-container/package.json
+++ b/dev/deploy-to-container/package.json
@@ -2,8 +2,8 @@
   "name": "deploy-to-container",
   "type": "module",
   "dependencies": {
-    "dockerode": "^4.0.3",
-    "fs-extra": "^11.2.0",
+    "dockerode": "^4.0.4",
+    "fs-extra": "^11.3.0",
     "nanoid": "5.0.9",
     "nanoid-dictionary": "5.0.0-beta.1",
     "slugify": "1.6.6",

From 373623da426867c32610e41f4444725fddc6b173 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 2 Feb 2025 18:05:24 -0500
Subject: [PATCH 251/656] chore(deps): bump the /dev/diff npm dependencies
 (#8439)

Bumps the npm group with 3 updates in the /dev/diff directory: [chalk](https://github.com/chalk/chalk), [dockerode](https://github.com/apocas/dockerode) and [fs-extra](https://github.com/jprichardson/node-fs-extra).


Updates `chalk` from 5.3.0 to 5.4.1
- [Release notes](https://github.com/chalk/chalk/releases)
- [Commits](https://github.com/chalk/chalk/compare/v5.3.0...v5.4.1)

Updates `dockerode` from 4.0.2 to 4.0.4
- [Release notes](https://github.com/apocas/dockerode/releases)
- [Commits](https://github.com/apocas/dockerode/compare/v4.0.2...v4.0.4)

Updates `fs-extra` from 11.2.0 to 11.3.0
- [Changelog](https://github.com/jprichardson/node-fs-extra/blob/master/CHANGELOG.md)
- [Commits](https://github.com/jprichardson/node-fs-extra/compare/11.2.0...11.3.0)

---
updated-dependencies:
- dependency-name: chalk
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm
- dependency-name: dockerode
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
- dependency-name: fs-extra
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/diff/package-lock.json | 407 +++++++++++++++++++++++++++++++------
 dev/diff/package.json      |   6 +-
 2 files changed, 348 insertions(+), 65 deletions(-)

diff --git a/dev/diff/package-lock.json b/dev/diff/package-lock.json
index 87aaf3f16f..4e92d2a6e0 100644
--- a/dev/diff/package-lock.json
+++ b/dev/diff/package-lock.json
@@ -6,11 +6,11 @@
     "": {
       "name": "diff",
       "dependencies": {
-        "chalk": "^5.3.0",
-        "dockerode": "^4.0.2",
+        "chalk": "^5.4.1",
+        "dockerode": "^4.0.4",
         "enquirer": "^2.4.1",
         "extract-zip": "^2.0.1",
-        "fs-extra": "^11.2.0",
+        "fs-extra": "^11.3.0",
         "got": "^13.0.0",
         "keypress": "^0.2.1",
         "listr2": "^6.6.1",
@@ -29,6 +29,37 @@
       "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz",
       "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q=="
     },
+    "node_modules/@grpc/grpc-js": {
+      "version": "1.12.5",
+      "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.12.5.tgz",
+      "integrity": "sha512-d3iiHxdpg5+ZcJ6jnDSOT8Z0O0VMVGy34jAnYLUX8yd36b1qn8f1TwOA/Lc7TsOh03IkPJ38eGI5qD2EjNkoEA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@grpc/proto-loader": "^0.7.13",
+        "@js-sdsl/ordered-map": "^4.4.2"
+      },
+      "engines": {
+        "node": ">=12.10.0"
+      }
+    },
+    "node_modules/@grpc/proto-loader": {
+      "version": "0.7.13",
+      "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.13.tgz",
+      "integrity": "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "lodash.camelcase": "^4.3.0",
+        "long": "^5.0.0",
+        "protobufjs": "^7.2.5",
+        "yargs": "^17.7.2"
+      },
+      "bin": {
+        "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/@isaacs/cliui": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
@@ -129,6 +160,16 @@
         "node": ">=18.0.0"
       }
     },
+    "node_modules/@js-sdsl/ordered-map": {
+      "version": "4.4.2",
+      "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz",
+      "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==",
+      "license": "MIT",
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/js-sdsl"
+      }
+    },
     "node_modules/@pkgjs/parseargs": {
       "version": "0.11.0",
       "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
@@ -138,6 +179,70 @@
         "node": ">=14"
       }
     },
+    "node_modules/@protobufjs/aspromise": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
+      "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/base64": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
+      "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/codegen": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
+      "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/eventemitter": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
+      "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/fetch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
+      "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.1",
+        "@protobufjs/inquire": "^1.1.0"
+      }
+    },
+    "node_modules/@protobufjs/float": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
+      "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/inquire": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
+      "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/path": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
+      "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/pool": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
+      "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/@protobufjs/utf8": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
+      "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==",
+      "license": "BSD-3-Clause"
+    },
     "node_modules/@sindresorhus/is": {
       "version": "5.3.0",
       "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.3.0.tgz",
@@ -168,8 +273,7 @@
     "node_modules/@types/node": {
       "version": "18.6.5",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-18.6.5.tgz",
-      "integrity": "sha512-Xjt5ZGUa5WusGZJ4WJPbOT8QOqp6nDynVFRKcUt32bOgvXEoc6o085WNkYTMO7ifAj2isEfQQ2cseE+wT6jsRw==",
-      "optional": true
+      "integrity": "sha512-Xjt5ZGUa5WusGZJ4WJPbOT8QOqp6nDynVFRKcUt32bOgvXEoc6o085WNkYTMO7ifAj2isEfQQ2cseE+wT6jsRw=="
     },
     "node_modules/@types/yauzl": {
       "version": "2.10.0",
@@ -228,6 +332,7 @@
       "version": "0.2.6",
       "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz",
       "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==",
+      "license": "MIT",
       "dependencies": {
         "safer-buffer": "~2.1.0"
       }
@@ -260,6 +365,7 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
       "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==",
+      "license": "BSD-3-Clause",
       "dependencies": {
         "tweetnacl": "^0.14.3"
       }
@@ -315,9 +421,10 @@
       }
     },
     "node_modules/chalk": {
-      "version": "5.3.0",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz",
-      "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==",
+      "version": "5.4.1",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz",
+      "integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==",
+      "license": "MIT",
       "engines": {
         "node": "^12.17.0 || ^14.13 || >=16.0.0"
       },
@@ -440,14 +547,14 @@
       "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="
     },
     "node_modules/cpu-features": {
-      "version": "0.0.9",
-      "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.9.tgz",
-      "integrity": "sha512-AKjgn2rP2yJyfbepsmLfiYcmtNn/2eUvocUyM/09yB0YDiz39HteK/5/T4Onf0pmdYDMgkBoGvRLvEguzyL7wQ==",
+      "version": "0.0.10",
+      "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz",
+      "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==",
       "hasInstallScript": true,
       "optional": true,
       "dependencies": {
         "buildcheck": "~0.0.6",
-        "nan": "^2.17.0"
+        "nan": "^2.19.0"
       },
       "engines": {
         "node": ">=10.0.0"
@@ -516,9 +623,10 @@
       }
     },
     "node_modules/docker-modem": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.3.tgz",
-      "integrity": "sha512-89zhop5YVhcPEt5FpUFGr3cDyceGhq/F9J+ZndQ4KfqNvfbJpPMfgeixFgUj5OjCYAboElqODxY5Z1EBsSa6sg==",
+      "version": "5.0.6",
+      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz",
+      "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==",
+      "license": "Apache-2.0",
       "dependencies": {
         "debug": "^4.1.1",
         "readable-stream": "^3.5.0",
@@ -530,13 +638,18 @@
       }
     },
     "node_modules/dockerode": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.2.tgz",
-      "integrity": "sha512-9wM1BVpVMFr2Pw3eJNXrYYt6DT9k0xMcsSCjtPvyQ+xa1iPg/Mo3T/gUcwI0B2cczqCeCYRPF8yFYDwtFXT0+w==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.4.tgz",
+      "integrity": "sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==",
+      "license": "Apache-2.0",
       "dependencies": {
         "@balena/dockerignore": "^1.0.2",
-        "docker-modem": "^5.0.3",
-        "tar-fs": "~2.0.1"
+        "@grpc/grpc-js": "^1.11.1",
+        "@grpc/proto-loader": "^0.7.13",
+        "docker-modem": "^5.0.6",
+        "protobufjs": "^7.3.2",
+        "tar-fs": "~2.0.1",
+        "uuid": "^10.0.0"
       },
       "engines": {
         "node": ">= 8.0"
@@ -666,9 +779,10 @@
       "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
     },
     "node_modules/fs-extra": {
-      "version": "11.2.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz",
-      "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==",
+      "version": "11.3.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz",
+      "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==",
+      "license": "MIT",
       "dependencies": {
         "graceful-fs": "^4.2.0",
         "jsonfile": "^6.0.1",
@@ -949,6 +1063,12 @@
       "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz",
       "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw=="
     },
+    "node_modules/lodash.camelcase": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
+      "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==",
+      "license": "MIT"
+    },
     "node_modules/log-update": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/log-update/-/log-update-5.0.1.tgz",
@@ -1040,6 +1160,12 @@
         "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
       }
     },
+    "node_modules/long": {
+      "version": "5.2.4",
+      "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz",
+      "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg==",
+      "license": "Apache-2.0"
+    },
     "node_modules/lowercase-keys": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
@@ -1145,9 +1271,10 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
     "node_modules/nan": {
-      "version": "2.18.0",
-      "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz",
-      "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==",
+      "version": "2.22.0",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz",
+      "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==",
+      "license": "MIT",
       "optional": true
     },
     "node_modules/normalize-url": {
@@ -1230,6 +1357,30 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/protobufjs": {
+      "version": "7.4.0",
+      "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz",
+      "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==",
+      "hasInstallScript": true,
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "@protobufjs/aspromise": "^1.1.2",
+        "@protobufjs/base64": "^1.1.2",
+        "@protobufjs/codegen": "^2.0.4",
+        "@protobufjs/eventemitter": "^1.1.0",
+        "@protobufjs/fetch": "^1.1.0",
+        "@protobufjs/float": "^1.0.2",
+        "@protobufjs/inquire": "^1.1.0",
+        "@protobufjs/path": "^1.1.2",
+        "@protobufjs/pool": "^1.1.0",
+        "@protobufjs/utf8": "^1.1.0",
+        "@types/node": ">=13.7.0",
+        "long": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/pump": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
@@ -1349,7 +1500,8 @@
     "node_modules/safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
-      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
+      "license": "MIT"
     },
     "node_modules/shebang-command": {
       "version": "2.0.0",
@@ -1415,12 +1567,13 @@
     "node_modules/split-ca": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz",
-      "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ=="
+      "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==",
+      "license": "ISC"
     },
     "node_modules/ssh2": {
-      "version": "1.15.0",
-      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.15.0.tgz",
-      "integrity": "sha512-C0PHgX4h6lBxYx7hcXwu3QWdh4tg6tZZsTfXcdvc5caW/EMxaB4H9dWsl7qk+F7LAW762hp8VbXOX7x4xUYvEw==",
+      "version": "1.16.0",
+      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz",
+      "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==",
       "hasInstallScript": true,
       "dependencies": {
         "asn1": "^0.2.6",
@@ -1430,8 +1583,8 @@
         "node": ">=10.16.0"
       },
       "optionalDependencies": {
-        "cpu-features": "~0.0.9",
-        "nan": "^2.18.0"
+        "cpu-features": "~0.0.10",
+        "nan": "^2.20.0"
       }
     },
     "node_modules/string_decoder": {
@@ -1578,7 +1731,8 @@
     "node_modules/tweetnacl": {
       "version": "0.14.5",
       "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
-      "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA=="
+      "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==",
+      "license": "Unlicense"
     },
     "node_modules/type-fest": {
       "version": "1.4.0",
@@ -1604,6 +1758,19 @@
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
       "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
     },
+    "node_modules/uuid": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
+      "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",
+      "funding": [
+        "https://github.com/sponsors/broofa",
+        "https://github.com/sponsors/ctavan"
+      ],
+      "license": "MIT",
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
     "node_modules/which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@@ -1713,6 +1880,26 @@
       "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz",
       "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q=="
     },
+    "@grpc/grpc-js": {
+      "version": "1.12.5",
+      "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.12.5.tgz",
+      "integrity": "sha512-d3iiHxdpg5+ZcJ6jnDSOT8Z0O0VMVGy34jAnYLUX8yd36b1qn8f1TwOA/Lc7TsOh03IkPJ38eGI5qD2EjNkoEA==",
+      "requires": {
+        "@grpc/proto-loader": "^0.7.13",
+        "@js-sdsl/ordered-map": "^4.4.2"
+      }
+    },
+    "@grpc/proto-loader": {
+      "version": "0.7.13",
+      "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.13.tgz",
+      "integrity": "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==",
+      "requires": {
+        "lodash.camelcase": "^4.3.0",
+        "long": "^5.0.0",
+        "protobufjs": "^7.2.5",
+        "yargs": "^17.7.2"
+      }
+    },
     "@isaacs/cliui": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
@@ -1779,12 +1966,71 @@
         "minipass": "^7.0.4"
       }
     },
+    "@js-sdsl/ordered-map": {
+      "version": "4.4.2",
+      "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz",
+      "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw=="
+    },
     "@pkgjs/parseargs": {
       "version": "0.11.0",
       "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
       "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
       "optional": true
     },
+    "@protobufjs/aspromise": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
+      "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="
+    },
+    "@protobufjs/base64": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
+      "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="
+    },
+    "@protobufjs/codegen": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
+      "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="
+    },
+    "@protobufjs/eventemitter": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
+      "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="
+    },
+    "@protobufjs/fetch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
+      "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
+      "requires": {
+        "@protobufjs/aspromise": "^1.1.1",
+        "@protobufjs/inquire": "^1.1.0"
+      }
+    },
+    "@protobufjs/float": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
+      "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="
+    },
+    "@protobufjs/inquire": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
+      "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="
+    },
+    "@protobufjs/path": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
+      "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="
+    },
+    "@protobufjs/pool": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
+      "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="
+    },
+    "@protobufjs/utf8": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
+      "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="
+    },
     "@sindresorhus/is": {
       "version": "5.3.0",
       "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.3.0.tgz",
@@ -1806,8 +2052,7 @@
     "@types/node": {
       "version": "18.6.5",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-18.6.5.tgz",
-      "integrity": "sha512-Xjt5ZGUa5WusGZJ4WJPbOT8QOqp6nDynVFRKcUt32bOgvXEoc6o085WNkYTMO7ifAj2isEfQQ2cseE+wT6jsRw==",
-      "optional": true
+      "integrity": "sha512-Xjt5ZGUa5WusGZJ4WJPbOT8QOqp6nDynVFRKcUt32bOgvXEoc6o085WNkYTMO7ifAj2isEfQQ2cseE+wT6jsRw=="
     },
     "@types/yauzl": {
       "version": "2.10.0",
@@ -1909,9 +2154,9 @@
       }
     },
     "chalk": {
-      "version": "5.3.0",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz",
-      "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w=="
+      "version": "5.4.1",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz",
+      "integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w=="
     },
     "chownr": {
       "version": "1.1.4",
@@ -1994,13 +2239,13 @@
       "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="
     },
     "cpu-features": {
-      "version": "0.0.9",
-      "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.9.tgz",
-      "integrity": "sha512-AKjgn2rP2yJyfbepsmLfiYcmtNn/2eUvocUyM/09yB0YDiz39HteK/5/T4Onf0pmdYDMgkBoGvRLvEguzyL7wQ==",
+      "version": "0.0.10",
+      "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz",
+      "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==",
       "optional": true,
       "requires": {
         "buildcheck": "~0.0.6",
-        "nan": "^2.17.0"
+        "nan": "^2.19.0"
       }
     },
     "cross-spawn": {
@@ -2042,9 +2287,9 @@
       "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg=="
     },
     "docker-modem": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.3.tgz",
-      "integrity": "sha512-89zhop5YVhcPEt5FpUFGr3cDyceGhq/F9J+ZndQ4KfqNvfbJpPMfgeixFgUj5OjCYAboElqODxY5Z1EBsSa6sg==",
+      "version": "5.0.6",
+      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz",
+      "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==",
       "requires": {
         "debug": "^4.1.1",
         "readable-stream": "^3.5.0",
@@ -2053,13 +2298,17 @@
       }
     },
     "dockerode": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.2.tgz",
-      "integrity": "sha512-9wM1BVpVMFr2Pw3eJNXrYYt6DT9k0xMcsSCjtPvyQ+xa1iPg/Mo3T/gUcwI0B2cczqCeCYRPF8yFYDwtFXT0+w==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.4.tgz",
+      "integrity": "sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==",
       "requires": {
         "@balena/dockerignore": "^1.0.2",
-        "docker-modem": "^5.0.3",
-        "tar-fs": "~2.0.1"
+        "@grpc/grpc-js": "^1.11.1",
+        "@grpc/proto-loader": "^0.7.13",
+        "docker-modem": "^5.0.6",
+        "protobufjs": "^7.3.2",
+        "tar-fs": "~2.0.1",
+        "uuid": "^10.0.0"
       }
     },
     "eastasianwidth": {
@@ -2155,9 +2404,9 @@
       "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
     },
     "fs-extra": {
-      "version": "11.2.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz",
-      "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==",
+      "version": "11.3.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz",
+      "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==",
       "requires": {
         "graceful-fs": "^4.2.0",
         "jsonfile": "^6.0.1",
@@ -2342,6 +2591,11 @@
       "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz",
       "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw=="
     },
+    "lodash.camelcase": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
+      "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
+    },
     "log-update": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/log-update/-/log-update-5.0.1.tgz",
@@ -2399,6 +2653,11 @@
         }
       }
     },
+    "long": {
+      "version": "5.2.4",
+      "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz",
+      "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg=="
+    },
     "lowercase-keys": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
@@ -2462,9 +2721,9 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
     "nan": {
-      "version": "2.18.0",
-      "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz",
-      "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==",
+      "version": "2.22.0",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz",
+      "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==",
       "optional": true
     },
     "normalize-url": {
@@ -2517,6 +2776,25 @@
       "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-6.1.1.tgz",
       "integrity": "sha512-mQUvGU6aUFQ+rNvTIAcZuWGRT9a6f6Yrg9bHs4ImKF+HZCEK+plBvnAZYSIQztknZF2qnzNtr6F8s0+IuptdlQ=="
     },
+    "protobufjs": {
+      "version": "7.4.0",
+      "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz",
+      "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==",
+      "requires": {
+        "@protobufjs/aspromise": "^1.1.2",
+        "@protobufjs/base64": "^1.1.2",
+        "@protobufjs/codegen": "^2.0.4",
+        "@protobufjs/eventemitter": "^1.1.0",
+        "@protobufjs/fetch": "^1.1.0",
+        "@protobufjs/float": "^1.0.2",
+        "@protobufjs/inquire": "^1.1.0",
+        "@protobufjs/path": "^1.1.2",
+        "@protobufjs/pool": "^1.1.0",
+        "@protobufjs/utf8": "^1.1.0",
+        "@types/node": ">=13.7.0",
+        "long": "^5.0.0"
+      }
+    },
     "pump": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
@@ -2636,14 +2914,14 @@
       "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ=="
     },
     "ssh2": {
-      "version": "1.15.0",
-      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.15.0.tgz",
-      "integrity": "sha512-C0PHgX4h6lBxYx7hcXwu3QWdh4tg6tZZsTfXcdvc5caW/EMxaB4H9dWsl7qk+F7LAW762hp8VbXOX7x4xUYvEw==",
+      "version": "1.16.0",
+      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz",
+      "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==",
       "requires": {
         "asn1": "^0.2.6",
         "bcrypt-pbkdf": "^1.0.2",
-        "cpu-features": "~0.0.9",
-        "nan": "^2.18.0"
+        "cpu-features": "~0.0.10",
+        "nan": "^2.20.0"
       }
     },
     "string_decoder": {
@@ -2774,6 +3052,11 @@
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
       "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
     },
+    "uuid": {
+      "version": "10.0.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
+      "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ=="
+    },
     "which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
diff --git a/dev/diff/package.json b/dev/diff/package.json
index 4e0e1be8f4..875e242d3d 100644
--- a/dev/diff/package.json
+++ b/dev/diff/package.json
@@ -2,11 +2,11 @@
   "name": "diff",
   "type": "module",
   "dependencies": {
-    "chalk": "^5.3.0",
-    "dockerode": "^4.0.2",
+    "chalk": "^5.4.1",
+    "dockerode": "^4.0.4",
     "enquirer": "^2.4.1",
     "extract-zip": "^2.0.1",
-    "fs-extra": "^11.2.0",
+    "fs-extra": "^11.3.0",
     "got": "^13.0.0",
     "keypress": "^0.2.1",
     "listr2": "^6.6.1",

From 1fbedd7df17c2b123c64cf6cd17898ef2663bedb Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 5 Feb 2025 16:31:35 -0400
Subject: [PATCH 252/656] chore: typing fixes for factory-boy 3.3.3 (#8501)

* chore: typing fixes for factory-boy 3.3.3

* chore: more comments
---
 ietf/doc/factories.py       | 13 ++++++++-----
 ietf/ipr/tests.py           |  2 +-
 ietf/meeting/tests_views.py |  3 ++-
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py
index 50fba50c42..19aa9ecc9c 100644
--- a/ietf/doc/factories.py
+++ b/ietf/doc/factories.py
@@ -7,7 +7,7 @@
 import factory.fuzzy
 import datetime
 
-from typing import Optional         # pyflakes:ignore
+from typing import Any  # pyflakes:ignore
 
 from django.conf import settings
 from django.utils import timezone
@@ -37,13 +37,16 @@ class Meta:
         model = Document
         skip_postgeneration_save = True
 
+    # n.b., a few attributes are typed as Any so mypy won't complain when we override in subclasses
     title = factory.Faker('sentence',nb_words=5)
-    abstract = factory.Faker('paragraph', nb_sentences=5)
+    abstract: Any = factory.Faker('paragraph', nb_sentences=5)
     rev = '00'
-    std_level_id = None                 # type: Optional[str]
+    std_level_id: Any = None
     intended_std_level_id = None
     time = timezone.now()
-    expires = factory.LazyAttribute(lambda o: o.time+datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE))
+    expires: Any = factory.LazyAttribute(
+        lambda o: o.time+datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE)
+    )
     pages = factory.fuzzy.FuzzyInteger(2,400)
 
 
@@ -282,7 +285,7 @@ class Meta:
 
     type = 'added_comment'
     by = factory.SubFactory('ietf.person.factories.PersonFactory')
-    doc = factory.SubFactory(DocumentFactory)
+    doc: Any = factory.SubFactory(DocumentFactory)  # `Any` to appease mypy when a subclass overrides doc
     desc = factory.Faker('sentence',nb_words=6)
 
     @factory.lazy_attribute
diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py
index beb4d7fc1c..e0d00b5d1a 100644
--- a/ietf/ipr/tests.py
+++ b/ietf/ipr/tests.py
@@ -726,7 +726,7 @@ def test_notify_generic(self):
         self.assertIn(f'{settings.IDTRACKER_BASE_URL}{urlreverse("ietf.ipr.views.showlist")}', get_payload_text(outbox[1]).replace('\n',' '))
 
     def send_ipr_email_helper(self) -> tuple[str, IprEvent, HolderIprDisclosure]:
-        ipr = HolderIprDisclosureFactory()
+        ipr = HolderIprDisclosureFactory.create()  # call create() explicitly so mypy sees correct type
         url = urlreverse('ietf.ipr.views.email',kwargs={ "id": ipr.id })
         self.client.login(username="secretary", password="secretary+password")
         yesterday = date_today() - datetime.timedelta(1)
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 3e948f8386..94f06dc89c 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -2124,7 +2124,8 @@ def create_timeslots_url(meeting):
     @staticmethod
     def create_bare_meeting(number=120) -> Meeting:
         """Create a basic IETF meeting"""
-        return MeetingFactory(
+        # Call create() explicitly so mypy sees the correct type
+        return MeetingFactory.create(
             type_id='ietf',
             number=number,
             date=date_today() + datetime.timedelta(days=10),

From 060320d766803e479d860832a2ae9ebeb5a9d4d3 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 5 Feb 2025 18:29:32 -0400
Subject: [PATCH 253/656] feat: keep proceedings cache up to date via celery
 (#8449)

* refactor: better control proceedings caching

* refactor: move methods from views to utils

* chore: revert accidental settings change

* fix: eliminate circular import

get_schedule() with name=None should perhaps be an anti-pattern

* feat: task to recompute proceedings daily

* chore: proceedings cache lifetime = 1 day

* fix: ensure finalization is immediately reflected

* chore: update beat comments in docker-compose

* style: undo a couple whitespace changes

* test: update / refactor tests

* test: test task

* refactor: disallow positional arg to task

* refactor: add trivial test of old task
---
 docker-compose.yml                            |   1 +
 ietf/meeting/tasks.py                         |  32 ++
 ietf/meeting/tests_tasks.py                   |  51 +++
 ietf/meeting/tests_views.py                   |  91 ++++-
 ietf/meeting/utils.py                         | 171 +++++++-
 ietf/meeting/views.py                         | 157 +-------
 ietf/templates/meeting/proceedings.html       | 380 ++++++++----------
 .../meeting/proceedings_wrapper.html          |  27 ++
 8 files changed, 542 insertions(+), 368 deletions(-)
 create mode 100644 ietf/meeting/tests_tasks.py
 create mode 100644 ietf/templates/meeting/proceedings_wrapper.html

diff --git a/docker-compose.yml b/docker-compose.yml
index 65b28f54fe..9988b10c2e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -101,6 +101,7 @@ services:
 #        stop_grace_period: 1m
 #        volumes:
 #            - .:/workspace
+#            - app-assets:/assets
 
 volumes:
     postgresdb-data:
diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py
index 43cbb0a75f..2b7c2fca91 100644
--- a/ietf/meeting/tasks.py
+++ b/ietf/meeting/tasks.py
@@ -3,10 +3,42 @@
 # Celery task definitions
 #
 from celery import shared_task
+from django.utils import timezone
 
+from ietf.utils import log
+from .models import Meeting
+from .utils import generate_proceedings_content
 from .views import generate_agenda_data
 
 
 @shared_task
 def agenda_data_refresh():
     generate_agenda_data(force_refresh=True)
+
+
+@shared_task
+def proceedings_content_refresh_task(*, all=False):
+    """Refresh meeting proceedings cache
+
+    If `all` is `False`, then refreshes the cache for meetings whose numbers modulo
+    24 equal the current hour number (0-23). Scheduling the task once per hour will
+    then result in all proceedings being recomputed daily, with no more than two per
+    hour (now) or a few per hour in the next decade. That keeps the computation time
+    to under a couple minutes on our current production system.
+
+    If `all` is True, refreshes all meetings
+    """
+    now = timezone.now()
+
+    for meeting in Meeting.objects.filter(type_id="ietf").order_by("number"):
+        if meeting.proceedings_format_version == 1:
+            continue  # skip v1 proceedings, they're stored externally
+        num = meeting.get_number()  # convert str -> int
+        if num is None:
+            log.log(
+                f"Not refreshing proceedings for meeting {meeting.number}: "
+                f"type is 'ietf' but get_number() returned None"
+            )
+        elif all or (num % 24 == now.hour):
+            log.log(f"Refreshing proceedings for meeting {meeting.number}...")
+            generate_proceedings_content(meeting, force_refresh=True)
diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py
new file mode 100644
index 0000000000..c026a99835
--- /dev/null
+++ b/ietf/meeting/tests_tasks.py
@@ -0,0 +1,51 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+import datetime
+from mock import patch, call
+from ietf.utils.test_utils import TestCase
+from .factories import MeetingFactory
+from .tasks import proceedings_content_refresh_task, agenda_data_refresh
+
+
+class TaskTests(TestCase):
+    @patch("ietf.meeting.tasks.generate_agenda_data")
+    def test_agenda_data_refresh(self, mock_generate):
+        agenda_data_refresh()
+        self.assertTrue(mock_generate.called)
+        self.assertEqual(mock_generate.call_args, call(force_refresh=True))
+
+    @patch("ietf.meeting.tasks.generate_proceedings_content")
+    def test_proceedings_content_refresh_task(self, mock_generate):
+        # Generate a couple of meetings
+        meeting120 = MeetingFactory(type_id="ietf", number="120")  # 24 * 5
+        meeting127 = MeetingFactory(type_id="ietf", number="127")  # 24 * 5 + 7
+        
+        # Times to be returned
+        now_utc = datetime.datetime.now(tz=datetime.timezone.utc)
+        hour_00_utc = now_utc.replace(hour=0)
+        hour_01_utc = now_utc.replace(hour=1)
+        hour_07_utc = now_utc.replace(hour=7)
+
+        # hour 00 - should call meeting with number % 24 == 0
+        with patch("ietf.meeting.tasks.timezone.now", return_value=hour_00_utc):
+            proceedings_content_refresh_task()
+        self.assertEqual(mock_generate.call_count, 1)
+        self.assertEqual(mock_generate.call_args, call(meeting120, force_refresh=True))
+        mock_generate.reset_mock()
+    
+        # hour 01 - should call no meetings
+        with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc):
+            proceedings_content_refresh_task()
+        self.assertEqual(mock_generate.call_count, 0)
+    
+        # hour 07 - should call meeting with number % 24 == 0
+        with patch("ietf.meeting.tasks.timezone.now", return_value=hour_07_utc):
+            proceedings_content_refresh_task()
+        self.assertEqual(mock_generate.call_count, 1)
+        self.assertEqual(mock_generate.call_args, call(meeting127, force_refresh=True))
+        mock_generate.reset_mock()
+        
+        # With all=True, all should be called regardless of time. Reuse hour_01_utc which called none before
+        with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc):
+            proceedings_content_refresh_task(all=True)
+        self.assertEqual(mock_generate.call_count, 2)
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 94f06dc89c..581725dbc8 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -32,6 +32,7 @@
 from django.http import QueryDict, FileResponse
 from django.template import Context, Template
 from django.utils import timezone
+from django.utils.safestring import mark_safe
 from django.utils.text import slugify
 
 import debug           # pyflakes:ignore
@@ -46,7 +47,7 @@
 from ietf.meeting.helpers import send_interim_minutes_reminder, populate_important_dates, update_important_dates
 from ietf.meeting.models import Session, TimeSlot, Meeting, SchedTimeSessAssignment, Schedule, SessionPresentation, SlideSubmission, SchedulingEvent, Room, Constraint, ConstraintName
 from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting, make_interim_test_data
-from ietf.meeting.utils import condition_slide_order
+from ietf.meeting.utils import condition_slide_order, generate_proceedings_content
 from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting
 from ietf.meeting.utils import create_recording, delete_recording, get_next_sequence, bluesheet_data
 from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule
@@ -8296,8 +8297,7 @@ def _proceedings_file():
         path = Path(settings.BASE_DIR) / 'meeting/test_procmat.pdf'
         return path.open('rb')
 
-    def _assertMeetingHostsDisplayed(self, response, meeting):
-        pq = PyQuery(response.content)
+    def _assertMeetingHostsDisplayed(self, pq: PyQuery, meeting):
         host_divs = pq('div.host-logo')
         self.assertEqual(len(host_divs), meeting.meetinghosts.count(), 'Should have a logo for every meeting host')
         self.assertEqual(
@@ -8313,12 +8313,11 @@ def _assertMeetingHostsDisplayed(self, response, meeting):
             'Correct image and name for each host should appear in the correct order'
         )
 
-    def _assertProceedingsMaterialsDisplayed(self, response, meeting):
+    def _assertProceedingsMaterialsDisplayed(self, pq: PyQuery, meeting):
         """Checks that all (and only) active materials are linked with correct href and title"""
         expected_materials = [
             m for m in meeting.proceedings_materials.order_by('type__order') if m.active()
         ]
-        pq = PyQuery(response.content)
         links = pq('div.proceedings-material a')
         self.assertEqual(len(links), len(expected_materials), 'Should have an entry for each active ProceedingsMaterial')
         self.assertEqual(
@@ -8327,9 +8326,8 @@ def _assertProceedingsMaterialsDisplayed(self, response, meeting):
             'Correct title and link for each ProceedingsMaterial should appear in the correct order'
         )
 
-    def _assertGroupSessions(self, response, meeting):
+    def _assertGroupSessions(self,  pq: PyQuery):
         """Checks that group/sessions are present"""
-        pq = PyQuery(response.content)
         sections = ["plenaries", "gen", "iab", "editorial", "irtf", "training"]
         for section in sections:
             self.assertEqual(len(pq(f"#{section}")), 1, f"{section} section should exists in proceedings")
@@ -8337,10 +8335,9 @@ def _assertGroupSessions(self, response, meeting):
     def test_proceedings(self):
         """Proceedings should be displayed correctly
 
-        Currently only tests that the view responds with a 200 response code and checks the ProceedingsMaterials
-        at the top of the proceedings. Ought to actually test the display of the individual group/session
-        materials as well.
+        Proceedings contents are tested in detail when testing generate_proceedings_content.
         """
+        # number must be >97 (settings.PROCEEDINGS_VERSION_CHANGES)
         meeting = make_meeting_test_data(meeting=MeetingFactory(type_id='ietf', number='100'))
         session = Session.objects.filter(meeting=meeting, group__acronym="mars").first()
         GroupEventFactory(group=session.group,type='status_update')
@@ -8365,16 +8362,72 @@ def test_proceedings(self):
         self._create_proceedings_materials(meeting)
 
         url = urlreverse("ietf.meeting.views.proceedings", kwargs=dict(num=meeting.number))
-        r = self.client.get(url)
+        cached_content = mark_safe("<p>Fake proceedings content</p>")
+        with patch("ietf.meeting.views.generate_proceedings_content") as mock_gpc:
+            mock_gpc.return_value = cached_content
+            r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
+        self.assertIn(cached_content, r.content.decode())
+        self.assertTemplateUsed(r, "meeting/proceedings_wrapper.html")
+        self.assertTemplateNotUsed(r, "meeting/proceedings.html")
 
+        # These are rendered in proceedings_wrapper.html, so test them here
         if len(meeting.city) > 0:
             self.assertContains(r, meeting.city)
         if len(meeting.venue_name) > 0:
             self.assertContains(r, meeting.venue_name)
+        self._assertMeetingHostsDisplayed(PyQuery(r.content), meeting)
+
+    @patch("ietf.meeting.utils.caches")
+    def test_generate_proceedings_content(self, mock_caches):
+        # number must be >97 (settings.PROCEEDINGS_VERSION_CHANGES)
+        meeting = make_meeting_test_data(meeting=MeetingFactory(type_id='ietf', number='100'))
+
+        # First, check that by default a value in the cache is used without doing any other computation 
+        mock_default_cache = mock_caches["default"]
+        mock_default_cache.get.return_value = "a cached value"
+        result = generate_proceedings_content(meeting)
+        self.assertEqual(result, "a cached value")
+        self.assertFalse(mock_default_cache.set.called)
+        self.assertTrue(mock_default_cache.get.called)
+        cache_key = mock_default_cache.get.call_args.args[0]
+        mock_default_cache.get.reset_mock()
+    
+        # Now set up for actual computation of the proceedings content.
+        session = Session.objects.filter(meeting=meeting, group__acronym="mars").first()
+        GroupEventFactory(group=session.group,type='status_update')
+        SessionPresentationFactory(document__type_id='recording',session=session)
+        SessionPresentationFactory(document__type_id='recording',session=session,document__title="Audio recording for tests")
+
+        # Add various group sessions
+        groups = []
+        parent_groups = [
+                GroupFactory.create(type_id="area", acronym="gen"),
+                GroupFactory.create(acronym="iab"),
+                GroupFactory.create(acronym="irtf"),
+                ]
+        for parent in parent_groups:
+            groups.append(GroupFactory.create(parent=parent))
+        for acronym in ["rsab", "edu"]:
+            groups.append(GroupFactory.create(acronym=acronym))
+        for group in groups:
+            SessionFactory(meeting=meeting, group=group)
+
+        self.write_materials_files(meeting, session)
+        self._create_proceedings_materials(meeting)
+
+        # Now "empty" the mock cache and see that we compute the expected proceedings content.
+        mock_default_cache.get.return_value = None
+        proceedings_content = generate_proceedings_content(meeting)
+        self.assertTrue(mock_default_cache.get.called)
+        self.assertEqual(mock_default_cache.get.call_args.args[0], cache_key, "same cache key each time")
+        self.assertTrue(mock_default_cache.set.called)
+        self.assertEqual(mock_default_cache.set.call_args, call(cache_key, proceedings_content, timeout=86400))
+        mock_default_cache.get.reset_mock()
+        mock_default_cache.set.reset_mock()
 
         # standard items on every proceedings
-        pq = PyQuery(r.content)
+        pq = PyQuery(proceedings_content)
         self.assertNotEqual(
             pq('a[href="{}"]'.format(
                 urlreverse('ietf.meeting.views.proceedings_overview', kwargs=dict(num=meeting.number)))
@@ -8405,9 +8458,17 @@ def test_proceedings(self):
         )
 
         # configurable contents
-        self._assertMeetingHostsDisplayed(r, meeting)
-        self._assertProceedingsMaterialsDisplayed(r, meeting)
-        self._assertGroupSessions(r, meeting)
+        self._assertProceedingsMaterialsDisplayed(pq, meeting)
+        self._assertGroupSessions(pq)
+
+        # Finally, repeat the first cache test, but now with force_refresh=True. The cached value
+        # should be ignored and we should recompute the proceedings as before.
+        mock_default_cache.get.return_value = "a cached value"
+        result = generate_proceedings_content(meeting, force_refresh=True)
+        self.assertEqual(result, proceedings_content)  # should have recomputed the same thing
+        self.assertFalse(mock_default_cache.get.called, "don't bother reading cache when force_refresh is True")
+        self.assertTrue(mock_default_cache.set.called)
+        self.assertEqual(mock_default_cache.set.call_args, call(cache_key, proceedings_content, timeout=86400))
 
     def test_named_session(self):
         """Session with a name should appear separately in the proceedings"""
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index 6e681fdc31..92bae5ac23 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -3,6 +3,8 @@
 import datetime
 import itertools
 import os
+from hashlib import sha384
+
 import pytz
 import subprocess
 
@@ -11,8 +13,9 @@
 
 from django.conf import settings
 from django.contrib import messages
+from django.core.cache import caches
 from django.core.files.base import ContentFile
-from django.db.models import OuterRef, Subquery, TextField, Q, Value
+from django.db.models import OuterRef, Subquery, TextField, Q, Value, Max
 from django.db.models.functions import Coalesce
 from django.template.loader import render_to_string
 from django.utils import timezone
@@ -995,3 +998,169 @@ def participants_for_meeting(meeting):
     sessions = meeting.session_set.filter(Q(type='plenary') | Q(group__type__in=['wg', 'rg']))
     attended = Attended.objects.filter(session__in=sessions).values_list('person', flat=True).distinct()
     return (checked_in, attended)
+
+
+def generate_proceedings_content(meeting, force_refresh=False):
+    """Render proceedings content for a meeting and update cache
+    
+    :meeting: meeting whose proceedings should be rendered
+    :force_refresh: true to force regeneration and cache refresh
+    """
+    cache = caches["default"]
+    cache_version = Document.objects.filter(session__meeting__number=meeting.number).aggregate(Max('time'))["time__max"]
+    # Include proceedings_final in the bare_key so we'll always reflect that accurately, even at the cost of
+    # a recomputation in the view
+    bare_key = f"proceedings.{meeting.number}.{cache_version}.final={meeting.proceedings_final}"
+    cache_key = sha384(bare_key.encode("utf8")).hexdigest()
+    if not force_refresh:
+        cached_content = cache.get(cache_key, None)
+        if cached_content is not None:
+            return cached_content
+
+    def area_and_group_acronyms_from_session(s):
+        area = s.group_parent_at_the_time()
+        if area == None:
+            area = s.group.parent
+        group = s.group_at_the_time()
+        return (area.acronym, group.acronym)
+
+    schedule = meeting.schedule
+    sessions  = (
+        meeting.session_set.with_current_status()
+        .filter(Q(timeslotassignments__schedule__in=[schedule, schedule.base if schedule else None])
+                | Q(current_status='notmeet'))
+        .select_related()
+        .order_by('-current_status')
+    )
+
+    plenaries, _ = organize_proceedings_sessions(
+        sessions.filter(name__icontains='plenary')
+        .exclude(current_status='notmeet')
+    )
+    irtf_meeting, irtf_not_meeting = organize_proceedings_sessions(
+        sessions.filter(group__parent__acronym = 'irtf').order_by('group__acronym')
+    )
+    # per Colin (datatracker #5010) - don't report not meeting rags
+    irtf_not_meeting = [item for item in irtf_not_meeting if item["group"].type_id != "rag"]
+    irtf = {"meeting_groups":irtf_meeting, "not_meeting_groups":irtf_not_meeting}
+
+    training, _ = organize_proceedings_sessions(
+        sessions.filter(group__acronym__in=['edu','iaoc'], type_id__in=['regular', 'other',])
+        .exclude(current_status='notmeet')
+    )
+    iab, _ = organize_proceedings_sessions(
+        sessions.filter(group__parent__acronym = 'iab')
+        .exclude(current_status='notmeet')
+    )
+    editorial, _ = organize_proceedings_sessions(
+        sessions.filter(group__acronym__in=['rsab','rswg'])
+        .exclude(current_status='notmeet')
+    )
+
+    ietf = sessions.filter(group__parent__type__slug = 'area').exclude(group__acronym__in=['edu','iepg','tools'])
+    ietf = list(ietf)
+    ietf.sort(key=lambda s: area_and_group_acronyms_from_session(s))
+    ietf_areas = []
+    for area, area_sessions in itertools.groupby(ietf, key=lambda s: s.group_parent_at_the_time()):
+        meeting_groups, not_meeting_groups = organize_proceedings_sessions(area_sessions)
+        ietf_areas.append((area, meeting_groups, not_meeting_groups))
+
+    with timezone.override(meeting.tz()):
+        rendered_content = render_to_string(
+            "meeting/proceedings.html", 
+            {
+                'meeting': meeting,
+                'plenaries': plenaries,
+                'training': training,
+                'irtf': irtf,
+                'iab': iab,
+                'editorial': editorial,
+                'ietf_areas': ietf_areas,
+                'meetinghost_logo': {
+                    'max_height': settings.MEETINGHOST_LOGO_MAX_DISPLAY_HEIGHT,
+                    'max_width': settings.MEETINGHOST_LOGO_MAX_DISPLAY_WIDTH,
+                }
+            },
+        )
+    cache.set(
+        cache_key,
+        rendered_content,
+        timeout=86400,  # one day, in seconds
+    )
+    return rendered_content
+
+
+def organize_proceedings_sessions(sessions):
+    # Collect sessions by Group, then bin by session name (including sessions with blank names).
+    # If all of a group's sessions are 'notmeet', the processed data goes in not_meeting_sessions.
+    # Otherwise, the data goes in meeting_sessions.
+    meeting_groups = []
+    not_meeting_groups = []
+    for group_acronym, group_sessions in itertools.groupby(sessions, key=lambda s: s.group.acronym):
+        by_name = {}
+        is_meeting = False
+        all_canceled = True
+        group = None
+        for s in sorted(
+                group_sessions,
+                key=lambda gs: (
+                        gs.official_timeslotassignment().timeslot.time
+                        if gs.official_timeslotassignment() else datetime.datetime(datetime.MAXYEAR, 1, 1)
+                ),
+        ):
+            group = s.group
+            if s.current_status != 'notmeet':
+                is_meeting = True
+            if s.current_status != 'canceled':
+                all_canceled = False
+            by_name.setdefault(s.name, [])
+            if s.current_status != 'notmeet' or s.presentations.exists():
+                by_name[s.name].append(s)  # for notmeet, only include sessions with materials
+        for sess_name, ss in by_name.items():
+            session = ss[0] if ss else None
+            def _format_materials(items):
+                """Format session/material for template
+
+                Input is a list of (session, materials) pairs. The materials value can be a single value or a list.
+                """
+                material_times = {}  # key is material, value is first timestamp it appeared
+                for s, mats in items:
+                    tsa = s.official_timeslotassignment()
+                    timestamp = tsa.timeslot.time if tsa else None
+                    if not isinstance(mats, list):
+                        mats = [mats]
+                    for mat in mats:
+                        if mat and mat not in material_times:
+                            material_times[mat] = timestamp
+                n_mats = len(material_times)
+                result = []
+                if n_mats == 1:
+                    result.append({'material': list(material_times)[0]})  # no 'time' when only a single material
+                elif n_mats > 1:
+                    for mat, timestamp in material_times.items():
+                        result.append({'material': mat, 'time': timestamp})
+                return result
+
+            entry = {
+                'group': group,
+                'name': sess_name,
+                'session': session,
+                'canceled': all_canceled,
+                'has_materials': s.presentations.exists(),
+                'agendas': _format_materials((s, s.agenda()) for s in ss),
+                'minutes': _format_materials((s, s.minutes()) for s in ss),
+                'bluesheets': _format_materials((s, s.bluesheets()) for s in ss),
+                'recordings': _format_materials((s, s.recordings()) for s in ss),
+                'meetecho_recordings': _format_materials((s, [s.session_recording_url()]) for s in ss),
+                'chatlogs': _format_materials((s, s.chatlogs()) for s in ss),
+                'slides': _format_materials((s, s.slides()) for s in ss),
+                'drafts': _format_materials((s, s.drafts()) for s in ss),
+                'last_update': session.last_update if hasattr(session, 'last_update') else None
+            }
+            if session and session.meeting.type_id == 'ietf' and not session.meeting.proceedings_final:
+                entry['attendances'] = _format_materials((s, s) for s in ss if Attended.objects.filter(session=s).exists())
+            if is_meeting:
+                meeting_groups.append(entry)
+            else:
+                not_meeting_groups.append(entry)
+    return meeting_groups, not_meeting_groups
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 2f2464028b..1226e30d60 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -75,7 +75,13 @@
 from ietf.meeting.helpers import send_interim_approval
 from ietf.meeting.helpers import send_interim_approval_request
 from ietf.meeting.helpers import send_interim_announcement_request, sessions_post_cancel
-from ietf.meeting.utils import finalize, sort_accept_tuple, condition_slide_order
+from ietf.meeting.utils import (
+    condition_slide_order,
+    finalize,
+    generate_proceedings_content,
+    organize_proceedings_sessions,
+    sort_accept_tuple,
+)
 from ietf.meeting.utils import add_event_info_to_session_qs
 from ietf.meeting.utils import session_time_for_sorting
 from ietf.meeting.utils import session_requested_by, SaveMaterialsError
@@ -4128,93 +4134,10 @@ def upcoming_json(request):
     response = HttpResponse(json.dumps(data, indent=2, sort_keys=False), content_type='application/json;charset=%s'%settings.DEFAULT_CHARSET)
     return response
 
-def organize_proceedings_sessions(sessions):
-    # Collect sessions by Group, then bin by session name (including sessions with blank names).
-    # If all of a group's sessions are 'notmeet', the processed data goes in not_meeting_sessions.
-    # Otherwise, the data goes in meeting_sessions.
-    meeting_groups = []
-    not_meeting_groups = []
-    for group_acronym, group_sessions in itertools.groupby(sessions, key=lambda s: s.group.acronym):
-        by_name = {}
-        is_meeting = False
-        all_canceled = True
-        group = None
-        for s in sorted(
-                group_sessions,
-                key=lambda gs: (
-                        gs.official_timeslotassignment().timeslot.time
-                        if gs.official_timeslotassignment() else datetime.datetime(datetime.MAXYEAR, 1, 1)
-                ),
-        ):
-            group = s.group
-            if s.current_status != 'notmeet':
-                is_meeting = True
-            if s.current_status != 'canceled':
-                all_canceled = False
-            by_name.setdefault(s.name, [])
-            if s.current_status != 'notmeet' or s.presentations.exists():
-                by_name[s.name].append(s)  # for notmeet, only include sessions with materials
-        for sess_name, ss in by_name.items():
-            session = ss[0] if ss else None
-            def _format_materials(items):
-                """Format session/material for template
-
-                Input is a list of (session, materials) pairs. The materials value can be a single value or a list.
-                """
-                material_times = {}  # key is material, value is first timestamp it appeared
-                for s, mats in items:
-                    tsa = s.official_timeslotassignment()
-                    timestamp = tsa.timeslot.time if tsa else None
-                    if not isinstance(mats, list):
-                        mats = [mats]
-                    for mat in mats:
-                        if mat and mat not in material_times:
-                            material_times[mat] = timestamp
-                n_mats = len(material_times)
-                result = []
-                if n_mats == 1:
-                    result.append({'material': list(material_times)[0]})  # no 'time' when only a single material
-                elif n_mats > 1:
-                    for mat, timestamp in material_times.items():
-                        result.append({'material': mat, 'time': timestamp})
-                return result
-
-            entry = {
-                'group': group,
-                'name': sess_name,
-                'session': session,
-                'canceled': all_canceled,
-                'has_materials': s.presentations.exists(),
-                'agendas': _format_materials((s, s.agenda()) for s in ss),
-                'minutes': _format_materials((s, s.minutes()) for s in ss),
-                'bluesheets': _format_materials((s, s.bluesheets()) for s in ss),
-                'recordings': _format_materials((s, s.recordings()) for s in ss),
-                'meetecho_recordings': _format_materials((s, [s.session_recording_url()]) for s in ss),
-                'chatlogs': _format_materials((s, s.chatlogs()) for s in ss),
-                'slides': _format_materials((s, s.slides()) for s in ss),
-                'drafts': _format_materials((s, s.drafts()) for s in ss),
-                'last_update': session.last_update if hasattr(session, 'last_update') else None
-            }
-            if session and session.meeting.type_id == 'ietf' and not session.meeting.proceedings_final:
-                entry['attendances'] = _format_materials((s, s) for s in ss if Attended.objects.filter(session=s).exists())
-            if is_meeting:
-                meeting_groups.append(entry)
-            else:
-                not_meeting_groups.append(entry)
-    return meeting_groups, not_meeting_groups
-
 
 def proceedings(request, num=None):
-
-    def area_and_group_acronyms_from_session(s):
-        area = s.group_parent_at_the_time()
-        if area == None:
-            area = s.group.parent
-        group = s.group_at_the_time()
-        return (area.acronym, group.acronym)
-
     meeting = get_meeting(num)
-
+    
     # Early proceedings were hosted on www.ietf.org rather than the datatracker
     if meeting.proceedings_format_version == 1:
         return HttpResponseRedirect(settings.PROCEEDINGS_V1_BASE_URL.format(meeting=meeting))
@@ -4225,72 +4148,12 @@ def area_and_group_acronyms_from_session(s):
             kwargs['num'] = num
         return redirect('ietf.meeting.views.materials', **kwargs)
 
-    begin_date = meeting.get_submission_start_date()
-    cut_off_date = meeting.get_submission_cut_off_date()
-    cor_cut_off_date = meeting.get_submission_correction_date()
-    today_utc = date_today(datetime.timezone.utc)
-
-    schedule = get_schedule(meeting, None)
-    sessions  = (
-        meeting.session_set.with_current_status()
-        .filter(Q(timeslotassignments__schedule__in=[schedule, schedule.base if schedule else None])
-                | Q(current_status='notmeet'))
-        .select_related()
-        .order_by('-current_status')
-    )
-
-    plenaries, _ = organize_proceedings_sessions(
-        sessions.filter(name__icontains='plenary')
-        .exclude(current_status='notmeet')
-    )
-    irtf_meeting, irtf_not_meeting = organize_proceedings_sessions(
-        sessions.filter(group__parent__acronym = 'irtf').order_by('group__acronym')
-    )
-    # per Colin (datatracker #5010) - don't report not meeting rags
-    irtf_not_meeting = [item for item in irtf_not_meeting if item["group"].type_id != "rag"]
-    irtf = {"meeting_groups":irtf_meeting, "not_meeting_groups":irtf_not_meeting}
-
-    training, _ = organize_proceedings_sessions(
-        sessions.filter(group__acronym__in=['edu','iaoc'], type_id__in=['regular', 'other',])
-        .exclude(current_status='notmeet')
-    )
-    iab, _ = organize_proceedings_sessions(
-        sessions.filter(group__parent__acronym = 'iab')
-        .exclude(current_status='notmeet')
-    )
-    editorial, _ = organize_proceedings_sessions(
-        sessions.filter(group__acronym__in=['rsab','rswg'])
-        .exclude(current_status='notmeet')
-    )
-
-    ietf = sessions.filter(group__parent__type__slug = 'area').exclude(group__acronym__in=['edu','iepg','tools'])
-    ietf = list(ietf)
-    ietf.sort(key=lambda s: area_and_group_acronyms_from_session(s))
-    ietf_areas = []
-    for area, area_sessions in itertools.groupby(ietf, key=lambda s: s.group_parent_at_the_time()):
-        meeting_groups, not_meeting_groups = organize_proceedings_sessions(area_sessions)
-        ietf_areas.append((area, meeting_groups, not_meeting_groups))
-
-    cache_version = Document.objects.filter(session__meeting__number=meeting.number).aggregate(Max('time'))["time__max"]
 
     with timezone.override(meeting.tz()):
-        return render(request, "meeting/proceedings.html", {
+        return render(request, "meeting/proceedings_wrapper.html", {
             'meeting': meeting,
-            'plenaries': plenaries,
-            'training': training,
-            'irtf': irtf,
-            'iab': iab,
-            'editorial': editorial,
-            'ietf_areas': ietf_areas,
-            'cut_off_date': cut_off_date,
-            'cor_cut_off_date': cor_cut_off_date,
-            'submission_started': today_utc > begin_date,
-            'cache_version': cache_version,
             'attendance': meeting.get_attendance(),
-            'meetinghost_logo': {
-                'max_height': settings.MEETINGHOST_LOGO_MAX_DISPLAY_HEIGHT,
-                'max_width': settings.MEETINGHOST_LOGO_MAX_DISPLAY_WIDTH,
-            }
+            'proceedings_content': generate_proceedings_content(meeting),
         })
 
 @role_required('Secretariat')
diff --git a/ietf/templates/meeting/proceedings.html b/ietf/templates/meeting/proceedings.html
index b5d4a6198a..0aa8197fe9 100644
--- a/ietf/templates/meeting/proceedings.html
+++ b/ietf/templates/meeting/proceedings.html
@@ -1,184 +1,160 @@
-{% extends "base.html" %}
-{# Copyright The IETF Trust 2015, All Rights Reserved #}
-{% load origin %}
-{% load ietf_filters static %}
-{% block pagehead %}
-    <link rel="stylesheet" href="{% static "ietf/css/list.css" %}">
-{% endblock %}
-{% block title %}
-    IETF {{ meeting.number }}
-    {% if not meeting.proceedings_final %}Draft{% endif %}
-    Proceedings
-{% endblock %}
-{% block content %}
-    {% origin %}
-    {% include 'meeting/proceedings/title.html' with meeting=meeting attendance=attendance only %}
-    {% if user|has_role:"Secretariat" and not meeting.proceedings_final %}
-        <a class="btn btn-warning finalize-button"
-           href="{% url 'ietf.meeting.views.finalize_proceedings' num=meeting.number %}">
-            Finalize proceedings
-        </a>
-    {% endif %}
-    {# cache for 15 minutes, as long as there's no proceedings activity. takes 4-8 seconds to generate. #}
-    {% load cache %}
-    {% cache 900 ietf_meeting_proceedings meeting.number cache_version %}
-    {% include 'meeting/proceedings/introduction.html' with meeting=meeting only %}
-    <!-- Plenaries -->
-    {% if plenaries %}
-        <h2 class="mt-5" id="plenaries">Plenaries</h2>
-        <table class="table table-sm table-striped tablesorter">
-            <thead>
-                <tr>
-                    <th scope="col" data-sort="group">Group</th>
-                    <th scope="col" data-sort="artifacts">Artifacts</th>
-                    <th scope="col" data-sort="recordings">Recordings</th>
-                    <th scope="col" data-sort="slides">Slides</th>
-                    <th scope="col" data-sort="drafts">Internet-Drafts</th>
-                </tr>
-            </thead>
-            <tbody>
-                {% for entry in plenaries %}
-                    {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
-                {% endfor %}
-            </tbody>
-        </table>
-    {% endif %}
-    <!-- Working groups -->
-    {% for area, meeting_groups, not_meeting_groups in ietf_areas %}
-        <h2 class="mt-5" id="{{ area.acronym }}">
-            {{ area.acronym|upper }} <small class="text-body-secondary">{{ area.name }}</small>
-        </h2>
-        {% if meeting_groups %}
-            <table class="table table-sm table-striped tablesorter">
-                <thead>
-                    <tr>
-                        <th scope="col" data-sort="group">Group</th>
-                        <th scope="col" data-sort="artifacts">Artifacts</th>
-                        <th scope="col" data-sort="recordings">Recordings</th>
-                        <th scope="col" data-sort="slides">Slides</th>
-                        <th scope="col" data-sort="drafts">Internet-Drafts</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    {% for entry in meeting_groups %}
-                        {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
-                    {% endfor %}
-                </tbody>
-            </table>
-        {% endif %}
-        {% if not_meeting_groups %}
-            <p>
-                {{ area.name }} groups not meeting:
-                {% for entry in not_meeting_groups %}
-                    {% if entry.name == "" %}{# do not show named sessions in this list #}
-                        <a href="{% url 'ietf.group.views.group_home' acronym=entry.group.acronym %}">
-                            {{ entry.group.acronym }}
-                        </a>{% if not forloop.last %},{% endif %}
-                    {% endif %}
-                {% endfor %}
-            </p>
-            <table class="table table-sm table-striped">
-                <thead>
-                    <tr>
-                        <th scope="col"></th>
-                        <th scope="col"></th>
-                        <th scope="col"></th>
-                        <th scope="col"></th>
-                        <th scope="col"></th>
-                    </tr>
-                </thead>
-                <tbody>
-                    {% for entry in not_meeting_groups %}{% if entry.has_materials %}
-                        {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
-                    {% endif %}{% endfor %}
-                </tbody>
-            </table>
-        {% endif %}
-    {% endfor %}
-    <!-- Training Sessions -->
-    {% if training %}
-        <h2 class="mt-5" id="training">Training</h2>
+{% include 'meeting/proceedings/introduction.html' with meeting=meeting only %}
+<!-- Plenaries -->
+{% if plenaries %}
+    <h2 class="mt-5" id="plenaries">Plenaries</h2>
+    <table class="table table-sm table-striped tablesorter">
+        <thead>
+        <tr>
+            <th scope="col" data-sort="group">Group</th>
+            <th scope="col" data-sort="artifacts">Artifacts</th>
+            <th scope="col" data-sort="recordings">Recordings</th>
+            <th scope="col" data-sort="slides">Slides</th>
+            <th scope="col" data-sort="drafts">Internet-Drafts</th>
+        </tr>
+        </thead>
+        <tbody>
+        {% for entry in plenaries %}
+            {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
+        {% endfor %}
+        </tbody>
+    </table>
+{% endif %}
+<!-- Working groups -->
+{% for area, meeting_groups, not_meeting_groups in ietf_areas %}
+    <h2 class="mt-5" id="{{ area.acronym }}">
+        {{ area.acronym|upper }} <small class="text-body-secondary">{{ area.name }}</small>
+    </h2>
+    {% if meeting_groups %}
         <table class="table table-sm table-striped tablesorter">
             <thead>
-                <tr>
-                    <th scope="col" data-sort="group">Group</th>
-                    <th scope="col" data-sort="artifacts">Artifacts</th>
-                    <th scope="col" data-sort="recordings">Recordings</th>
-                    <th scope="col" data-sort="slides">Slides</th>
-                    <th scope="col" data-sort="drafts">Internet-Drafts</th>
-                </tr>
+            <tr>
+                <th scope="col" data-sort="group">Group</th>
+                <th scope="col" data-sort="artifacts">Artifacts</th>
+                <th scope="col" data-sort="recordings">Recordings</th>
+                <th scope="col" data-sort="slides">Slides</th>
+                <th scope="col" data-sort="drafts">Internet-Drafts</th>
+            </tr>
             </thead>
             <tbody>
-                {% for entry in training %}
-                        {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=False only %}
-                {% endfor %}
+            {% for entry in meeting_groups %}
+                {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
+            {% endfor %}
             </tbody>
         </table>
     {% endif %}
-    <!-- IAB Sessions -->
-    {% if iab %}
-        <h2 class="mt-5" id="iab">
-            IAB <small class="text-body-secondary">Internet Architecture Board</small>
-        </h2>
-        <table class="table table-sm table-striped tablesorter">
+    {% if not_meeting_groups %}
+        <p>
+            {{ area.name }} groups not meeting:
+            {% for entry in not_meeting_groups %}
+                {% if entry.name == "" %}{# do not show named sessions in this list #}
+                    <a href="{% url 'ietf.group.views.group_home' acronym=entry.group.acronym %}">
+                        {{ entry.group.acronym }}
+                    </a>{% if not forloop.last %},{% endif %}
+                {% endif %}
+            {% endfor %}
+        </p>
+        <table class="table table-sm table-striped">
             <thead>
-                <tr>
-                    <th scope="col" data-sort="group">
-                        Group
-                    </th>
-                    <th scope="col" data-sort="artifacts">
-                        Artifacts
-                    </th>
-                    <th scope="col" data-sort="recordings">
-                        Recordings
-                    </th>
-                    <th scope="col" data-sort="slides">
-                        Slides
-                    </th>
-                    <th scope="col" data-sort="drafts">
-                        Internet-Drafts
-                    </th>
-                </tr>
+            <tr>
+                <th scope="col"></th>
+                <th scope="col"></th>
+                <th scope="col"></th>
+                <th scope="col"></th>
+                <th scope="col"></th>
+            </tr>
             </thead>
             <tbody>
-                {% for entry in iab %}
-                    {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
-                {% endfor %}
+            {% for entry in not_meeting_groups %}{% if entry.has_materials %}
+                {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
+            {% endif %}{% endfor %}
             </tbody>
         </table>
     {% endif %}
-    <!-- IRTF Sessions -->
-    {% if irtf.meeting_groups  %}
-        <h2 class="mt-5" id="irtf">
-            IRTF <small class="text-body-secondary">Internet Research Task Force</small>
-        </h2>
-        <table class="table table-sm table-striped tablesorter">
-            <thead>
-                <tr>
-                    <th scope="col" data-sort="group">
-                        Group
-                    </th>
-                    <th scope="col" data-sort="artifacts">
-                        Artifacts
-                    </th>
-                    <th scope="col" data-sort="recordings">
-                        Recordings
-                    </th>
-                    <th scope="col" data-sort="slides">
-                        Slides
-                    </th>
-                    <th scope="col" data-sort="drafts">
-                        Internet-Drafts
-                    </th>
-                </tr>
-            </thead>
-            <tbody>
-                {% for entry in irtf.meeting_groups %}
-                    {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
-                {% endfor %}
-            </tbody>
-        </table>
-        {% if irtf.not_meeting_groups %}
+{% endfor %}
+<!-- Training Sessions -->
+{% if training %}
+    <h2 class="mt-5" id="training">Training</h2>
+    <table class="table table-sm table-striped tablesorter">
+        <thead>
+        <tr>
+            <th scope="col" data-sort="group">Group</th>
+            <th scope="col" data-sort="artifacts">Artifacts</th>
+            <th scope="col" data-sort="recordings">Recordings</th>
+            <th scope="col" data-sort="slides">Slides</th>
+            <th scope="col" data-sort="drafts">Internet-Drafts</th>
+        </tr>
+        </thead>
+        <tbody>
+        {% for entry in training %}
+            {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=False only %}
+        {% endfor %}
+        </tbody>
+    </table>
+{% endif %}
+<!-- IAB Sessions -->
+{% if iab %}
+    <h2 class="mt-5" id="iab">
+        IAB <small class="text-body-secondary">Internet Architecture Board</small>
+    </h2>
+    <table class="table table-sm table-striped tablesorter">
+        <thead>
+        <tr>
+            <th scope="col" data-sort="group">
+                Group
+            </th>
+            <th scope="col" data-sort="artifacts">
+                Artifacts
+            </th>
+            <th scope="col" data-sort="recordings">
+                Recordings
+            </th>
+            <th scope="col" data-sort="slides">
+                Slides
+            </th>
+            <th scope="col" data-sort="drafts">
+                Internet-Drafts
+            </th>
+        </tr>
+        </thead>
+        <tbody>
+        {% for entry in iab %}
+            {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
+        {% endfor %}
+        </tbody>
+    </table>
+{% endif %}
+<!-- IRTF Sessions -->
+{% if irtf.meeting_groups %}
+    <h2 class="mt-5" id="irtf">
+        IRTF <small class="text-body-secondary">Internet Research Task Force</small>
+    </h2>
+    <table class="table table-sm table-striped tablesorter">
+        <thead>
+        <tr>
+            <th scope="col" data-sort="group">
+                Group
+            </th>
+            <th scope="col" data-sort="artifacts">
+                Artifacts
+            </th>
+            <th scope="col" data-sort="recordings">
+                Recordings
+            </th>
+            <th scope="col" data-sort="slides">
+                Slides
+            </th>
+            <th scope="col" data-sort="drafts">
+                Internet-Drafts
+            </th>
+        </tr>
+        </thead>
+        <tbody>
+        {% for entry in irtf.meeting_groups %}
+            {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
+        {% endfor %}
+        </tbody>
+    </table>
+    {% if irtf.not_meeting_groups %}
         <p>
             IRTF groups not meeting:
             {% for entry in irtf.not_meeting_groups %}
@@ -191,18 +167,18 @@ <h2 class="mt-5" id="irtf">
         </p>
         <table class="table table-sm table-striped">
             <thead>
-                <tr>
-                    <th scope="col"></th>
-                    <th scope="col"></th>
-                    <th scope="col"></th>
-                    <th scope="col"></th>
-                    <th scope="col"></th>
-                </tr>
+            <tr>
+                <th scope="col"></th>
+                <th scope="col"></th>
+                <th scope="col"></th>
+                <th scope="col"></th>
+                <th scope="col"></th>
+            </tr>
             </thead>
             <tbody>
-                {% for entry in irtf.not_meeting %}{% if entry.has_materials %}
-                    {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
-                {% endif %}{% endfor %}
+            {% for entry in irtf.not_meeting %}{% if entry.has_materials %}
+                {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
+            {% endif %}{% endfor %}
             </tbody>
         </table>
     {% endif %}
@@ -211,35 +187,29 @@ <h2 class="mt-5" id="irtf">
         <h2 class="mt-5" id="editorial">Editorial Stream</h2>
         <table class="table table-sm table-striped tablesorter">
             <thead>
-                <tr>
-                    <th scope="col" data-sort="group">
-                        Group
-                    </th>
-                    <th scope="col" data-sort="artifacts">
-                        Artifacts
-                    </th>
-                    <th scope="col" data-sort="recordings">
-                        Recordings
-                    </th>
-                    <th scope="col" data-sort="slides">
-                        Slides
-                    </th>
-                    <th scope="col" data-sort="drafts">
-                        Internet-Drafts
-                    </th>
-                </tr>
+            <tr>
+                <th scope="col" data-sort="group">
+                    Group
+                </th>
+                <th scope="col" data-sort="artifacts">
+                    Artifacts
+                </th>
+                <th scope="col" data-sort="recordings">
+                    Recordings
+                </th>
+                <th scope="col" data-sort="slides">
+                    Slides
+                </th>
+                <th scope="col" data-sort="drafts">
+                    Internet-Drafts
+                </th>
+            </tr>
             </thead>
             <tbody>
-                {% for entry in editorial %}
-                    {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
-                {% endfor %}
+            {% for entry in editorial %}
+                {% include "meeting/group_proceedings.html" with entry=entry meeting=meeting show_agenda=True only %}
+            {% endfor %}
             </tbody>
         </table>
     {% endif %}
-    {% endif %}
-{% endcache %}
-{% endblock %}
-{% block js %}
-    <script src="{% static "ietf/js/list.js" %}">
-    </script>
-{% endblock %}
+{% endif %}
diff --git a/ietf/templates/meeting/proceedings_wrapper.html b/ietf/templates/meeting/proceedings_wrapper.html
new file mode 100644
index 0000000000..a20291a693
--- /dev/null
+++ b/ietf/templates/meeting/proceedings_wrapper.html
@@ -0,0 +1,27 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{% load origin %}
+{% load ietf_filters static %}
+{% block pagehead %}
+    <link rel="stylesheet" href="{% static "ietf/css/list.css" %}">
+{% endblock %}
+{% block title %}
+    IETF {{ meeting.number }}
+    {% if not meeting.proceedings_final %}Draft{% endif %}
+    Proceedings
+{% endblock %}
+{% block content %}
+    {% origin %}
+    {% include 'meeting/proceedings/title.html' with meeting=meeting attendance=attendance only %}
+    {% if user|has_role:"Secretariat" and not meeting.proceedings_final %}
+        <a class="btn btn-warning finalize-button"
+           href="{% url 'ietf.meeting.views.finalize_proceedings' num=meeting.number %}">
+            Finalize proceedings
+        </a>
+    {% endif %}
+    {{ proceedings_content }}
+{% endblock %}
+{% block js %}
+    <script src="{% static "ietf/js/list.js" %}">
+    </script>
+{% endblock %}

From ed19b54d8a39fca72a292181bdb2c12e9b63a5b6 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 6 Feb 2025 14:35:07 -0400
Subject: [PATCH 254/656] fix: deal with "GMT" time zone in iCalendar files
 (#8506)

* fix: omit vtimezone when None

* fix: fix ics time format for tz=GMT
---
 ietf/doc/templatetags/ietf_filters.py |  5 ++++-
 ietf/templates/meeting/agenda.ics     | 10 +++++-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py
index e1a80a26b2..86507eeaaa 100644
--- a/ietf/doc/templatetags/ietf_filters.py
+++ b/ietf/doc/templatetags/ietf_filters.py
@@ -533,11 +533,14 @@ def ics_date_time(dt, tzname):
     >>> ics_date_time(datetime.datetime(2022,1,2,3,4,5), 'UTC')
     ':20220102T030405Z'
 
+    >>> ics_date_time(datetime.datetime(2022,1,2,3,4,5), 'GmT')
+    ':20220102T030405Z'
+
     >>> ics_date_time(datetime.datetime(2022,1,2,3,4,5), 'America/Los_Angeles')
     ';TZID=America/Los_Angeles:20220102T030405'
     """
     timestamp = dt.strftime('%Y%m%dT%H%M%S')
-    if tzname.lower() == 'utc':
+    if tzname.lower() in ('gmt', 'utc'):
         return f':{timestamp}Z'
     else:
         return f';TZID={ics_esc(tzname)}:{timestamp}'
diff --git a/ietf/templates/meeting/agenda.ics b/ietf/templates/meeting/agenda.ics
index 72b7ab806c..8bc8222bbe 100644
--- a/ietf/templates/meeting/agenda.ics
+++ b/ietf/templates/meeting/agenda.ics
@@ -1,15 +1,15 @@
-{% load humanize tz %}{% autoescape off %}{% timezone schedule.meeting.tz %}{% load ietf_filters textfilters %}{% load cache %}{% cache 1800 ietf_meeting_agenda_ics schedule.meeting.number request.path request.GET %}BEGIN:VCALENDAR
+{% load humanize tz %}{% autoescape off %}{% timezone schedule.meeting.tz %}{% with tzname=schedule.meeting.time_zone|lower %}{% load ietf_filters textfilters %}{% load cache %}{% cache 1800 ietf_meeting_agenda_ics schedule.meeting.number request.path request.GET %}BEGIN:VCALENDAR
 VERSION:2.0
 METHOD:PUBLISH
 PRODID:-//IETF//datatracker.ietf.org ical agenda//EN
-{{schedule.meeting.vtimezone}}{% for item in assignments %}BEGIN:VEVENT
+{% if tzname != "utc" and tzname != "gmt" %}{% firstof schedule.meeting.vtimezone "" %}{% endif %}{% for item in assignments %}BEGIN:VEVENT
 UID:ietf-{{schedule.meeting.number}}-{{item.timeslot.pk}}-{{item.session.group.acronym}}
 SUMMARY:{% if item.session.name %}{{item.session.name|ics_esc}}{% else %}{{item.session.group_at_the_time.acronym|lower}} - {{item.session.group_at_the_time.name}}{%endif%}{% if item.session.agenda_note %} ({{item.session.agenda_note}}){% endif %}
 {% if item.timeslot.show_location %}LOCATION:{{item.timeslot.get_location}}
 {% endif %}STATUS:{{item.session.ical_status}}
 CLASS:PUBLIC
-DTSTART{% ics_date_time item.timeslot.local_start_time schedule.meeting.time_zone %}
-DTEND{% ics_date_time item.timeslot.local_end_time schedule.meeting.time_zone %}
+DTSTART{% ics_date_time item.timeslot.local_start_time tzname %}
+DTEND{% ics_date_time item.timeslot.local_end_time tzname %}
 DTSTAMP{% ics_date_time item.timeslot.modified|utc 'utc' %}{% if item.session.agenda %}
 URL:{{item.session.agenda.get_versionless_href}}{% endif %}
 DESCRIPTION:{{item.timeslot.name|ics_esc}}\n{% if item.session.agenda_note %}
@@ -29,4 +29,4 @@ DESCRIPTION:{{item.timeslot.name|ics_esc}}\n{% if item.session.agenda_note %}
  \n{# link agenda for ietf meetings #}
  See in schedule: {% absurl 'agenda' num=schedule.meeting.number %}#row-{{ item.slug }}\n{% endif %}
 END:VEVENT
-{% endfor %}END:VCALENDAR{% endcache %}{% endtimezone %}{% endautoescape %}
+{% endfor %}END:VCALENDAR{% endcache %}{% endwith %}{% endtimezone %}{% endautoescape %}

From d1ec7378da3ecb66d3e492219db75fda69681df4 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 14 Feb 2025 18:14:03 -0500
Subject: [PATCH 255/656] fix: use groupAcronym to determine hackathon icon in
 agenda (#8540)

* fix: use groupAcronym to determine hackathon icon in agenda

* test: fix agenda tests related to hackathon
---
 client/agenda/AgendaScheduleList.vue    | 4 ++--
 playwright/helpers/meeting.js           | 2 +-
 playwright/tests/meeting/agenda.spec.js | 6 +++---
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index e7c14797ee..905677b4da 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -307,7 +307,7 @@ const meetingEvents = computed(() => {
         })
       }
       // -> Point to Wiki for Hackathon sessions, HedgeDocs otherwise
-      if (item.name.toLowerCase().includes('hackathon')) {
+      if (item.groupAcronym === 'hackathon') {
         links.push({
           id: `lnk-${item.id}-wiki`,
           label: 'Wiki',
@@ -461,7 +461,7 @@ const meetingEvents = computed(() => {
       case 'other':
         if (item.name.toLowerCase().indexOf('office hours') >= 0) {
           icon = 'bi-building'
-        } else if (item.name.toLowerCase().indexOf('hackathon') >= 0) {
+        } else if (item.groupAcronym === 'hackathon') {
           icon = 'bi-command bi-pink'
         }
         break
diff --git a/playwright/helpers/meeting.js b/playwright/helpers/meeting.js
index 52bc331fd6..9722ffc68b 100644
--- a/playwright/helpers/meeting.js
+++ b/playwright/helpers/meeting.js
@@ -395,7 +395,7 @@ module.exports = {
         name: 'Hackathon Kickoff',
         startDateTime: day1.set({ hour: 10, minute: 30 }),
         duration: '30m',
-        ...findAreaGroup('hackathon-kickoff', categories[2]),
+        ...findAreaGroup('hackathon', categories[2]),
         showAgenda: true,
         hasAgenda: true,
         hasRecordings: true,
diff --git a/playwright/tests/meeting/agenda.spec.js b/playwright/tests/meeting/agenda.spec.js
index d31dbd5c2c..e6c6e17031 100644
--- a/playwright/tests/meeting/agenda.spec.js
+++ b/playwright/tests/meeting/agenda.spec.js
@@ -238,7 +238,7 @@ test.describe('past - desktop', () => {
       // Name column
       // -----------
       // Event icon
-      if (['break', 'plenary'].includes(event.type) || (event.type === 'other' && ['office hours', 'hackathon'].some(s => event.name.toLowerCase().indexOf(s) >= 0))) {
+      if (['break', 'plenary'].includes(event.type) || (event.type === 'other' && event.name.toLowerCase().indexOf('office hours') >= 0)) {
         await expect(row.locator('.agenda-table-cell-name > i.bi')).toBeVisible()
       }
       // Name link
@@ -286,7 +286,7 @@ test.describe('past - desktop', () => {
               // No meeting materials yet warning badge
               await expect(eventButtons.locator('.no-meeting-materials')).toBeVisible()
             }
-            if (event.name.toLowerCase().includes('hackathon')) {
+            if (event.groupAcronym === 'hackathon') {
               // Hackathon Wiki button
               const hackathonWikiLink = `https://wiki.ietf.org/meeting/${meetingData.meeting.number}/hackathon`
               await expect(eventButtons.locator(`#btn-lnk-${event.id}-wiki`)).toHaveAttribute('href', hackathonWikiLink)
@@ -1169,7 +1169,7 @@ test.describe('future - desktop', () => {
             // No meeting materials yet warning badge
             await expect(eventButtons.locator('.no-meeting-materials')).toBeVisible()
           }
-          if (event.name.toLowerCase().includes('hackathon')) {
+          if (event.groupAcronym === 'hackathon') {
             // Hackathon Wiki button
             const hackathonWikiLink = `https://wiki.ietf.org/meeting/${meetingData.meeting.number}/hackathon`
             await expect(eventButtons.locator(`#btn-lnk-${event.id}-wiki`)).toHaveAttribute('href', hackathonWikiLink)

From 853de88546b25d1a59477e31580014351c9f0913 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Sat, 15 Feb 2025 12:30:24 +1300
Subject: [PATCH 256/656] feat: CSS other-session-selected. Fixes #8525 (#8529)

---
 ietf/static/css/ietf.scss | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss
index f1f2b94a19..aee93dcfe1 100644
--- a/ietf/static/css/ietf.scss
+++ b/ietf/static/css/ietf.scss
@@ -695,8 +695,8 @@ td.position-empty {
 }
 
 .edit-meeting-schedule .session.other-session-selected {
-    outline: var(--bs-info) solid 0.2em;
-    /* width matches margin on .session */
+    outline: 0.3em solid var(--bs-info);
+    box-shadow: 0 0 1em var(--bs-info);
     z-index: 2;
     /* render above timeslot outlines */
 }

From 04952d58fa05f8b47884f621d9c1c510df0eb9a6 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 18 Feb 2025 12:33:33 -0400
Subject: [PATCH 257/656] fix: remove all staging files for submission (#8535)

---
 ietf/submit/utils.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index 49f28c4196..fe52f1c774 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -671,20 +671,16 @@ def move_files_to_repository(submission):
             raise ValueError("Intended to move '%s' to '%s', but found source and destination missing.")
 
 
-def remove_staging_files(name, rev, exts=None):
-    """Remove staging files corresponding to a submission
-    
-    exts is a list of extensions to be removed. If None, defaults to settings.IDSUBMIT_FILE_TYPES.
-    """
-    if exts is None:
-        exts = [f'.{ext}' for ext in settings.IDSUBMIT_FILE_TYPES]
+def remove_staging_files(name, rev):
+    """Remove staging files corresponding to a submission"""
     basename = pathlib.Path(settings.IDSUBMIT_STAGING_PATH) / f'{name}-{rev}' 
+    exts = [f'.{ext}' for ext in settings.IDSUBMIT_FILE_TYPES]
     for ext in exts:
         basename.with_suffix(ext).unlink(missing_ok=True)
 
 
 def remove_submission_files(submission):
-    remove_staging_files(submission.name, submission.rev, submission.file_types.split(','))
+    remove_staging_files(submission.name, submission.rev)
 
 
 def approvable_submissions_for_user(user):

From e71272fd2f2047d092fca76ad56b1ebe7899a27c Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 19 Feb 2025 15:09:17 -0600
Subject: [PATCH 258/656] chore: mark a function unreachable (#8553)

---
 ietf/submit/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index fe52f1c774..f19f2384c8 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -646,6 +646,7 @@ def cancel_submission(submission):
 
 
 def rename_submission_files(submission, prev_rev, new_rev):
+    log.unreachable("2025-2-19")
     for ext in settings.IDSUBMIT_FILE_TYPES:
         staging_path = Path(settings.IDSUBMIT_STAGING_PATH) 
         source = staging_path / f"{submission.name}-{prev_rev}.{ext}"

From 997239a2eaf70d39a95c1bb465f4106c2ab1d544 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 19 Feb 2025 17:41:10 -0600
Subject: [PATCH 259/656] feat: write objects to blob storage (#8557)

* feat: basic blobstore infrastructure for dev

* refactor: (broken) attempt to put minio console behind nginx

* feat: initialize blobstore with boto3

* fix: abandon attempt to proxy minio. Use docker compose instead.

* feat: beginning of blob writes

* feat: storage utilities

* feat: test buckets

* chore: black

* chore: remove unused import

* chore: avoid f string when not needed

* fix: inform all settings files about blobstores

* fix: declare types for some settings

* ci: point to new target base

* ci: adjust test workflow

* fix: give the tests debug environment a blobstore

* fix: "better" name declarations

* ci: use devblobstore container

* chore: identify places to write to blobstorage

* chore: remove unreachable code

* feat: store materials

* feat: store statements

* feat: store status changes

* feat: store liaison attachments

* feat: store agendas provided with Interim session requests

* chore: capture TODOs

* feat: store polls and chatlogs

* chore: remove unneeded TODO

* feat: store drafts on submit and post

* fix: handle storage during doc expiration and resurrection

* fix: mirror an unlink

* chore: add/refine TODOs

* feat: store slide submissions

* fix: structure slide test correctly

* fix: correct sense of existence check

* feat: store some indexes

* feat: BlobShadowFileSystemStorage

* feat: shadow floorplans / host logos to the blob

* chore: remove unused import

* feat: strip path from blob shadow names

* feat: shadow photos / thumbs

* refactor: combine photo and photothumb blob kinds

The photos / thumbs were already dropped in the same
directory, so let's not add a distinction at this point.

* style: whitespace

* refactor: use kwargs consistently

* chore: migrations

* refactor: better deconstruct(); rebuild migrations

* fix: use new class in mack patch

* chore: add TODO

* feat: store group index documents

* chore: identify more TODO

* feat: store reviews

* fix: repair merge

* chore: remove unnecessary TODO

* feat: StoredObject metadata

* fix: deburr some debugging code

* fix: only set the deleted timestamp once

* chore: correct typo

* fix: get_or_create vs get and test

* fix: avoid the questionable is_seekable helper

* chore: capture future design consideration

* chore: blob store cfg for k8s

* chore: black

* chore: copyright

* ci: bucket name prefix option + run Black

Adds/uses DATATRACKER_BLOB_STORE_BUCKET_PREFIX option. Other changes
are just Black styling.

* ci: fix typo in bucket name expression

* chore: parameters in app-configure-blobstore

Allows use with other blob stores.

* ci: remove verify=False option

* fix: don't return value from __init__

* feat: option to log timing of S3Storage calls

* chore: units

* fix: deleted->null when storing a file

* style: Black

* feat: log as JSON; refactor to share code; handle exceptions

* ci: add ietf_log_blob_timing option for k8s

* test: --no-manage-blobstore option for running tests

* test: use blob store settings from env, if set

* test: actually set a couple more storage opts

* feat: offswitch (#8541)

* feat: offswitch

* fix: apply ENABLE_BLOBSTORAGE to BlobShadowFileSystemStorage behavior

* chore: log timing of blob reads

* chore: import Config from botocore.config

* chore(deps): import boto3-stubs / botocore

botocore is implicitly imported, but make it explicit
since we refer to it directly

* chore: drop type annotation that mypy loudly ignores

* refactor: add storage methods via mixin

Shares code between Document and DocHistory without
putting it in the base DocumentInfo class, which
lacks the name field. Also makes mypy happy.

* feat: add timeout / retry limit to boto client

* ci: let k8s config the timeouts via env

* chore: repair merge resolution typo

* chore: tweak settings imports

* chore: simplify k8s/settings_local.py imports

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 .devcontainer/docker-compose.extend.yml       |   4 +
 .github/workflows/tests.yml                   |   2 +
 README.md                                     |  17 ++
 dev/deploy-to-container/settings_local.py     |  23 ++-
 dev/diff/settings_local.py                    |  23 ++-
 dev/tests/docker-compose.debug.yml            |   3 +
 dev/tests/settings_local.py                   |  23 ++-
 docker-compose.yml                            |  10 +
 docker/app.Dockerfile                         |   4 +-
 docker/configs/settings_local.py              |  27 ++-
 docker/docker-compose.extend.yml              |   4 +
 docker/scripts/app-configure-blobstore.py     |  28 +++
 docker/scripts/app-init.sh                    |   5 +
 ietf/api/tests.py                             |   5 +
 ietf/doc/admin.py                             |   8 +-
 ietf/doc/expire.py                            |  14 ++
 ...ject_storedobject_unique_name_per_store.py |  66 ++++++
 ietf/doc/models.py                            |  85 +++++++-
 ietf/doc/resources.py                         |  25 ++-
 ietf/doc/storage_backends.py                  | 192 ++++++++++++++++++
 ietf/doc/storage_utils.py                     | 103 ++++++++++
 ietf/doc/tasks.py                             |   4 +-
 ietf/doc/tests_bofreq.py                      |   3 +
 ietf/doc/tests_charter.py                     |   6 +
 ietf/doc/tests_conflict_review.py             |   2 +
 ietf/doc/tests_draft.py                       |  13 ++
 ietf/doc/tests_material.py                    |   6 +
 ietf/doc/tests_review.py                      |   5 +
 ietf/doc/tests_statement.py                   |  17 ++
 ietf/doc/tests_status_change.py               |  18 +-
 ietf/doc/utils.py                             |   2 +-
 ietf/doc/views_bofreq.py                      |   2 +
 ietf/doc/views_charter.py                     |   8 +-
 ietf/doc/views_conflict_review.py             |   6 +-
 ietf/doc/views_draft.py                       |   6 +
 ietf/doc/views_material.py                    |   2 +
 ietf/doc/views_review.py                      |   1 +
 ietf/doc/views_statement.py                   |  13 +-
 ietf/doc/views_status_change.py               |   6 +-
 ietf/group/tasks.py                           |  11 +
 ietf/group/tests_info.py                      |  35 ++--
 ietf/idindex/tasks.py                         |   4 +
 ietf/idindex/tests.py                         |   5 +
 ietf/liaisons/forms.py                        |   2 +
 ietf/liaisons/tests.py                        |  38 +++-
 ietf/meeting/factories.py                     |   5 +
 ietf/meeting/forms.py                         |   1 +
 ietf/meeting/helpers.py                       |   5 +
 ..._floorplan_image_alter_meetinghost_logo.py |  56 +++++
 ietf/meeting/models.py                        |  15 +-
 ietf/meeting/tests_views.py                   | 126 ++++++++++--
 ietf/meeting/utils.py                         |  17 +-
 ietf/meeting/views.py                         |  17 +-
 ietf/nomcom/models.py                         |   1 +
 ...r_person_photo_alter_person_photo_thumb.py |  38 ++++
 ietf/person/models.py                         |  16 +-
 ietf/settings.py                              |  38 ++++
 ietf/settings_test.py                         |  30 ++-
 ietf/submit/tests.py                          |  94 ++++++++-
 ietf/submit/utils.py                          |  20 +-
 ietf/utils/storage.py                         |  56 ++++-
 ietf/utils/test_runner.py                     |  70 ++++++-
 k8s/settings_local.py                         | 103 ++++++++--
 requirements.txt                              |   6 +-
 64 files changed, 1484 insertions(+), 116 deletions(-)
 create mode 100755 docker/scripts/app-configure-blobstore.py
 create mode 100644 ietf/doc/migrations/0025_storedobject_storedobject_unique_name_per_store.py
 create mode 100644 ietf/doc/storage_backends.py
 create mode 100644 ietf/doc/storage_utils.py
 create mode 100644 ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py
 create mode 100644 ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py

diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml
index fa9a412cf2..286eefb29c 100644
--- a/.devcontainer/docker-compose.extend.yml
+++ b/.devcontainer/docker-compose.extend.yml
@@ -14,6 +14,10 @@ services:
             # - datatracker-vscode-ext:/root/.vscode-server/extensions
         # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
         network_mode: service:db
+    blobstore:
+        ports:
+            - '9000'
+            - '9001'
 
 volumes:
     datatracker-vscode-ext:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 5457415f59..f10c1db9a3 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -28,6 +28,8 @@ jobs:
     services:
       db:
         image: ghcr.io/ietf-tools/datatracker-db:latest
+      blobstore:
+        image: ghcr.io/ietf-tools/datatracker-devblobstore:latest
     
     steps:
     - uses: actions/checkout@v4
diff --git a/README.md b/README.md
index ee9865ba21..0ece0eb03b 100644
--- a/README.md
+++ b/README.md
@@ -106,6 +106,23 @@ Nightly database dumps of the datatracker are available as Docker images: `ghcr.
 
 > Note that to update the database in your dev environment to the latest version, you should run the `docker/cleandb` script.
 
+### Blob storage for dev/test
+
+The dev and test environments use [minio](https://github.com/minio/minio) to provide local blob storage. See the settings files for how the app container communicates with the blobstore container. If you need to work with minio directly from outside the containers (to interact with its api or console), use `docker compose` from the top level directory of your clone to expose it at an ephemeral port.
+
+```
+$ docker compose port blobstore 9001
+0.0.0.0:<some ephemeral port>
+
+$ curl -I http://localhost:<some ephemeral port>
+HTTP/1.1 200 OK
+...
+```
+
+
+The minio container exposes the minio api at port 9000 and the minio console at port 9001
+
+
 ### Frontend Development
 
 #### Intro
diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py
index 0a991ae9fe..e878206bd5 100644
--- a/dev/deploy-to-container/settings_local.py
+++ b/dev/deploy-to-container/settings_local.py
@@ -1,7 +1,9 @@
 # Copyright The IETF Trust 2007-2019, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-from ietf.settings import *                                          # pyflakes:ignore
+from ietf.settings import *   # pyflakes:ignore
+from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
+import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
@@ -79,3 +81,22 @@
 
 # OIDC configuration
 SITE_URL = 'https://__HOSTNAME__'
+
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=botocore.config.Config(
+                signature_version="s3v4",
+                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
+                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
+                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
+            ),
+            verify=False,
+            bucket_name=f"test-{storagename}",
+        ),
+    }
diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py
index 95d1e481c9..9e0806a8a6 100644
--- a/dev/diff/settings_local.py
+++ b/dev/diff/settings_local.py
@@ -1,7 +1,9 @@
 # Copyright The IETF Trust 2007-2019, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-from ietf.settings import *                                          # pyflakes:ignore
+from ietf.settings import *  # pyflakes:ignore
+from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
+import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
@@ -66,3 +68,22 @@
 SLIDE_STAGING_PATH = 'test/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
+
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=botocore.config.Config(
+                signature_version="s3v4",
+                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
+                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
+                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
+            ),
+            verify=False,
+            bucket_name=f"test-{storagename}",
+        ),
+    }
diff --git a/dev/tests/docker-compose.debug.yml b/dev/tests/docker-compose.debug.yml
index 8d939e0ea2..8117b92375 100644
--- a/dev/tests/docker-compose.debug.yml
+++ b/dev/tests/docker-compose.debug.yml
@@ -28,5 +28,8 @@ services:
         volumes:
             - postgresdb-data:/var/lib/postgresql/data
 
+    blobstore:
+      image: ghcr.io/ietf-tools/datatracker-devblobstore:latest
+
 volumes:
     postgresdb-data:
diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py
index 7b10bee06a..f2166053a7 100644
--- a/dev/tests/settings_local.py
+++ b/dev/tests/settings_local.py
@@ -1,7 +1,9 @@
 # Copyright The IETF Trust 2007-2019, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-from ietf.settings import *                                          # pyflakes:ignore
+from ietf.settings import *  # pyflakes:ignore
+from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
+import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
@@ -65,3 +67,22 @@
 SLIDE_STAGING_PATH = 'test/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
+
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=botocore.config.Config(
+                signature_version="s3v4",
+                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
+                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
+                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
+            ),
+            verify=False,
+            bucket_name=f"test-{storagename}",
+        ),
+    }
diff --git a/docker-compose.yml b/docker-compose.yml
index 9988b10c2e..30ce8ba4d2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -15,6 +15,7 @@ services:
         depends_on:
             - db
             - mq
+            - blobstore
 
         ipc: host
 
@@ -83,6 +84,14 @@ services:
             - .:/workspace
             - app-assets:/assets
 
+    blobstore:
+        image: ghcr.io/ietf-tools/datatracker-devblobstore:latest
+        restart: unless-stopped
+        volumes:
+            - "minio-data:/data"
+
+
+
 #  Celery Beat is a periodic task runner. It is not normally needed for development,
 #  but can be enabled by uncommenting the following.
 #
@@ -106,3 +115,4 @@ services:
 volumes:
     postgresdb-data:
     app-assets:
+    minio-data:
diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile
index b7dd44b6f1..fee3833733 100644
--- a/docker/app.Dockerfile
+++ b/docker/app.Dockerfile
@@ -43,8 +43,8 @@ RUN rm -rf /tmp/library-scripts
 # Copy the startup file
 COPY docker/scripts/app-init.sh /docker-init.sh
 COPY docker/scripts/app-start.sh /docker-start.sh
-RUN sed -i 's/\r$//' /docker-init.sh && chmod +x /docker-init.sh
-RUN sed -i 's/\r$//' /docker-start.sh && chmod +x /docker-start.sh
+RUN sed -i 's/\r$//' /docker-init.sh && chmod +rx /docker-init.sh
+RUN sed -i 's/\r$//' /docker-start.sh && chmod +rx /docker-start.sh
 
 # Fix user UID / GID to match host
 RUN groupmod --gid $USER_GID $USERNAME \
diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py
index 5df5d15e82..46833451c1 100644
--- a/docker/configs/settings_local.py
+++ b/docker/configs/settings_local.py
@@ -1,11 +1,13 @@
-# Copyright The IETF Trust 2007-2019, All Rights Reserved
+# Copyright The IETF Trust 2007-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-from ietf.settings import *                                          # pyflakes:ignore
+from ietf.settings import *  # pyflakes:ignore
+from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
+import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
-from ietf.settings_postgresqldb import DATABASES   # pyflakes:ignore
+from ietf.settings_postgresqldb import DATABASES  # pyflakes:ignore
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/"
@@ -37,6 +39,25 @@
 # DEV_TEMPLATE_CONTEXT_PROCESSORS = [
 #    'ietf.context_processors.sql_debug',
 # ]
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=botocore.config.Config(
+                signature_version="s3v4",
+                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
+                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
+                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
+            ),
+            verify=False,
+            bucket_name=storagename,
+        ),
+    }
+
 
 DOCUMENT_PATH_PATTERN = '/assets/ietfdata/doc/{doc.type_id}/'
 INTERNET_DRAFT_PATH = '/assets/ietf-ftp/internet-drafts/'
diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml
index 0538c0d3e9..a69a453110 100644
--- a/docker/docker-compose.extend.yml
+++ b/docker/docker-compose.extend.yml
@@ -16,6 +16,10 @@ services:
     pgadmin:
         ports:
             - '5433'
+    blobstore:
+        ports:
+            - '9000'
+            - '9001'
     celery:
         volumes:
             - .:/workspace
diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py
new file mode 100755
index 0000000000..7b5ce962eb
--- /dev/null
+++ b/docker/scripts/app-configure-blobstore.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# Copyright The IETF Trust 2024, All Rights Reserved
+
+import boto3
+import os
+import sys
+
+from ietf.settings import MORE_STORAGE_NAMES
+
+
+def init_blobstore():
+    blobstore = boto3.resource(
+        "s3",
+        endpoint_url=os.environ.get("BLOB_STORE_ENDPOINT_URL", "http://blobstore:9000"),
+        aws_access_key_id=os.environ.get("BLOB_STORE_ACCESS_KEY", "minio_root"),
+        aws_secret_access_key=os.environ.get("BLOB_STORE_SECRET_KEY", "minio_pass"),
+        aws_session_token=None,
+        config=botocore.config.Config(signature_version="s3v4"),
+        verify=False,
+    )
+    for bucketname in MORE_STORAGE_NAMES:
+        blobstore.create_bucket(
+            Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip()
+        )
+
+
+if __name__ == "__main__":
+    sys.exit(init_blobstore())
diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh
index b96b88f1f5..e970398ac2 100755
--- a/docker/scripts/app-init.sh
+++ b/docker/scripts/app-init.sh
@@ -73,6 +73,11 @@ echo "Creating data directories..."
 chmod +x ./docker/scripts/app-create-dirs.sh
 ./docker/scripts/app-create-dirs.sh
 
+# Configure the development blobstore
+
+echo "Configuring blobstore..."
+PYTHONPATH=/workspace python ./docker/scripts/app-configure-blobstore.py
+
 # Download latest coverage results file
 
 echo "Downloading latest coverage results file..."
diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index d9af457e95..ac0b37a608 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -25,6 +25,7 @@
 import debug                            # pyflakes:ignore
 
 import ietf
+from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils import get_unicode_document_content
 from ietf.doc.models import RelatedDocument, State
 from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory
@@ -553,6 +554,10 @@ def test_api_upload_polls_and_chatlog(self):
             newdoc = session.presentations.get(document__type_id=type_id).document
             newdoccontent = get_unicode_document_content(newdoc.name, Path(session.meeting.get_materials_path()) / type_id / newdoc.uploaded_filename)
             self.assertEqual(json.loads(content), json.loads(newdoccontent))
+            self.assertEqual(
+                json.loads(retrieve_str(type_id, newdoc.uploaded_filename)),
+                json.loads(content)
+            )
 
     def test_api_upload_bluesheet(self):
         url = urlreverse("ietf.meeting.views.api_upload_bluesheet")
diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py
index 301d32d7cc..db3b24b2d2 100644
--- a/ietf/doc/admin.py
+++ b/ietf/doc/admin.py
@@ -12,7 +12,7 @@
     TelechatDocEvent, BallotPositionDocEvent, ReviewRequestDocEvent, InitialReviewDocEvent,
     AddedMessageEvent, SubmissionDocEvent, DeletedEvent, EditedAuthorsDocEvent, DocumentURL,
     ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder,
-    BofreqEditorDocEvent, BofreqResponsibleDocEvent )
+    BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject )
 
 from ietf.utils.validators import validate_external_resource_value
 
@@ -218,3 +218,9 @@ class DocExtResourceAdmin(admin.ModelAdmin):
     search_fields = ['doc__name', 'value', 'display_name', 'name__slug',]
     raw_id_fields = ['doc', ]
 admin.site.register(DocExtResource, DocExtResourceAdmin)
+
+class StoredObjectAdmin(admin.ModelAdmin):
+    list_display = ['store', 'name', 'modified', 'deleted']
+    list_filter = ['deleted']
+    search_fields = ['store', 'name', 'doc_name', 'doc_rev', 'deleted']
+admin.site.register(StoredObject, StoredObjectAdmin)
diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py
index 98554bae0e..bf8523aa98 100644
--- a/ietf/doc/expire.py
+++ b/ietf/doc/expire.py
@@ -13,6 +13,7 @@
 
 from typing import List, Optional      # pyflakes:ignore
 
+from ietf.doc.storage_utils import exists_in_storage, remove_from_storage
 from ietf.doc.utils import update_action_holders
 from ietf.utils import log
 from ietf.utils.mail import send_mail
@@ -156,11 +157,17 @@ def remove_ftp_copy(f):
         if mark.exists():
             mark.unlink()
 
+    def remove_from_active_draft_storage(file):
+        # Assumes the glob will never find a file with no suffix
+        ext = file.suffix[1:]
+        remove_from_storage("active-draft", f"{ext}/{file.name}", warn_if_missing=False)
 
+    # Note that the object is already in the "draft" storage.
     src_dir = Path(settings.INTERNET_DRAFT_PATH)
     for file in src_dir.glob("%s-%s.*" % (doc.name, rev)):
         move_file(str(file.name))
         remove_ftp_copy(str(file.name))
+        remove_from_active_draft_storage(file)
 
 def expire_draft(doc):
     # clean up files
@@ -218,6 +225,13 @@ def move_file_to(subdir):
             mark = Path(settings.FTP_DIR) / "internet-drafts" / basename
             if mark.exists():
                 mark.unlink()
+            if ext:
+                # Note that we're not moving these strays anywhere - the assumption
+                # is that the active-draft blobstore will not get strays.
+                # See, however, the note about "major system failures" at "unknown_ids"
+                blobname = f"{ext[1:]}/{basename}"
+                if exists_in_storage("active-draft", blobname):
+                    remove_from_storage("active-draft", blobname)
 
         try:
             doc = Document.objects.get(name=filename, rev=revision)
diff --git a/ietf/doc/migrations/0025_storedobject_storedobject_unique_name_per_store.py b/ietf/doc/migrations/0025_storedobject_storedobject_unique_name_per_store.py
new file mode 100644
index 0000000000..e948ca3011
--- /dev/null
+++ b/ietf/doc/migrations/0025_storedobject_storedobject_unique_name_per_store.py
@@ -0,0 +1,66 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("doc", "0024_remove_ad_is_watching_states"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="StoredObject",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("store", models.CharField(max_length=256)),
+                ("name", models.CharField(max_length=1024)),
+                ("sha384", models.CharField(max_length=96)),
+                ("len", models.PositiveBigIntegerField()),
+                (
+                    "store_created",
+                    models.DateTimeField(
+                        help_text="The instant the object ws first placed in the store"
+                    ),
+                ),
+                (
+                    "created",
+                    models.DateTimeField(
+                        help_text="Instant object became known. May not be the same as the storage's created value for the instance. It will hold ctime for objects imported from older disk storage"
+                    ),
+                ),
+                (
+                    "modified",
+                    models.DateTimeField(
+                        help_text="Last instant object was modified. May not be the same as the storage's modified value for the instance. It will hold mtime for objects imported from older disk storage unless they've actually been overwritten more recently"
+                    ),
+                ),
+                ("doc_name", models.CharField(blank=True, max_length=255, null=True)),
+                ("doc_rev", models.CharField(blank=True, max_length=16, null=True)),
+                ("deleted", models.DateTimeField(null=True)),
+            ],
+            options={
+                "indexes": [
+                    models.Index(
+                        fields=["doc_name", "doc_rev"],
+                        name="doc_storedo_doc_nam_d04465_idx",
+                    )
+                ],
+            },
+        ),
+        migrations.AddConstraint(
+            model_name="storedobject",
+            constraint=models.UniqueConstraint(
+                fields=("store", "name"), name="unique_name_per_store"
+            ),
+        ),
+    ]
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index 03698c80c3..55da70972c 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -9,14 +9,16 @@
 import django.db
 import rfc2html
 
+from io import BufferedReader
 from pathlib import Path
 from lxml import etree
-from typing import Optional, TYPE_CHECKING
+from typing import Optional, Protocol, TYPE_CHECKING, Union
 from weasyprint import HTML as wpHTML
 from weasyprint.text.fonts import FontConfiguration
 
 from django.db import models
 from django.core import checks
+from django.core.files.base import File
 from django.core.cache import caches
 from django.core.validators import URLValidator, RegexValidator
 from django.urls import reverse as urlreverse
@@ -30,6 +32,11 @@
 import debug                            # pyflakes:ignore
 
 from ietf.group.models import Group
+from ietf.doc.storage_utils import (
+    store_str as utils_store_str,
+    store_bytes as utils_store_bytes,
+    store_file as utils_store_file
+)
 from ietf.name.models import ( DocTypeName, DocTagName, StreamName, IntendedStdLevelName, StdLevelName,
     DocRelationshipName, DocReminderTypeName, BallotPositionName, ReviewRequestStateName, ReviewAssignmentStateName, FormalLanguageName,
     DocUrlTagName, ExtResourceName)
@@ -714,10 +721,52 @@ def referenced_by_rfcs_as_rfc_or_draft(self):
         if self.type_id == "rfc" and self.came_from_draft():
             refs_to |= self.came_from_draft().referenced_by_rfcs()
         return refs_to
-
+    
     class Meta:
         abstract = True
 
+
+class HasNameRevAndTypeIdProtocol(Protocol):
+    """Typing Protocol describing a class that has name, rev, and type_id properties"""
+    @property
+    def name(self) -> str: ...
+    @property
+    def rev(self) -> str: ...
+    @property
+    def type_id(self) -> str: ...
+
+
+class StorableMixin:
+    """Mixin that adds storage helpers to a DocumentInfo subclass"""
+    def store_str(
+        self: HasNameRevAndTypeIdProtocol,
+        name: str,
+        content: str,
+        allow_overwrite: bool = False
+    ) -> None:
+        return utils_store_str(self.type_id, name, content, allow_overwrite, self.name, self.rev)
+
+    def store_bytes(
+        self: HasNameRevAndTypeIdProtocol,
+        name: str,
+        content: bytes,
+        allow_overwrite: bool = False,
+        doc_name: Optional[str] = None,
+        doc_rev: Optional[str] = None
+    ) -> None:
+        return utils_store_bytes(self.type_id, name, content, allow_overwrite, self.name, self.rev)
+
+    def store_file(
+        self: HasNameRevAndTypeIdProtocol,
+        name: str,
+        file: Union[File, BufferedReader],
+        allow_overwrite: bool = False,
+        doc_name: Optional[str] = None,
+        doc_rev: Optional[str] = None
+    ) -> None:
+        return utils_store_file(self.type_id, name, file, allow_overwrite, self.name, self.rev)
+
+
 STATUSCHANGE_RELATIONS = ('tops','tois','tohist','toinf','tobcp','toexp')
 
 class RelatedDocument(models.Model):
@@ -870,7 +919,7 @@ def role_for_doc(self):
     'invalid'
 )
 
-class Document(DocumentInfo):
+class Document(StorableMixin, DocumentInfo):
     name = models.CharField(max_length=255, validators=[validate_docname,], unique=True)           # immutable
     
     action_holders = models.ManyToManyField(Person, through=DocumentActionHolder, blank=True)
@@ -1192,7 +1241,7 @@ class DocHistoryAuthor(DocumentAuthorInfo):
     def __str__(self):
         return u"%s %s (%s)" % (self.document.doc.name, self.person, self.order)
 
-class DocHistory(DocumentInfo):
+class DocHistory(StorableMixin, DocumentInfo):
     doc = ForeignKey(Document, related_name="history_set")
 
     name = models.CharField(max_length=255)
@@ -1538,3 +1587,31 @@ class BofreqEditorDocEvent(DocEvent):
 class BofreqResponsibleDocEvent(DocEvent):
     """ Capture the responsible leadership (IAB and IESG members) for a BOF Request """
     responsible = models.ManyToManyField('person.Person', blank=True)
+
+class StoredObject(models.Model):
+    """Hold metadata about objects placed in object storage"""
+
+    store = models.CharField(max_length=256)
+    name = models.CharField(max_length=1024, null=False, blank=False) # N.B. the 1024 limit on name comes from S3
+    sha384 = models.CharField(max_length=96)
+    len = models.PositiveBigIntegerField()
+    store_created = models.DateTimeField(help_text="The instant the object ws first placed in the store")
+    created = models.DateTimeField(
+        null=False,
+        help_text="Instant object became known. May not be the same as the storage's created value for the instance. It will hold ctime for objects imported from older disk storage"
+    )
+    modified = models.DateTimeField(
+        null=False,
+        help_text="Last instant object was modified. May not be the same as the storage's modified value for the instance. It will hold mtime for objects imported from older disk storage unless they've actually been overwritten more recently"
+    )
+    doc_name = models.CharField(max_length=255, null=True, blank=True)
+    doc_rev = models.CharField(max_length=16, null=True, blank=True)
+    deleted = models.DateTimeField(null=True)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(fields=['store', 'name'], name='unique_name_per_store'),
+        ]
+        indexes = [
+            models.Index(fields=["doc_name", "doc_rev"]),
+        ]
diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py
index bba57013b9..157a3ad556 100644
--- a/ietf/doc/resources.py
+++ b/ietf/doc/resources.py
@@ -18,7 +18,7 @@
     RelatedDocHistory, BallotPositionDocEvent, AddedMessageEvent, SubmissionDocEvent,
     ReviewRequestDocEvent, ReviewAssignmentDocEvent, EditedAuthorsDocEvent, DocumentURL,
     IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, 
-    BofreqEditorDocEvent,BofreqResponsibleDocEvent)
+    BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject)
 
 from ietf.name.resources import BallotPositionNameResource, DocTypeNameResource
 class BallotTypeResource(ModelResource):
@@ -842,3 +842,26 @@ class Meta:
             "responsible": ALL_WITH_RELATIONS,
         }
 api.doc.register(BofreqResponsibleDocEventResource())
+
+
+class StoredObjectResource(ModelResource):
+    class Meta:
+        queryset = StoredObject.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'storedobject'
+        ordering = ['id', ]
+        filtering = { 
+            "id": ALL,
+            "store": ALL,
+            "name": ALL,
+            "sha384": ALL,
+            "len": ALL,
+            "store_created": ALL,
+            "created": ALL,
+            "modified": ALL,
+            "doc_name": ALL,
+            "doc_rev": ALL,
+            "deleted": ALL,
+        }
+api.doc.register(StoredObjectResource())
diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py
new file mode 100644
index 0000000000..5eeab040e5
--- /dev/null
+++ b/ietf/doc/storage_backends.py
@@ -0,0 +1,192 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+import debug  # pyflakes:ignore
+import json
+
+from contextlib import contextmanager
+from hashlib import sha384
+from io import BufferedReader
+from storages.backends.s3 import S3Storage
+from typing import Optional, Union
+
+from django.core.files.base import File
+
+from ietf.doc.models import StoredObject
+from ietf.utils.log import log
+from ietf.utils.timezone import timezone
+
+
+@contextmanager
+def maybe_log_timing(enabled, op, **kwargs):
+    """If enabled, log elapsed time and additional data from kwargs
+
+    Emits log even if an exception occurs
+    """
+    before = timezone.now()
+    exception = None
+    try:
+        yield
+    except Exception as err:
+        exception = err
+        raise
+    finally:
+        if enabled:
+            dt = timezone.now() - before
+            log(
+                json.dumps(
+                    {
+                        "log": "S3Storage_timing",
+                        "seconds": dt.total_seconds(),
+                        "op": op,
+                        "exception": "" if exception is None else repr(exception),
+                        **kwargs,
+                    }
+                )
+            )
+
+
+# TODO-BLOBSTORE
+# Consider overriding save directly so that
+# we capture metadata for, e.g., ImageField objects
+class CustomS3Storage(S3Storage):
+
+    def __init__(self, **settings):
+        self.in_flight_custom_metadata = {}  # type is Dict[str, Dict[str, str]]
+        super().__init__(**settings)
+
+    def get_default_settings(self):
+        # add a default for the ietf_log_blob_timing boolean
+        return super().get_default_settings() | {"ietf_log_blob_timing": False}
+
+    def _save(self, name, content):
+        with maybe_log_timing(
+            self.ietf_log_blob_timing, "_save", bucket_name=self.bucket_name, name=name
+        ):
+            return super()._save(name, content)
+
+    def _open(self, name, mode="rb"):
+        with maybe_log_timing(
+            self.ietf_log_blob_timing,
+            "_open",
+            bucket_name=self.bucket_name,
+            name=name,
+            mode=mode,
+        ):
+            return super()._open(name, mode)
+
+    def delete(self, name):
+        with maybe_log_timing(
+            self.ietf_log_blob_timing, "delete", bucket_name=self.bucket_name, name=name
+        ):
+            super().delete(name)
+
+    def store_file(
+        self,
+        kind: str,
+        name: str,
+        file: Union[File, BufferedReader],
+        allow_overwrite: bool = False,
+        doc_name: Optional[str] = None,
+        doc_rev: Optional[str] = None,
+    ):
+        is_new = not self.exists_in_storage(kind, name)
+        # debug.show('f"Asked to store {name} in {kind}: is_new={is_new}, allow_overwrite={allow_overwrite}"')
+        if not allow_overwrite and not is_new:
+            log(f"Failed to save {kind}:{name} - name already exists in store")
+            debug.show('f"Failed to save {kind}:{name} - name already exists in store"')
+            # raise Exception("Not ignoring overwrite attempts while testing")
+        else:
+            try:
+                new_name = self.save(name, file)
+                now = timezone.now()
+                record, created = StoredObject.objects.get_or_create(
+                    store=kind,
+                    name=name,
+                    defaults=dict(
+                        sha384=self.in_flight_custom_metadata[name]["sha384"],
+                        len=int(self.in_flight_custom_metadata[name]["len"]),
+                        store_created=now,
+                        created=now,
+                        modified=now,
+                        doc_name=doc_name,  # Note that these are assumed to be invariant
+                        doc_rev=doc_rev,  # for a given name
+                    ),
+                )
+                if not created:
+                    record.sha384 = self.in_flight_custom_metadata[name]["sha384"]
+                    record.len = int(self.in_flight_custom_metadata[name]["len"])
+                    record.modified = now
+                    record.deleted = None
+                    record.save()
+                if new_name != name:
+                    complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead."
+                    log(complaint)
+                    debug.show("complaint")
+                    # Note that we are otherwise ignoring this condition - it should become an error later.
+            except Exception as e:
+                # Log and then swallow the exception while we're learning.
+                # Don't let failure pass so quietly when these are the autoritative bits.
+                complaint = f"Failed to save {kind}:{name}"
+                log(complaint, e)
+                debug.show('f"{complaint}: {e}"')
+            finally:
+                del self.in_flight_custom_metadata[name]
+        return None
+
+    def exists_in_storage(self, kind: str, name: str) -> bool:
+        try:
+            # open is realized with a HEAD
+            # See https://github.com/jschneier/django-storages/blob/b79ea310201e7afd659fe47e2882fe59aae5b517/storages/backends/s3.py#L528
+            with self.open(name):
+                return True
+        except FileNotFoundError:
+            return False
+
+    def remove_from_storage(
+        self, kind: str, name: str, warn_if_missing: bool = True
+    ) -> None:
+        now = timezone.now()
+        try:
+            with self.open(name):
+                pass
+            self.delete(name)
+            # debug.show('f"deleted {name} from {kind} storage"')
+        except FileNotFoundError:
+            if warn_if_missing:
+                complaint = (
+                    f"WARNING: Asked to delete non-existent {name} from {kind} storage"
+                )
+                log(complaint)
+                debug.show("complaint")
+        existing_record = StoredObject.objects.filter(store=kind, name=name)
+        if not existing_record.exists() and warn_if_missing:
+            complaint = f"WARNING: Asked to delete {name} from {kind} storage, but there was no matching StorageObject"
+            log(complaint)
+            debug.show("complaint")
+        else:
+            # Note that existing_record is a queryset that will have one matching object
+            existing_record.filter(deleted__isnull=True).update(deleted=now)
+
+    def _get_write_parameters(self, name, content=None):
+        # debug.show('f"getting write parameters for {name}"')
+        params = super()._get_write_parameters(name, content)
+        if "Metadata" not in params:
+            params["Metadata"] = {}
+        try:
+            content.seek(0)
+        except AttributeError:  # TODO-BLOBSTORE
+            debug.say("Encountered Non-Seekable content")
+            raise NotImplementedError("cannot handle unseekable content")
+        content_bytes = content.read()
+        if not isinstance(
+            content_bytes, bytes
+        ):  # TODO-BLOBSTORE: This is sketch-development only -remove before committing
+            raise Exception(f"Expected bytes - got {type(content_bytes)}")
+        content.seek(0)
+        metadata = {
+            "len": f"{len(content_bytes)}",
+            "sha384": f"{sha384(content_bytes).hexdigest()}",
+        }
+        params["Metadata"].update(metadata)
+        self.in_flight_custom_metadata[name] = metadata
+        return params
diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py
new file mode 100644
index 0000000000..4f0516339a
--- /dev/null
+++ b/ietf/doc/storage_utils.py
@@ -0,0 +1,103 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from io import BufferedReader
+from typing import Optional, Union
+import debug  # pyflakes ignore
+
+from django.conf import settings
+from django.core.files.base import ContentFile, File
+from django.core.files.storage import storages
+
+
+# TODO-BLOBSTORE (Future, maybe after leaving 3.9) : add a return type
+def _get_storage(kind: str):
+
+    if kind in settings.MORE_STORAGE_NAMES:
+        # TODO-BLOBSTORE - add a checker that verifies configuration will only return CustomS3Storages
+        return storages[kind]
+    else:
+        debug.say(f"Got into not-implemented looking for {kind}")
+        raise NotImplementedError(f"Don't know how to store {kind}")
+
+
+def exists_in_storage(kind: str, name: str) -> bool:
+    if settings.ENABLE_BLOBSTORAGE:
+        store = _get_storage(kind)
+        return store.exists_in_storage(kind, name)
+    else:
+        return False
+
+
+def remove_from_storage(kind: str, name: str, warn_if_missing: bool = True) -> None:
+    if settings.ENABLE_BLOBSTORAGE:
+        store = _get_storage(kind)
+        store.remove_from_storage(kind, name, warn_if_missing)
+    return None
+
+
+# TODO-BLOBSTORE: Try to refactor `kind` out of the signature of the methods already on the custom store (which knows its kind)
+def store_file(
+    kind: str,
+    name: str,
+    file: Union[File, BufferedReader],
+    allow_overwrite: bool = False,
+    doc_name: Optional[str] = None,
+    doc_rev: Optional[str] = None,
+) -> None:
+    # debug.show('f"asked to store {name} into {kind}"')
+    if settings.ENABLE_BLOBSTORAGE:
+        store = _get_storage(kind)
+        store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev)
+    return None
+
+
+def store_bytes(
+    kind: str,
+    name: str,
+    content: bytes,
+    allow_overwrite: bool = False,
+    doc_name: Optional[str] = None,
+    doc_rev: Optional[str] = None,
+) -> None:
+    if settings.ENABLE_BLOBSTORAGE:
+        store_file(kind, name, ContentFile(content), allow_overwrite)
+    return None
+
+
+def store_str(
+    kind: str,
+    name: str,
+    content: str,
+    allow_overwrite: bool = False,
+    doc_name: Optional[str] = None,
+    doc_rev: Optional[str] = None,
+) -> None:
+    if settings.ENABLE_BLOBSTORAGE:
+        content_bytes = content.encode("utf-8")
+        store_bytes(kind, name, content_bytes, allow_overwrite)
+    return None
+
+
+def retrieve_bytes(kind: str, name: str) -> bytes:
+    from ietf.doc.storage_backends import maybe_log_timing
+    content = b""
+    if settings.ENABLE_BLOBSTORAGE:
+        store = _get_storage(kind)
+        with store.open(name) as f:
+            with maybe_log_timing(
+                hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing,
+                "read",
+                bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "",
+                name=name,
+            ):
+                content = f.read()
+    return content
+
+
+def retrieve_str(kind: str, name: str) -> str:
+    content = ""
+    if settings.ENABLE_BLOBSTORAGE:
+        content_bytes = retrieve_bytes(kind, name)
+        # TODO-BLOBSTORE: try to decode all the different ways doc.text() does
+        content = content_bytes.decode("utf-8")
+    return content
diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py
index 6eb901e6c7..e24c58e1e7 100644
--- a/ietf/doc/tasks.py
+++ b/ietf/doc/tasks.py
@@ -84,7 +84,7 @@ def generate_idnits2_rfc_status_task():
     outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status"
     blob = generate_idnits2_rfc_status()
     try:
-        outpath.write_text(blob, encoding="utf8")
+        outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE
     except Exception as e:
         log.log(f"failed to write idnits2-rfc-status: {e}")
 
@@ -94,7 +94,7 @@ def generate_idnits2_rfcs_obsoleted_task():
     outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted"
     blob = generate_idnits2_rfcs_obsoleted()
     try:
-        outpath.write_text(blob, encoding="utf8")
+        outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE
     except Exception as e:
         log.log(f"failed to write idnits2-rfcs-obsoleted: {e}")
 
diff --git a/ietf/doc/tests_bofreq.py b/ietf/doc/tests_bofreq.py
index 2e27efd627..6a7c9393ef 100644
--- a/ietf/doc/tests_bofreq.py
+++ b/ietf/doc/tests_bofreq.py
@@ -16,6 +16,7 @@
 from django.template.loader import render_to_string
 from django.utils import timezone
 
+from ietf.doc.storage_utils import retrieve_str
 from ietf.group.factories import RoleFactory
 from ietf.doc.factories import BofreqFactory, NewRevisionDocEventFactory
 from ietf.doc.models import State, Document, NewRevisionDocEvent
@@ -340,6 +341,7 @@ def test_submit(self):
                         doc = reload_db_objects(doc)
                         self.assertEqual('%02d'%(int(rev)+1) ,doc.rev)
                         self.assertEqual(f'# {username}', doc.text())
+                        self.assertEqual(f'# {username}', retrieve_str('bofreq',doc.get_base_name()))
                         self.assertEqual(docevent_count+1, doc.docevent_set.count())
                         self.assertEqual(1, len(outbox))
                         rev = doc.rev
@@ -379,6 +381,7 @@ def test_start_new_bofreq(self):
                     self.assertEqual(list(bofreq_editors(bofreq)), [nobody])
                     self.assertEqual(bofreq.latest_event(NewRevisionDocEvent).rev, '00')
                     self.assertEqual(bofreq.text_or_error(), 'some stuff')
+                    self.assertEqual(retrieve_str('bofreq',bofreq.get_base_name()), 'some stuff')
                     self.assertEqual(len(outbox),1)
         finally:
             os.unlink(file.name)
diff --git a/ietf/doc/tests_charter.py b/ietf/doc/tests_charter.py
index e0207fe842..62e49559e2 100644
--- a/ietf/doc/tests_charter.py
+++ b/ietf/doc/tests_charter.py
@@ -16,6 +16,7 @@
 from ietf.doc.factories import CharterFactory, NewRevisionDocEventFactory, TelechatDocEventFactory
 from ietf.doc.models import ( Document, State, BallotDocEvent, BallotType, NewRevisionDocEvent,
     TelechatDocEvent, WriteupDocEvent )
+from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils_charter import ( next_revision, default_review_text, default_action_text,
     charter_name_for_group )
 from ietf.doc.utils import close_open_ballots
@@ -519,6 +520,11 @@ def test_submit_charter(self):
         ftp_charter_path = Path(settings.FTP_DIR) / "charter" / charter_path.name
         self.assertTrue(ftp_charter_path.exists())
         self.assertTrue(charter_path.samefile(ftp_charter_path))
+        blobstore_contents = retrieve_str("charter", charter.get_base_name())
+        self.assertEqual(
+            blobstore_contents,
+            "Windows line\nMac line\nUnix line\n" + utf_8_snippet.decode("utf-8"),
+        )        
 
 
     def test_submit_initial_charter(self):
diff --git a/ietf/doc/tests_conflict_review.py b/ietf/doc/tests_conflict_review.py
index d2f94922b2..791db17f5a 100644
--- a/ietf/doc/tests_conflict_review.py
+++ b/ietf/doc/tests_conflict_review.py
@@ -16,6 +16,7 @@
 
 from ietf.doc.factories import IndividualDraftFactory, ConflictReviewFactory, RgDraftFactory
 from ietf.doc.models import Document, DocEvent, NewRevisionDocEvent, BallotPositionDocEvent, TelechatDocEvent, State, DocTagName
+from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils import create_ballot_if_not_open
 from ietf.doc.views_conflict_review import default_approval_text
 from ietf.group.models import Person
@@ -422,6 +423,7 @@ def test_initial_submission(self):
             f.close()
         self.assertTrue(ftp_path.exists())
         self.assertTrue( "submission-00" in doc.latest_event(NewRevisionDocEvent).desc)
+        self.assertEqual(retrieve_str("conflrev",basename), "Some initial review text\n")
 
     def test_subsequent_submission(self):
         doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py
index 2405806682..4753c4ff0c 100644
--- a/ietf/doc/tests_draft.py
+++ b/ietf/doc/tests_draft.py
@@ -24,6 +24,7 @@
 from ietf.doc.models import ( Document, DocReminder, DocEvent,
     ConsensusDocEvent, LastCallDocEvent, RelatedDocument, State, TelechatDocEvent, 
     WriteupDocEvent, DocRelationshipName, IanaExpertDocEvent )
+from ietf.doc.storage_utils import exists_in_storage, store_str
 from ietf.doc.utils import get_tags_for_stream_id, create_ballot_if_not_open
 from ietf.doc.views_draft import AdoptDraftForm
 from ietf.name.models import DocTagName, RoleName
@@ -577,6 +578,11 @@ def setUp(self):
     def write_draft_file(self, name, size):
         with (Path(settings.INTERNET_DRAFT_PATH) / name).open('w') as f:
             f.write("a" * size)
+        _, ext = os.path.splitext(name)
+        if ext:
+            ext=ext[1:]
+            store_str("active-draft", f"{ext}/{name}", "a"*size, allow_overwrite=True)
+            store_str("draft", f"{ext}/{name}", "a"*size, allow_overwrite=True)
 
 
 class ResurrectTests(DraftFileMixin, TestCase):
@@ -649,6 +655,7 @@ def test_resurrect(self):
         # ensure file restored from archive directory
         self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, txt)))
         self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, txt)))
+        self.assertTrue(exists_in_storage("active-draft",f"txt/{txt}"))
 
 
 class ExpireIDsTests(DraftFileMixin, TestCase):
@@ -775,6 +782,7 @@ def test_expire_drafts(self):
         self.assertEqual(draft.action_holders.count(), 0)
         self.assertIn('Removed all action holders', draft.latest_event(type='changed_action_holders').desc)
         self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, txt)))
+        self.assertFalse(exists_in_storage("active-draft", f"txt/{txt}"))
         self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, txt)))
 
         draft.delete()
@@ -798,6 +806,7 @@ def test_clean_up_draft_files(self):
         clean_up_draft_files()
         
         self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, unknown)))
+        self.assertFalse(exists_in_storage("active-draft", f"txt/{unknown}"))
         self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, "unknown_ids", unknown)))
 
         
@@ -808,6 +817,7 @@ def test_clean_up_draft_files(self):
         clean_up_draft_files()
         
         self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, malformed)))
+        self.assertFalse(exists_in_storage("active-draft", f"txt/{malformed}"))
         self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, "unknown_ids", malformed)))
 
         
@@ -822,9 +832,11 @@ def test_clean_up_draft_files(self):
         clean_up_draft_files()
         
         self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, txt)))
+        self.assertFalse(exists_in_storage("active-draft", f"txt/{txt}"))
         self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, txt)))
 
         self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, pdf)))
+        self.assertFalse(exists_in_storage("active-draft", f"pdf/{pdf}"))
         self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, pdf)))
 
         # expire draft
@@ -843,6 +855,7 @@ def test_clean_up_draft_files(self):
         clean_up_draft_files()
         
         self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, txt)))
+        self.assertFalse(exists_in_storage("active-draft", f"txt/{txt}"))
         self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, txt)))
 
 
diff --git a/ietf/doc/tests_material.py b/ietf/doc/tests_material.py
index aaea8fec3d..c87341c95b 100644
--- a/ietf/doc/tests_material.py
+++ b/ietf/doc/tests_material.py
@@ -18,6 +18,7 @@
 from django.utils import timezone
 
 from ietf.doc.models import Document, State, NewRevisionDocEvent
+from ietf.doc.storage_utils import retrieve_str
 from ietf.group.factories import RoleFactory
 from ietf.group.models import Group
 from ietf.meeting.factories import MeetingFactory, SessionFactory, SessionPresentationFactory
@@ -123,6 +124,9 @@ def test_upload_slides(self):
         ftp_filepath=Path(settings.FTP_DIR) / "slides" / basename
         with ftp_filepath.open() as f:
             self.assertEqual(f.read(), content)
+        # This test is very sloppy wrt the actual file content.
+        # Working with/around that for the moment.
+        self.assertEqual(retrieve_str("slides", basename), content)
 
         # check that posting same name is prevented
         test_file.seek(0)
@@ -237,4 +241,6 @@ def test_revise(self, mock_slides_manager_cls):
 
         with io.open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + ".txt")) as f:
             self.assertEqual(f.read(), content)
+        self.assertEqual(retrieve_str("slides", f"{doc.name}-{doc.rev}.txt"), content)
+
 
diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py
index a956fd3287..e93bc02181 100644
--- a/ietf/doc/tests_review.py
+++ b/ietf/doc/tests_review.py
@@ -20,6 +20,7 @@
 
 import debug                            # pyflakes:ignore
 
+from ietf.doc.storage_utils import retrieve_str
 import ietf.review.mailarch
 
 from ietf.doc.factories import ( NewRevisionDocEventFactory, IndividualDraftFactory, WgDraftFactory,
@@ -63,6 +64,10 @@ def verify_review_files_were_written(self, assignment, expected_content = "This
         review_file = Path(self.review_subdir) / f"{assignment.review.name}.txt"
         content = review_file.read_text()
         self.assertEqual(content, expected_content)
+        self.assertEqual(
+            retrieve_str("review", review_file.name),
+            expected_content
+        )
         review_ftp_file = Path(settings.FTP_DIR) / "review" / review_file.name
         self.assertTrue(review_file.samefile(review_ftp_file))
 
diff --git a/ietf/doc/tests_statement.py b/ietf/doc/tests_statement.py
index 2071018b10..fea42b97d6 100644
--- a/ietf/doc/tests_statement.py
+++ b/ietf/doc/tests_statement.py
@@ -14,6 +14,7 @@
 
 from ietf.doc.factories import StatementFactory, DocEventFactory
 from ietf.doc.models import Document, State, NewRevisionDocEvent
+from ietf.doc.storage_utils import retrieve_str
 from ietf.group.models import Group
 from ietf.person.factories import PersonFactory
 from ietf.utils.mail import outbox, empty_outbox
@@ -185,8 +186,16 @@ def test_submit(self):
                 self.assertEqual("%02d" % (int(rev) + 1), doc.rev)
                 if postdict["statement_submission"] == "enter":
                     self.assertEqual(f"# {username}", doc.text())
+                    self.assertEqual(
+                        retrieve_str("statement", f"{doc.name}-{doc.rev}.md"),
+                        f"# {username}"
+                    )
                 else:
                     self.assertEqual("not valid pdf", doc.text())
+                    self.assertEqual(
+                        retrieve_str("statement", f"{doc.name}-{doc.rev}.pdf"),
+                        "not valid pdf"
+                    )
                 self.assertEqual(docevent_count + 1, doc.docevent_set.count())
                 self.assertEqual(0, len(outbox))
                 rev = doc.rev
@@ -255,8 +264,16 @@ def test_start_new_statement(self):
             self.assertIsNotNone(statement.history_set.last().latest_event(type="published_statement"))
             if postdict["statement_submission"] == "enter":
                 self.assertEqual(statement.text_or_error(), "some stuff")
+                self.assertEqual(
+                    retrieve_str("statement", statement.uploaded_filename),
+                    "some stuff"
+                )
             else:
                 self.assertTrue(statement.uploaded_filename.endswith("pdf"))
+                self.assertEqual(
+                    retrieve_str("statement", f"{statement.name}-{statement.rev}.pdf"),
+                    "not valid pdf"
+                )
             self.assertEqual(len(outbox), 0)
 
         existing_statement = StatementFactory()
diff --git a/ietf/doc/tests_status_change.py b/ietf/doc/tests_status_change.py
index bd4da4c092..cbdc1a049a 100644
--- a/ietf/doc/tests_status_change.py
+++ b/ietf/doc/tests_status_change.py
@@ -19,6 +19,7 @@
     WgRfcFactory, DocEventFactory, WgDraftFactory )
 from ietf.doc.models import ( Document, State, DocEvent,
     BallotPositionDocEvent, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent )
+from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils import create_ballot_if_not_open
 from ietf.doc.views_status_change import default_approval_text
 from ietf.group.models import Person
@@ -71,7 +72,7 @@ def test_start_review(self):
             statchg_relation_row_blah="tois")
         )
         self.assertEqual(r.status_code, 302)
-        status_change = Document.objects.get(name='status-change-imaginary-new')        
+        status_change = Document.objects.get(name='status-change-imaginary-new')
         self.assertEqual(status_change.get_state('statchg').slug,'adrev')
         self.assertEqual(status_change.rev,'00')
         self.assertEqual(status_change.ad.name,'Areað Irector')
@@ -563,6 +564,8 @@ def test_initial_submission(self):
         ftp_filepath = Path(settings.FTP_DIR) / "status-changes" / basename
         self.assertFalse(filepath.exists())
         self.assertFalse(ftp_filepath.exists())
+        with self.assertRaises(FileNotFoundError):
+            retrieve_str("statchg",basename)
         r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1"))
         self.assertEqual(r.status_code,302)
         doc = Document.objects.get(name='status-change-imaginary-mid-review')
@@ -571,6 +574,10 @@ def test_initial_submission(self):
             self.assertEqual(f.read(),"Some initial review text\n")
         with ftp_filepath.open() as f:
             self.assertEqual(f.read(),"Some initial review text\n")
+        self.assertEqual(
+            retrieve_str("statchg", basename),
+            "Some initial review text\n"
+        )
         self.assertTrue( "mid-review-00" in doc.latest_event(NewRevisionDocEvent).desc)
 
     def test_subsequent_submission(self):
@@ -607,7 +614,8 @@ def test_subsequent_submission(self):
         self.assertContains(r, "does not appear to be a text file")
 
         # sane post uploading a file
-        test_file = StringIO("This is a new proposal.")
+        test_content = "This is a new proposal."
+        test_file = StringIO(test_content)
         test_file.name = "unnamed"
         r = self.client.post(url,dict(txt=test_file,submit_response="1"))
         self.assertEqual(r.status_code, 302)
@@ -615,8 +623,12 @@ def test_subsequent_submission(self):
         self.assertEqual(doc.rev,'01')
         path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.name, doc.rev))
         with io.open(path) as f:
-            self.assertEqual(f.read(),"This is a new proposal.")
+            self.assertEqual(f.read(), test_content)
             f.close()
+        self.assertEqual(
+            retrieve_str("statchg", f"{doc.name}-{doc.rev}.txt"),
+            test_content
+        )
         self.assertTrue( "mid-review-01" in doc.latest_event(NewRevisionDocEvent).desc)
 
         # verify reset text button works
diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index 10fe9ff2d7..3ddd904c75 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -1510,7 +1510,7 @@ def update_or_create_draft_bibxml_file(doc, rev):
         existing_bibxml = ""
     if normalized_bibxml.strip() != existing_bibxml.strip():
         log.log(f"Writing {ref_rev_file_path}")
-        ref_rev_file_path.write_text(normalized_bibxml, encoding="utf8")
+        ref_rev_file_path.write_text(normalized_bibxml, encoding="utf8") # TODO-BLOBSTORE
 
 
 def ensure_draft_bibxml_path_exists():
diff --git a/ietf/doc/views_bofreq.py b/ietf/doc/views_bofreq.py
index 3bd10287b2..71cbe30491 100644
--- a/ietf/doc/views_bofreq.py
+++ b/ietf/doc/views_bofreq.py
@@ -101,6 +101,7 @@ def submit(request, name):
                 content = form.cleaned_data['bofreq_content']
             with io.open(bofreq.get_file_name(), 'w', encoding='utf-8') as destination:
                 destination.write(content)
+            bofreq.store_str(bofreq.get_base_name(), content)
             email_bofreq_new_revision(request, bofreq)
             return redirect('ietf.doc.views_doc.document_main', name=bofreq.name)
 
@@ -175,6 +176,7 @@ def new_bof_request(request):
                 content = form.cleaned_data['bofreq_content']
             with io.open(bofreq.get_file_name(), 'w', encoding='utf-8') as destination:
                 destination.write(content)
+            bofreq.store_str(bofreq.get_base_name(), content)
             email_bofreq_new_revision(request, bofreq)
             return redirect('ietf.doc.views_doc.document_main', name=bofreq.name)
 
diff --git a/ietf/doc/views_charter.py b/ietf/doc/views_charter.py
index f8748d2126..e899f59227 100644
--- a/ietf/doc/views_charter.py
+++ b/ietf/doc/views_charter.py
@@ -441,9 +441,10 @@ def submit(request, name, option=None):
             )  # update rev
             with charter_filename.open("w", encoding="utf-8") as destination:
                 if form.cleaned_data["txt"]:
-                    destination.write(form.cleaned_data["txt"])
+                    content=form.cleaned_data["txt"]
                 else:
-                    destination.write(form.cleaned_data["content"])
+                    content=form.cleaned_data["content"]
+                destination.write(content)
             # Also provide a copy to the legacy ftp source directory, which is served by rsync
             # This replaces the hardlink copy that ghostlink has made in the past
             # Still using a hardlink as long as these are on the same filesystem.
@@ -454,7 +455,8 @@ def submit(request, name, option=None):
                 log(
                     "There was an error creating a hardlink at %s pointing to %s"
                     % (ftp_filename, charter_filename)
-                )     
+                )
+            charter.store_str(charter_filename.name, content)     
 
 
             if option in ["initcharter", "recharter"] and charter.ad == None:
diff --git a/ietf/doc/views_conflict_review.py b/ietf/doc/views_conflict_review.py
index e55661ccdf..159f1340a4 100644
--- a/ietf/doc/views_conflict_review.py
+++ b/ietf/doc/views_conflict_review.py
@@ -186,9 +186,10 @@ def save(self, review):
         filepath = Path(settings.CONFLICT_REVIEW_PATH) / basename
         with filepath.open('w', encoding='utf-8') as destination:
             if self.cleaned_data['txt']:
-                destination.write(self.cleaned_data['txt'])
+                content = self.cleaned_data['txt']
             else:
-                destination.write(self.cleaned_data['content'])
+                content = self.cleaned_data['content']
+            destination.write(content)
         ftp_filepath = Path(settings.FTP_DIR) / "conflict-reviews" / basename
         try:
             os.link(filepath, ftp_filepath) # Path.hardlink_to is not available until 3.10
@@ -197,6 +198,7 @@ def save(self, review):
                 "There was an error creating a hardlink at %s pointing to %s: %s"
                 % (ftp_filepath, filepath, e)
             )
+        review.store_str(basename, content)
 
 #This is very close to submit on charter - can we get better reuse?
 @role_required('Area Director','Secretariat')
diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py
index 34104b2005..c80537afb3 100644
--- a/ietf/doc/views_draft.py
+++ b/ietf/doc/views_draft.py
@@ -32,6 +32,7 @@
     generate_publication_request, email_adopted, email_intended_status_changed,
     email_iesg_processing_document, email_ad_approved_doc,
     email_iana_expert_review_state_changed )
+from ietf.doc.storage_utils import retrieve_bytes, store_bytes
 from ietf.doc.utils import ( add_state_change_event, can_adopt_draft, can_unadopt_draft,
     get_tags_for_stream_id, nice_consensus, update_action_holders,
     update_reminder, update_telechat, make_notify_changed_event, get_initial_notify,
@@ -897,6 +898,11 @@ def restore_draft_file(request, draft):
         except shutil.Error as ex:
             messages.warning(request, 'There was an error restoring the Internet-Draft file: {} ({})'.format(file, ex))
             log.log("  Exception %s when attempting to move %s" % (ex, file))
+        _, ext = os.path.splitext(os.path.basename(file))
+        if ext:
+            ext = ext[1:]
+            blobname = f"{ext}/{basename}.{ext}"
+            store_bytes("active-draft", blobname, retrieve_bytes("draft", blobname))
 
 
 class ShepherdWriteupUploadForm(forms.Form):
diff --git a/ietf/doc/views_material.py b/ietf/doc/views_material.py
index 361bf5f1e2..6f8b8a8f12 100644
--- a/ietf/doc/views_material.py
+++ b/ietf/doc/views_material.py
@@ -167,6 +167,8 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None):
                 with filepath.open('wb+') as dest:
                     for chunk in f.chunks():
                         dest.write(chunk)
+                f.seek(0)
+                doc.store_file(basename, f)
                 if not doc.meeting_related():
                     log.assertion('doc.type_id == "slides"')
                     ftp_filepath = Path(settings.FTP_DIR) / doc.type_id / basename
diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py
index bb9e56742d..1f23c435fa 100644
--- a/ietf/doc/views_review.py
+++ b/ietf/doc/views_review.py
@@ -805,6 +805,7 @@ def complete_review(request, name, assignment_id=None, acronym=None):
 
             review_path = Path(review.get_file_path()) / f"{review.name}.txt"
             review_path.write_text(content)
+            review.store_str(f"{review.name}.txt", content, allow_overwrite=True) # We have a bug that review revisions dont create a new version!
             review_ftp_path = Path(settings.FTP_DIR) / "review" / review_path.name
             # See https://github.com/ietf-tools/datatracker/issues/6941 - when that's
             # addressed, making this link should not be conditional
diff --git a/ietf/doc/views_statement.py b/ietf/doc/views_statement.py
index bf9f47ddfe..9dc8c8ad69 100644
--- a/ietf/doc/views_statement.py
+++ b/ietf/doc/views_statement.py
@@ -137,12 +137,15 @@ def submit(request, name):
                 mode="wb" if writing_pdf else "w"
             ) as destination:
                 if writing_pdf:
-                    for chunk in form.cleaned_data["statement_file"].chunks():
+                    f = form.cleaned_data["statement_file"]
+                    for chunk in f.chunks():
                         destination.write(chunk)
+                    f.seek(0)
+                    statement.store_file(statement.uploaded_filename, f)
                 else:
                     destination.write(markdown_content)
+                    statement.store_str(statement.uploaded_filename, markdown_content)
             return redirect("ietf.doc.views_doc.document_main", name=statement.name)
-
     else:
         if statement.uploaded_filename.endswith("pdf"):
             text = CONST_PDF_REV_NOTICE
@@ -254,10 +257,14 @@ def new_statement(request):
                 mode="wb" if writing_pdf else "w"
             ) as destination:
                 if writing_pdf:
-                    for chunk in form.cleaned_data["statement_file"].chunks():
+                    f = form.cleaned_data["statement_file"]
+                    for chunk in f.chunks():
                         destination.write(chunk)
+                        f.seek(0)
+                        statement.store_file(statement.uploaded_filename, f)
                 else:
                     destination.write(markdown_content)
+                    statement.store_str(statement.uploaded_filename, markdown_content)
             return redirect("ietf.doc.views_doc.document_main", name=statement.name)
 
     else:
diff --git a/ietf/doc/views_status_change.py b/ietf/doc/views_status_change.py
index 33b822348a..2bccc213c4 100644
--- a/ietf/doc/views_status_change.py
+++ b/ietf/doc/views_status_change.py
@@ -160,9 +160,11 @@ def save(self, doc):
         filename = Path(settings.STATUS_CHANGE_PATH) / basename
         with io.open(filename, 'w', encoding='utf-8') as destination:
             if self.cleaned_data['txt']:
-                destination.write(self.cleaned_data['txt'])
+                content = self.cleaned_data['txt']
             else:
-                destination.write(self.cleaned_data['content'])
+                content = self.cleaned_data['content']
+            destination.write(content)
+            doc.store_str(basename, content)
         try:
             ftp_filename = Path(settings.FTP_DIR) / "status-changes" / basename
             os.link(filename, ftp_filename) # Path.hardlink is not available until 3.10
diff --git a/ietf/group/tasks.py b/ietf/group/tasks.py
index 8b4c994ba1..693aafb385 100644
--- a/ietf/group/tasks.py
+++ b/ietf/group/tasks.py
@@ -10,6 +10,7 @@
 from django.conf import settings
 from django.template.loader import render_to_string
 
+from ietf.doc.storage_utils import store_file
 from ietf.utils import log
 
 from .models import Group
@@ -43,6 +44,11 @@ def generate_wg_charters_files_task():
         encoding="utf8",
     )
 
+    with charters_file.open("rb") as f:
+        store_file("indexes", "1wg-charters.txt", f, allow_overwrite=True)
+    with charters_by_acronym_file.open("rb") as f:
+        store_file("indexes", "1wg-charters-by-acronym.txt", f, allow_overwrite=True)
+
     charter_copy_dests = [
         getattr(settings, "CHARTER_COPY_PATH", None), 
         getattr(settings, "CHARTER_COPY_OTHER_PATH", None),
@@ -102,3 +108,8 @@ def generate_wg_summary_files_task():
         ),
         encoding="utf8",
     )
+
+    with summary_file.open("rb") as f:
+        store_file("indexes", "1wg-summary.txt", f, allow_overwrite=True)
+    with summary_by_acronym_file.open("rb") as f:
+        store_file("indexes", "1wg-summary-by-acronym.txt", f, allow_overwrite=True)
diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py
index 32d919c779..aaf937ee43 100644
--- a/ietf/group/tests_info.py
+++ b/ietf/group/tests_info.py
@@ -29,6 +29,7 @@
 from ietf.community.utils import reset_name_contains_index_for_rule
 from ietf.doc.factories import WgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory
 from ietf.doc.models import Document, DocEvent, State
+from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils_charter import charter_name_for_group
 from ietf.group.admin import GroupForm as AdminGroupForm
 from ietf.group.factories import (GroupFactory, RoleFactory, GroupEventFactory, 
@@ -303,20 +304,26 @@ def test_generate_wg_summary_files_task(self):
 
         generate_wg_summary_files_task()
 
-        summary_by_area_contents = (
-            Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary.txt"
-        ).read_text(encoding="utf8")
-        self.assertIn(group.parent.name, summary_by_area_contents)
-        self.assertIn(group.acronym, summary_by_area_contents)
-        self.assertIn(group.name, summary_by_area_contents)
-        self.assertIn(chair.address, summary_by_area_contents)
-
-        summary_by_acronym_contents = (
-            Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary-by-acronym.txt"
-        ).read_text(encoding="utf8")
-        self.assertIn(group.acronym, summary_by_acronym_contents)
-        self.assertIn(group.name, summary_by_acronym_contents)
-        self.assertIn(chair.address, summary_by_acronym_contents)
+        for summary_by_area_contents in [
+            (
+                Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary.txt"
+            ).read_text(encoding="utf8"),
+            retrieve_str("indexes", "1wg-summary.txt")
+        ]:
+            self.assertIn(group.parent.name, summary_by_area_contents)
+            self.assertIn(group.acronym, summary_by_area_contents)
+            self.assertIn(group.name, summary_by_area_contents)
+            self.assertIn(chair.address, summary_by_area_contents)
+
+        for summary_by_acronym_contents in [
+            (
+                Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary-by-acronym.txt"
+            ).read_text(encoding="utf8"),
+            retrieve_str("indexes", "1wg-summary-by-acronym.txt")
+        ]:
+            self.assertIn(group.acronym, summary_by_acronym_contents)
+            self.assertIn(group.name, summary_by_acronym_contents)
+            self.assertIn(chair.address, summary_by_acronym_contents)
 
     def test_chartering_groups(self):
         group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area'),states=[('charter','intrev')]).group
diff --git a/ietf/idindex/tasks.py b/ietf/idindex/tasks.py
index 5e7e193bba..2f5f1871d7 100644
--- a/ietf/idindex/tasks.py
+++ b/ietf/idindex/tasks.py
@@ -15,6 +15,8 @@
 
 from django.conf import settings
 
+from ietf.doc.storage_utils import store_file
+
 from .index import all_id_txt, all_id2_txt, id_index_txt
 
 
@@ -38,6 +40,8 @@ def move_into_place(self, src_path: Path, dest_path: Path, hardlink_dirs: List[P
             target = path / dest_path.name
             target.unlink(missing_ok=True)
             os.link(dest_path, target) # until python>=3.10
+        with dest_path.open("rb") as f:
+            store_file("indexes", dest_path.name, f, allow_overwrite=True)
 
     def cleanup(self):
         for tf_path in self.cleanup_list:
diff --git a/ietf/idindex/tests.py b/ietf/idindex/tests.py
index 44abf805f0..5cc7a7b3bb 100644
--- a/ietf/idindex/tests.py
+++ b/ietf/idindex/tests.py
@@ -15,6 +15,7 @@
 
 from ietf.doc.factories import WgDraftFactory, RfcFactory
 from ietf.doc.models import Document, RelatedDocument, State, LastCallDocEvent, NewRevisionDocEvent
+from ietf.doc.storage_utils import retrieve_str
 from ietf.group.factories import GroupFactory
 from ietf.name.models import DocRelationshipName
 from ietf.idindex.index import all_id_txt, all_id2_txt, id_index_txt
@@ -203,5 +204,9 @@ def test_temp_file_manager(self):
                 self.assertFalse(path2.exists())  # left behind
                 # check destination contents and permissions
                 self.assertEqual(dest.read_text(), "yay")
+                self.assertEqual(
+                    retrieve_str("indexes", "yay.txt"),
+                    "yay"
+                )
                 self.assertEqual(dest.stat().st_mode & 0o777, 0o644)
                 self.assertTrue(dest.samefile(other_path / "yay.txt"))
diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py
index 1d91041b25..1af29044b3 100644
--- a/ietf/liaisons/forms.py
+++ b/ietf/liaisons/forms.py
@@ -379,6 +379,8 @@ def save_attachments(self):
             attach_file = io.open(os.path.join(settings.LIAISON_ATTACH_PATH, attach.name + extension), 'wb')
             attach_file.write(attached_file.read())
             attach_file.close()
+            attached_file.seek(0)
+            attach.store_file(attach.uploaded_filename, attached_file)
 
             if not self.is_new:
                 # create modified event
diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py
index a0186f6a01..1742687f14 100644
--- a/ietf/liaisons/tests.py
+++ b/ietf/liaisons/tests.py
@@ -19,6 +19,7 @@
 from io import StringIO
 from pyquery import PyQuery
 
+from ietf.doc.storage_utils import retrieve_str
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized
 from ietf.utils.mail import outbox
 
@@ -414,7 +415,8 @@ def test_edit_liaison(self):
 
         # edit
         attachments_before = liaison.attachments.count()
-        test_file = StringIO("hello world")
+        test_content = "hello world"
+        test_file = StringIO(test_content)
         test_file.name = "unnamed"
         r = self.client.post(url,
                              dict(from_groups=str(from_group.pk),
@@ -452,9 +454,12 @@ def test_edit_liaison(self):
         self.assertEqual(attachment.title, "attachment")
         with (Path(settings.LIAISON_ATTACH_PATH) / attachment.uploaded_filename).open() as f:
             written_content = f.read()
+        self.assertEqual(written_content, test_content)
+        self.assertEqual(
+            retrieve_str(attachment.type_id, attachment.uploaded_filename),
+            test_content,
+        )
 
-        test_file.seek(0)
-        self.assertEqual(written_content, test_file.read())
 
     def test_incoming_access(self):
         '''Ensure only Secretariat, Liaison Managers, and Authorized Individuals
@@ -704,7 +709,8 @@ def test_add_incoming_liaison(self):
 
         # add new
         mailbox_before = len(outbox)
-        test_file = StringIO("hello world")
+        test_content = "hello world"
+        test_file = StringIO(test_content)
         test_file.name = "unnamed"
         from_groups = [ str(g.pk) for g in Group.objects.filter(type="sdo") ]
         to_group = Group.objects.get(acronym="mars")
@@ -756,6 +762,11 @@ def test_add_incoming_liaison(self):
         self.assertEqual(attachment.title, "attachment")
         with (Path(settings.LIAISON_ATTACH_PATH) / attachment.uploaded_filename).open() as f:
             written_content = f.read()
+        self.assertEqual(written_content, test_content)
+        self.assertEqual(
+            retrieve_str(attachment.type_id, attachment.uploaded_filename),
+            test_content
+        )
 
         test_file.seek(0)
         self.assertEqual(written_content, test_file.read())
@@ -783,7 +794,8 @@ def test_add_outgoing_liaison(self):
 
         # add new
         mailbox_before = len(outbox)
-        test_file = StringIO("hello world")
+        test_content = "hello world"
+        test_file = StringIO(test_content)
         test_file.name = "unnamed"
         from_group = Group.objects.get(acronym="mars")
         to_group = Group.objects.filter(type="sdo")[0]
@@ -835,9 +847,11 @@ def test_add_outgoing_liaison(self):
         self.assertEqual(attachment.title, "attachment")
         with (Path(settings.LIAISON_ATTACH_PATH) / attachment.uploaded_filename).open() as f:
             written_content = f.read()
-
-        test_file.seek(0)
-        self.assertEqual(written_content, test_file.read())
+        self.assertEqual(written_content, test_content)
+        self.assertEqual(
+            retrieve_str(attachment.type_id, attachment.uploaded_filename),
+            test_content
+        )
 
         self.assertEqual(len(outbox), mailbox_before + 1)
         self.assertTrue("Liaison Statement" in outbox[-1]["Subject"])
@@ -882,7 +896,8 @@ def test_liaison_add_attachment(self):
 
 
         # get minimum edit post data
-        file = StringIO('dummy file')
+        test_data = "dummy file"
+        file = StringIO(test_data)
         file.name = "upload.txt"
         post_data = dict(
             from_groups = ','.join([ str(x.pk) for x in liaison.from_groups.all() ]),
@@ -909,6 +924,11 @@ def test_liaison_add_attachment(self):
         self.assertEqual(liaison.attachments.count(),1)
         event = liaison.liaisonstatementevent_set.order_by('id').last()
         self.assertTrue(event.desc.startswith('Added attachment'))
+        attachment = liaison.attachments.get()
+        self.assertEqual(
+            retrieve_str(attachment.type_id, attachment.uploaded_filename),
+            test_data
+        )
 
     def test_liaison_edit_attachment(self):
 
diff --git a/ietf/meeting/factories.py b/ietf/meeting/factories.py
index 69c1f0421b..eb36e9e756 100644
--- a/ietf/meeting/factories.py
+++ b/ietf/meeting/factories.py
@@ -9,6 +9,7 @@
 from django.core.files.base import ContentFile
 from django.db.models import Q
 
+from ietf.doc.storage_utils import store_str
 from ietf.meeting.models import (Attended, Meeting, Session, SchedulingEvent, Schedule,
     TimeSlot, SessionPresentation, FloorPlan, Room, SlideSubmission, Constraint,
     MeetingHost, ProceedingsMaterial)
@@ -239,6 +240,10 @@ class Meta:
     make_file = factory.PostGeneration(
                     lambda obj, create, extracted, **kwargs: open(obj.staged_filepath(),'a').close()
                 )
+    
+    store_submission = factory.PostGeneration(
+        lambda obj, create, extracted, **kwargs: store_str("staging", obj.filename, "")
+    )
 
 class ConstraintFactory(factory.django.DjangoModelFactory):
     class Meta:
diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py
index 3b66d2cd29..e1d1e90b8d 100644
--- a/ietf/meeting/forms.py
+++ b/ietf/meeting/forms.py
@@ -361,6 +361,7 @@ def save_agenda(self):
             os.makedirs(directory)
         with io.open(path, "w", encoding='utf-8') as file:
             file.write(self.cleaned_data['agenda'])
+        doc.store_str(doc.uploaded_filename, self.cleaned_data['agenda'])
 
 
 class InterimAnnounceForm(forms.ModelForm):
diff --git a/ietf/meeting/helpers.py b/ietf/meeting/helpers.py
index 7f1c85990e..39d271ae6b 100644
--- a/ietf/meeting/helpers.py
+++ b/ietf/meeting/helpers.py
@@ -649,6 +649,11 @@ def read_session_file(type, num, doc):
 def read_agenda_file(num, doc):
     return read_session_file('agenda', num, doc)
 
+# TODO-BLOBSTORE: this is _yet another_ draft derived variant created when users
+# ask for drafts from the meeting agenda page. Consider whether to refactor this
+# now to not call out to external binaries, and consider whether we need this extra
+# format at all in the draft blobstore. if so, it would probably be stored under 
+# something like plainpdf/ 
 def convert_draft_to_pdf(doc_name):
     inpath = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, doc_name + ".txt")
     outpath = os.path.join(settings.INTERNET_DRAFT_PDF_PATH, doc_name + ".pdf")
diff --git a/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py b/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py
new file mode 100644
index 0000000000..594a1a4048
--- /dev/null
+++ b/ietf/meeting/migrations/0010_alter_floorplan_image_alter_meetinghost_logo.py
@@ -0,0 +1,56 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+import ietf.meeting.models
+import ietf.utils.fields
+import ietf.utils.storage
+import ietf.utils.validators
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("meeting", "0009_session_meetecho_recording_name"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="floorplan",
+            name="image",
+            field=models.ImageField(
+                blank=True,
+                default=None,
+                storage=ietf.utils.storage.BlobShadowFileSystemStorage(
+                    kind="", location=None
+                ),
+                upload_to=ietf.meeting.models.floorplan_path,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="meetinghost",
+            name="logo",
+            field=ietf.utils.fields.MissingOkImageField(
+                height_field="logo_height",
+                storage=ietf.utils.storage.BlobShadowFileSystemStorage(
+                    kind="", location=None
+                ),
+                upload_to=ietf.meeting.models._host_upload_path,
+                validators=[
+                    ietf.utils.validators.MaxImageSizeValidator(400, 400),
+                    ietf.utils.validators.WrappedValidator(
+                        ietf.utils.validators.validate_file_size, True
+                    ),
+                    ietf.utils.validators.WrappedValidator(
+                        ietf.utils.validators.validate_file_extension,
+                        [".png", ".jpg", ".jpeg"],
+                    ),
+                    ietf.utils.validators.WrappedValidator(
+                        ietf.utils.validators.validate_mime_type,
+                        ["image/jpeg", "image/png"],
+                        True,
+                    ),
+                ],
+                width_field="logo_width",
+            ),
+        ),
+    ]
diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index 8c6fb97413..5284420731 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -39,7 +39,7 @@
 from ietf.person.models import Person
 from ietf.utils.decorators import memoize
 from ietf.utils.history import find_history_replacements_active_at, find_history_active_at
-from ietf.utils.storage import NoLocationMigrationFileSystemStorage
+from ietf.utils.storage import BlobShadowFileSystemStorage
 from ietf.utils.text import xslugify
 from ietf.utils.timezone import datetime_from_date, date_today
 from ietf.utils.models import ForeignKey
@@ -527,7 +527,12 @@ class FloorPlan(models.Model):
     modified= models.DateTimeField(auto_now=True)
     meeting = ForeignKey(Meeting)
     order   = models.SmallIntegerField()
-    image   = models.ImageField(storage=NoLocationMigrationFileSystemStorage(), upload_to=floorplan_path, blank=True, default=None)
+    image   = models.ImageField(
+        storage=BlobShadowFileSystemStorage(kind="floorplan"),
+        upload_to=floorplan_path,
+        blank=True,
+        default=None,
+    )
     #
     class Meta:
         ordering = ['-id',]
@@ -1431,8 +1436,12 @@ class MeetingHost(models.Model):
     """Meeting sponsor"""
     meeting = ForeignKey(Meeting, related_name='meetinghosts')
     name = models.CharField(max_length=255, blank=False)
+    # TODO-BLOBSTORE - capture these logos and look for other ImageField like model fields.
     logo = MissingOkImageField(
-        storage=NoLocationMigrationFileSystemStorage(location=settings.MEETINGHOST_LOGO_PATH),
+        storage=BlobShadowFileSystemStorage(
+            kind="meetinghostlogo",
+            location=settings.MEETINGHOST_LOGO_PATH,
+        ),
         upload_to=_host_upload_path,
         width_field='logo_width',
         height_field='logo_height',
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 581725dbc8..848c9b7723 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -38,6 +38,7 @@
 import debug           # pyflakes:ignore
 
 from ietf.doc.models import Document, NewRevisionDocEvent
+from ietf.doc.storage_utils import exists_in_storage, remove_from_storage, retrieve_bytes, retrieve_str
 from ietf.group.models import Group, Role, GroupFeatures
 from ietf.group.utils import can_manage_group
 from ietf.person.models import Person
@@ -55,6 +56,7 @@
 from ietf.name.models import SessionStatusName, ImportantDateName, RoleName, ProceedingsMaterialTypeName
 from ietf.utils.decorators import skip_coverage
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
+from ietf.utils.test_runner import TestBlobstoreManager
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent
 from ietf.utils.timezone import date_today, time_now
 
@@ -112,7 +114,7 @@ def setUp(self):
         # files will upload to the locations specified in settings.py.
         # Note that this will affect any use of the storage class in
         # meeting.models - i.e., FloorPlan.image and MeetingHost.logo
-        self.patcher = patch('ietf.meeting.models.NoLocationMigrationFileSystemStorage.base_location',
+        self.patcher = patch('ietf.meeting.models.BlobShadowFileSystemStorage.base_location',
                              new_callable=PropertyMock)
         mocked = self.patcher.start()
         mocked.return_value = self.storage_dir
@@ -5228,6 +5230,7 @@ def test_interim_request_options(self):
 
     def do_interim_request_single_virtual(self, emails_expected):
         make_meeting_test_data()
+        TestBlobstoreManager().emptyTestBlobstores()
         group = Group.objects.get(acronym='mars')
         date = date_today() + datetime.timedelta(days=30)
         time = time_now().replace(microsecond=0,second=0)
@@ -5278,6 +5281,12 @@ def do_interim_request_single_virtual(self, emails_expected):
         doc = session.materials.first()
         path = os.path.join(doc.get_file_path(),doc.filename_with_rev())
         self.assertTrue(os.path.exists(path))
+        with Path(path).open() as f:
+            self.assertEqual(f.read(), agenda)
+        self.assertEqual(
+            retrieve_str("agenda",doc.uploaded_filename),
+            agenda
+        )
         # check notices to secretariat and chairs
         self.assertEqual(len(outbox), length_before + emails_expected)
         return meeting
@@ -5299,6 +5308,7 @@ def test_interim_request_single_virtual_settings_approval_not_required(self):
 
     def test_interim_request_single_in_person(self):
         make_meeting_test_data()
+        TestBlobstoreManager().emptyTestBlobstores()
         group = Group.objects.get(acronym='mars')
         date = date_today() + datetime.timedelta(days=30)
         time = time_now().replace(microsecond=0,second=0)
@@ -5345,6 +5355,10 @@ def test_interim_request_single_in_person(self):
         timeslot = session.official_timeslotassignment().timeslot
         self.assertEqual(timeslot.time,dt)
         self.assertEqual(timeslot.duration,duration)
+        self.assertEqual(
+            retrieve_str("agenda",session.agenda().uploaded_filename),
+            agenda
+        )
 
     def test_interim_request_multi_day(self):
         make_meeting_test_data()
@@ -5412,6 +5426,11 @@ def test_interim_request_multi_day(self):
         self.assertEqual(timeslot.time,dt2)
         self.assertEqual(timeslot.duration,duration)
         self.assertEqual(session.agenda_note,agenda_note)
+        for session in meeting.session_set.all():
+            self.assertEqual(
+                retrieve_str("agenda",session.agenda().uploaded_filename),
+                agenda
+            )
 
     def test_interim_request_multi_day_non_consecutive(self):
         make_meeting_test_data()
@@ -5474,6 +5493,7 @@ def test_interim_request_multi_day_cancel(self):
 
     def test_interim_request_series(self):
         make_meeting_test_data()
+        TestBlobstoreManager().emptyTestBlobstores()
         meeting_count_before = Meeting.objects.filter(type='interim').count()
         date = date_today() + datetime.timedelta(days=30)
         if (date.month, date.day) == (12, 31):
@@ -5561,6 +5581,11 @@ def test_interim_request_series(self):
         self.assertEqual(timeslot.time,dt2)
         self.assertEqual(timeslot.duration,duration)
         self.assertEqual(session.agenda_note,agenda_note)
+        for session in meeting.session_set.all():
+            self.assertEqual(
+                retrieve_str("agenda",session.agenda().uploaded_filename),
+                agenda
+            )
 
 
     # test_interim_pending subsumed by test_appears_on_pending
@@ -6099,6 +6124,7 @@ def strfdelta(self, tdelta, fmt):
     def test_interim_request_edit_agenda_updates_doc(self):
         """Updating the agenda through the request edit form should update the doc correctly"""
         make_interim_test_data()
+        TestBlobstoreManager().emptyTestBlobstores()
         meeting = add_event_info_to_session_qs(Session.objects.filter(meeting__type='interim', group__acronym='mars')).filter(current_status='sched').first().meeting
         group = meeting.session_set.first().group
         url = urlreverse('ietf.meeting.views.interim_request_edit', kwargs={'number': meeting.number})
@@ -6134,6 +6160,10 @@ def test_interim_request_edit_agenda_updates_doc(self):
         self.assertNotEqual(agenda_doc.uploaded_filename, uploaded_filename_before, 'Uploaded filename should be updated')
         with (Path(agenda_doc.get_file_path()) / agenda_doc.uploaded_filename).open() as f:
             self.assertEqual(f.read(), 'modified agenda contents', 'New agenda contents should be saved')
+        self.assertEqual(
+            retrieve_str(agenda_doc.type_id, agenda_doc.uploaded_filename),
+            "modified agenda contents"
+        )
 
     def test_interim_request_details_permissions(self):
         make_interim_test_data()
@@ -6354,12 +6384,14 @@ def test_upload_bluesheets(self):
         q = PyQuery(r.content)
         self.assertIn('Upload', str(q("title")))
         self.assertFalse(session.presentations.exists())
-        test_file = StringIO('%PDF-1.4\n%âãÏÓ\nthis is some text for a test')
+        test_content = '%PDF-1.4\n%âãÏÓ\nthis is some text for a test'
+        test_file = StringIO(test_content)
         test_file.name = "not_really.pdf"
         r = self.client.post(url,dict(file=test_file))
         self.assertEqual(r.status_code, 302)
         bs_doc = session.presentations.filter(document__type_id='bluesheets').first().document
         self.assertEqual(bs_doc.rev,'00')
+        self.assertEqual(retrieve_str("bluesheets", f"{bs_doc.name}-{bs_doc.rev}.pdf"), test_content)
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
@@ -6389,12 +6421,14 @@ def test_upload_bluesheets_interim(self):
         q = PyQuery(r.content)
         self.assertIn('Upload', str(q("title")))
         self.assertFalse(session.presentations.exists())
-        test_file = StringIO('%PDF-1.4\n%âãÏÓ\nthis is some text for a test')
+        test_content = '%PDF-1.4\n%âãÏÓ\nthis is some text for a test'
+        test_file = StringIO(test_content)
         test_file.name = "not_really.pdf"
         r = self.client.post(url,dict(file=test_file))
         self.assertEqual(r.status_code, 302)
         bs_doc = session.presentations.filter(document__type_id='bluesheets').first().document
         self.assertEqual(bs_doc.rev,'00')
+        self.assertEqual(retrieve_str("bluesheets", f"{bs_doc.name}-{bs_doc.rev}.pdf"), test_content)
 
     def test_upload_bluesheets_interim_chair_access(self):
         make_meeting_test_data()
@@ -6467,27 +6501,36 @@ def test_upload_minutes_agenda(self):
             text = doc.text()
             self.assertIn('Some text', text)
             self.assertNotIn('<section>', text)
-        
+            text = retrieve_str(doctype, f"{doc.name}-{doc.rev}.html")
+            self.assertIn('Some text', text)
+            self.assertNotIn('<section>', text)
+
             # txt upload
-            test_file = BytesIO(b'This is some text for a test, with the word\nvirtual at the beginning of a line.')
+            test_bytes = b'This is some text for a test, with the word\nvirtual at the beginning of a line.'
+            test_file = BytesIO(test_bytes)
             test_file.name = "some.txt"
             r = self.client.post(url,dict(submission_method="upload",file=test_file,apply_to_all=False))
             self.assertEqual(r.status_code, 302)
             doc = session.presentations.filter(document__type_id=doctype).first().document
             self.assertEqual(doc.rev,'01')
             self.assertFalse(session2.presentations.filter(document__type_id=doctype))
+            retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt")
+            self.assertEqual(retrieved_bytes, test_bytes)
     
             r = self.client.get(url)
             self.assertEqual(r.status_code, 200)
             q = PyQuery(r.content)
             self.assertIn('Revise', str(q("Title")))
-            test_file = BytesIO(b'this is some different text for a test')
+            test_bytes = b'this is some different text for a test'
+            test_file = BytesIO(test_bytes)
             test_file.name = "also_some.txt"
             r = self.client.post(url,dict(submission_method="upload",file=test_file,apply_to_all=True))
             self.assertEqual(r.status_code, 302)
             doc = Document.objects.get(pk=doc.pk)
             self.assertEqual(doc.rev,'02')
             self.assertTrue(session2.presentations.filter(document__type_id=doctype))
+            retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt")
+            self.assertEqual(retrieved_bytes, test_bytes)
 
             # Test bad encoding
             test_file = BytesIO('<html><h1>Title</h1><section>Some\x93text</section></html>'.encode('latin1'))
@@ -6540,12 +6583,15 @@ def test_upload_minutes_agenda_interim(self):
             q = PyQuery(r.content)
             self.assertIn('Upload', str(q("title")))
             self.assertFalse(session.presentations.filter(document__type_id=doctype))
-            test_file = BytesIO(b'this is some text for a test')
+            test_bytes = b'this is some text for a test'
+            test_file = BytesIO(test_bytes)
             test_file.name = "not_really.txt"
             r = self.client.post(url,dict(submission_method="upload",file=test_file))
             self.assertEqual(r.status_code, 302)
             doc = session.presentations.filter(document__type_id=doctype).first().document
             self.assertEqual(doc.rev,'00')
+            retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt")
+            self.assertEqual(retrieved_bytes, test_bytes)
 
             # Verify that we don't have dead links
             url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
@@ -6567,12 +6613,15 @@ def test_upload_narrativeminutes(self):
             q = PyQuery(r.content)
             self.assertIn('Upload', str(q("title")))
             self.assertFalse(session.presentations.filter(document__type_id=doctype))
-            test_file = BytesIO(b'this is some text for a test')
+            test_bytes = b'this is some text for a test'
+            test_file = BytesIO(test_bytes)
             test_file.name = "not_really.txt"
             r = self.client.post(url,dict(submission_method="upload",file=test_file))
             self.assertEqual(r.status_code, 302)
             doc = session.presentations.filter(document__type_id=doctype).first().document
             self.assertEqual(doc.rev,'00')
+            retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt")
+            self.assertEqual(retrieved_bytes, test_bytes)
 
             # Verify that we don't have dead links
             url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
@@ -6597,18 +6646,22 @@ def test_enter_agenda(self):
         self.assertRedirects(r, redirect_url)
         doc = session.presentations.filter(document__type_id='agenda').first().document
         self.assertEqual(doc.rev,'00')
+        self.assertEqual(retrieve_str("agenda",f"{doc.name}-{doc.rev}.md"), test_text)
 
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertIn('Revise', str(q("Title")))
 
-        test_file = BytesIO(b'Upload after enter')
+        test_bytes = b'Upload after enter'
+        test_file = BytesIO(test_bytes)
         test_file.name = "some.txt"
         r = self.client.post(url,dict(submission_method="upload",file=test_file))
         self.assertRedirects(r, redirect_url)
         doc = Document.objects.get(pk=doc.pk)
         self.assertEqual(doc.rev,'01')
+        retrieved_bytes = retrieve_bytes("agenda", f"{doc.name}-{doc.rev}.txt")
+        self.assertEqual(retrieved_bytes, test_bytes)
 
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
@@ -6620,6 +6673,8 @@ def test_enter_agenda(self):
         self.assertRedirects(r, redirect_url)
         doc = Document.objects.get(pk=doc.pk)
         self.assertEqual(doc.rev,'02')
+        self.assertEqual(retrieve_str("agenda",f"{doc.name}-{doc.rev}.md"), test_text)
+
 
     @override_settings(MEETECHO_API_CONFIG="fake settings")  # enough to trigger API calls
     @patch("ietf.meeting.views.SlidesManager")
@@ -6635,7 +6690,8 @@ def test_upload_slides(self, mock_slides_manager_cls):
         q = PyQuery(r.content)
         self.assertIn('Upload', str(q("title")))
         self.assertFalse(session1.presentations.filter(document__type_id='slides'))
-        test_file = BytesIO(b'this is not really a slide')
+        test_bytes = b'this is not really a slide'
+        test_file = BytesIO(test_bytes)
         test_file.name = 'not_really.txt'
         r = self.client.post(url,dict(file=test_file,title='a test slide file',apply_to_all=True,approved=True))
         self.assertEqual(r.status_code, 302)
@@ -6647,6 +6703,7 @@ def test_upload_slides(self, mock_slides_manager_cls):
         self.assertEqual(mock_slides_manager_cls.call_count, 1)
         self.assertEqual(mock_slides_manager_cls.call_args, call(api_config="fake settings"))
         self.assertEqual(mock_slides_manager_cls.return_value.add.call_count, 2)
+        self.assertEqual(retrieve_bytes("slides", f"{sp.document.name}-{sp.document.rev}.txt"), test_bytes)
         # don't care which order they were called in, just that both sessions were updated
         self.assertCountEqual(
             mock_slides_manager_cls.return_value.add.call_args_list,
@@ -6658,7 +6715,8 @@ def test_upload_slides(self, mock_slides_manager_cls):
         mock_slides_manager_cls.reset_mock()
 
         url = urlreverse('ietf.meeting.views.upload_session_slides',kwargs={'num':session2.meeting.number,'session_id':session2.id})
-        test_file = BytesIO(b'some other thing still not slidelike')
+        test_bytes = b'some other thing still not slidelike'
+        test_file = BytesIO(test_bytes)
         test_file.name = 'also_not_really.txt'
         r = self.client.post(url,dict(file=test_file,title='a different slide file',apply_to_all=False,approved=True))
         self.assertEqual(r.status_code, 302)
@@ -6671,6 +6729,7 @@ def test_upload_slides(self, mock_slides_manager_cls):
         self.assertEqual(mock_slides_manager_cls.call_count, 1)
         self.assertEqual(mock_slides_manager_cls.call_args, call(api_config="fake settings"))
         self.assertEqual(mock_slides_manager_cls.return_value.add.call_count, 1)
+        self.assertEqual(retrieve_bytes("slides", f"{sp.document.name}-{sp.document.rev}.txt"), test_bytes)
         self.assertEqual(
             mock_slides_manager_cls.return_value.add.call_args,
             call(session=session2, slides=sp.document, order=2),
@@ -6682,7 +6741,8 @@ def test_upload_slides(self, mock_slides_manager_cls):
         self.assertTrue(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertIn('Revise', str(q("title")))
-        test_file = BytesIO(b'new content for the second slide deck')
+        test_bytes = b'new content for the second slide deck'
+        test_file = BytesIO(test_bytes)
         test_file.name = 'doesnotmatter.txt'
         r = self.client.post(url,dict(file=test_file,title='rename the presentation',apply_to_all=False, approved=True))
         self.assertEqual(r.status_code, 302)
@@ -6692,6 +6752,7 @@ def test_upload_slides(self, mock_slides_manager_cls):
         self.assertEqual(replacement_sp.rev,'01')
         self.assertEqual(replacement_sp.document.rev,'01')
         self.assertEqual(mock_slides_manager_cls.call_count, 1)
+        self.assertEqual(retrieve_bytes("slides", f"{replacement_sp.document.name}-{replacement_sp.document.rev}.txt"), test_bytes)
         self.assertEqual(mock_slides_manager_cls.call_args, call(api_config="fake settings"))
         self.assertEqual(mock_slides_manager_cls.return_value.revise.call_count, 1)
         self.assertEqual(
@@ -6771,7 +6832,6 @@ def test_remove_sessionpresentation(self, mock_slides_manager_cls):
         self.assertEqual(2, agenda.docevent_set.count())
         self.assertFalse(mock_slides_manager_cls.called)
 
-
     def test_propose_session_slides(self):
         for type_id in ['ietf','interim']:
             session = SessionFactory(meeting__type_id=type_id)
@@ -6798,7 +6858,8 @@ def test_propose_session_slides(self):
             login_testing_unauthorized(self,newperson.user.username,upload_url)
             r = self.client.get(upload_url)
             self.assertEqual(r.status_code,200)
-            test_file = BytesIO(b'this is not really a slide')
+            test_bytes = b'this is not really a slide'
+            test_file = BytesIO(test_bytes)
             test_file.name = 'not_really.txt'
             empty_outbox()
             r = self.client.post(upload_url,dict(file=test_file,title='a test slide file',apply_to_all=True,approved=False))
@@ -6806,6 +6867,10 @@ def test_propose_session_slides(self):
             session = Session.objects.get(pk=session.pk)
             self.assertEqual(session.slidesubmission_set.count(),1)
             self.assertEqual(len(outbox),1)
+            self.assertEqual(
+                retrieve_bytes("staging", session.slidesubmission_set.get().filename),
+                test_bytes
+            )
 
             r = self.client.get(session_overview_url)
             self.assertEqual(r.status_code, 200)
@@ -6825,13 +6890,20 @@ def test_propose_session_slides(self):
             login_testing_unauthorized(self,chair.user.username,upload_url)
             r = self.client.get(upload_url)
             self.assertEqual(r.status_code,200)
-            test_file = BytesIO(b'this is not really a slide either')
+            test_bytes = b'this is not really a slide either'
+            test_file = BytesIO(test_bytes)
             test_file.name = 'again_not_really.txt'
             empty_outbox()
             r = self.client.post(upload_url,dict(file=test_file,title='a selfapproved test slide file',apply_to_all=True,approved=True))
             self.assertEqual(r.status_code, 302)
             self.assertEqual(len(outbox),0)
             self.assertEqual(session.slidesubmission_set.count(),2)
+            sp = session.presentations.get(document__title__contains="selfapproved")
+            self.assertFalse(exists_in_storage("staging", sp.document.uploaded_filename))
+            self.assertEqual(
+                retrieve_bytes("slides", sp.document.uploaded_filename),
+                test_bytes
+            )
             self.client.logout()
 
             self.client.login(username=chair.user.username, password=chair.user.username+"+password")
@@ -6854,6 +6926,8 @@ def test_disapprove_proposed_slides(self):
         self.assertEqual(r.status_code,302)
         self.assertEqual(SlideSubmission.objects.filter(status__slug = 'rejected').count(), 1)
         self.assertEqual(SlideSubmission.objects.filter(status__slug = 'pending').count(), 0)
+        if submission.filename is not None and submission.filename != "":
+            self.assertFalse(exists_in_storage("staging", submission.filename))
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         self.assertRegex(r.content.decode(), r"These\s+slides\s+have\s+already\s+been\s+rejected")
@@ -6872,6 +6946,7 @@ def test_approve_proposed_slides(self, mock_slides_manager_cls):
         r = self.client.get(url)
         self.assertEqual(r.status_code,200)
         empty_outbox()
+        self.assertTrue(exists_in_storage("staging", submission.filename))
         r = self.client.post(url,dict(title='different title',approve='approve'))
         self.assertEqual(r.status_code,302)
         self.assertEqual(SlideSubmission.objects.filter(status__slug = 'pending').count(), 0)
@@ -6881,6 +6956,8 @@ def test_approve_proposed_slides(self, mock_slides_manager_cls):
         self.assertIsNotNone(submission.doc)
         self.assertEqual(session.presentations.count(),1)
         self.assertEqual(session.presentations.first().document.title,'different title')
+        self.assertTrue(exists_in_storage("slides", submission.doc.uploaded_filename))
+        self.assertFalse(exists_in_storage("staging", submission.filename))
         self.assertEqual(mock_slides_manager_cls.call_count, 1)
         self.assertEqual(mock_slides_manager_cls.call_args, call(api_config="fake settings"))
         self.assertEqual(mock_slides_manager_cls.return_value.add.call_count, 1)
@@ -6900,6 +6977,7 @@ def test_approve_proposed_slides(self, mock_slides_manager_cls):
     @override_settings(MEETECHO_API_CONFIG="fake settings")  # enough to trigger API calls
     @patch("ietf.meeting.views.SlidesManager")
     def test_approve_proposed_slides_multisession_apply_one(self, mock_slides_manager_cls):
+        TestBlobstoreManager().emptyTestBlobstores()
         submission = SlideSubmissionFactory(session__meeting__type_id='ietf')
         session1 = submission.session
         session2 = SessionFactory(group=submission.session.group, meeting=submission.session.meeting)
@@ -6928,6 +7006,7 @@ def test_approve_proposed_slides_multisession_apply_one(self, mock_slides_manage
     @override_settings(MEETECHO_API_CONFIG="fake settings")  # enough to trigger API calls
     @patch("ietf.meeting.views.SlidesManager")
     def test_approve_proposed_slides_multisession_apply_all(self, mock_slides_manager_cls):
+        TestBlobstoreManager().emptyTestBlobstores()
         submission = SlideSubmissionFactory(session__meeting__type_id='ietf')
         session1 = submission.session
         session2 = SessionFactory(group=submission.session.group, meeting=submission.session.meeting)
@@ -6972,12 +7051,15 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls):
 
         submission = SlideSubmission.objects.get(session=session)
 
+        self.assertTrue(exists_in_storage("staging", submission.filename))
         approve_url = urlreverse('ietf.meeting.views.approve_proposed_slides', kwargs={'slidesubmission_id':submission.pk,'num':submission.session.meeting.number})
         login_testing_unauthorized(self, chair.user.username, approve_url)
         r = self.client.post(approve_url,dict(title=submission.title,approve='approve'))
         submission.refresh_from_db()
         self.assertEqual(r.status_code,302)
         self.client.logout()
+        self.assertFalse(exists_in_storage("staging", submission.filename))
+        self.assertTrue(exists_in_storage("slides", submission.doc.uploaded_filename))
         self.assertEqual(mock_slides_manager_cls.call_count, 1)
         self.assertEqual(mock_slides_manager_cls.call_args, call(api_config="fake settings"))
         self.assertEqual(mock_slides_manager_cls.return_value.add.call_count, 1)
@@ -7003,11 +7085,16 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls):
 
         (first_submission, second_submission) = SlideSubmission.objects.filter(session=session, status__slug = 'pending').order_by('id')
 
+        self.assertTrue(exists_in_storage("staging", first_submission.filename))
+        self.assertTrue(exists_in_storage("staging", second_submission.filename))
         approve_url = urlreverse('ietf.meeting.views.approve_proposed_slides', kwargs={'slidesubmission_id':second_submission.pk,'num':second_submission.session.meeting.number})
         login_testing_unauthorized(self, chair.user.username, approve_url)
         r = self.client.post(approve_url,dict(title=submission.title,approve='approve'))
         first_submission.refresh_from_db()
         second_submission.refresh_from_db()
+        self.assertTrue(exists_in_storage("staging", first_submission.filename))
+        self.assertFalse(exists_in_storage("staging", second_submission.filename))
+        self.assertTrue(exists_in_storage("slides", second_submission.doc.uploaded_filename))
         self.assertEqual(r.status_code,302)
         self.assertEqual(mock_slides_manager_cls.call_count, 1)
         self.assertEqual(mock_slides_manager_cls.call_args, call(api_config="fake settings"))
@@ -7024,6 +7111,7 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls):
         self.assertEqual(r.status_code,302)
         self.client.logout()
         self.assertFalse(mock_slides_manager_cls.called)
+        self.assertFalse(exists_in_storage("staging", first_submission.filename))
 
         self.assertEqual(SlideSubmission.objects.filter(status__slug = 'pending').count(),0)
         self.assertEqual(SlideSubmission.objects.filter(status__slug = 'rejected').count(),1)
@@ -7114,6 +7202,10 @@ def test_imports_previewed_text(self):
         minutes_path = Path(self.meeting.get_materials_path()) / 'minutes'
         with (minutes_path / self.session.minutes().uploaded_filename).open() as f:
             self.assertEqual(f.read(), 'original markdown text')
+        self.assertEqual(
+            retrieve_str("minutes", self.session.minutes().uploaded_filename),
+            'original markdown text'
+        )
 
     def test_refuses_identical_import(self):
         """Should not be able to import text identical to the current revision"""
@@ -7173,7 +7265,9 @@ def test_handles_missing_previous_revision_file(self):
         # remove the file uploaded for the first rev
         minutes_docs = self.session.presentations.filter(document__type='minutes')
         self.assertEqual(minutes_docs.count(), 1)
-        Path(minutes_docs.first().document.get_file_name()).unlink()
+        to_remove = Path(minutes_docs.first().document.get_file_name())
+        to_remove.unlink()
+        remove_from_storage("minutes", to_remove.name)
 
         self.assertEqual(r.status_code, 302)
         with requests_mock.Mocker() as mock:
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index 92bae5ac23..9603278399 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -24,6 +24,7 @@
 import debug                            # pyflakes:ignore
 
 from ietf.dbtemplate.models import DBTemplate
+from ietf.doc.storage_utils import store_bytes, store_str
 from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot,
     Constraint, SchedTimeSessAssignment, SessionPresentation, Attended)
 from ietf.doc.models import Document, State, NewRevisionDocEvent, StateDocEvent
@@ -772,7 +773,12 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N
             # Whole file sanitization; add back what's missing from a complete
             # document (sanitize will remove these).
             clean = clean_html(text)
-            destination.write(clean.encode("utf8"))
+            clean_bytes = clean.encode('utf8')
+            destination.write(clean_bytes)
+            # Assumes contents of subdir are always document type ids
+            # TODO-BLOBSTORE: see if we can refactor this so that the connection to the document isn't lost
+            # In the meantime, consider faking it by parsing filename (shudder).
+            store_bytes(subdir, filename.name, clean_bytes)
             if request and clean != text:
                 messages.warning(request,
                                  (
@@ -783,6 +789,11 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N
         else:
             for chunk in chunks:
                 destination.write(chunk)
+            file.seek(0)
+            if hasattr(file, "chunks"):
+                chunks = file.chunks()
+            # TODO-BLOBSTORE: See above question about refactoring
+            store_bytes(subdir, filename.name, b"".join(chunks))
 
     return None
 
@@ -809,13 +820,15 @@ def new_doc_for_session(type_id, session):
     session.presentations.create(document=doc,rev='00')
     return doc
 
+# TODO-BLOBSTORE - consider adding doc to this signature and factoring away type_id
 def write_doc_for_session(session, type_id, filename, contents):
     filename = Path(filename)
     path = Path(session.meeting.get_materials_path()) / type_id
     path.mkdir(parents=True, exist_ok=True)
     with open(path / filename, "wb") as file:
         file.write(contents.encode('utf-8'))
-    return
+    store_str(type_id, filename.name, contents)
+    return None
 
 def create_recording(session, url, title=None, user=None):
     '''
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 1226e30d60..3fa605ed7e 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -52,6 +52,7 @@
 
 from ietf.doc.fields import SearchableDocumentsField
 from ietf.doc.models import Document, State, DocEvent, NewRevisionDocEvent
+from ietf.doc.storage_utils import remove_from_storage, retrieve_bytes, store_file
 from ietf.group.models import Group
 from ietf.group.utils import can_manage_session_materials, can_manage_some_groups, can_manage_group
 from ietf.person.models import Person, User
@@ -3091,6 +3092,8 @@ def upload_session_slides(request, session_id, num, name=None):
                 for chunk in file.chunks():
                     destination.write(chunk)
                 destination.close()
+                file.seek(0)
+                store_file("staging", filename, file)
 
                 submission.filename = filename
                 submission.save()
@@ -4645,7 +4648,6 @@ def err(code, text):
         save_err = save_bluesheet(request, session, file)
     if save_err:
         return err(400, save_err)
-
     return HttpResponse("Done", status=200, content_type='text/plain')
 
 
@@ -4957,6 +4959,8 @@ def approve_proposed_slides(request, slidesubmission_id, num):
                 if not os.path.exists(path):
                     os.makedirs(path)
                 shutil.move(submission.staged_filepath(), os.path.join(path, target_filename))
+                doc.store_bytes(target_filename, retrieve_bytes("staging", submission.filename))
+                remove_from_storage("staging", submission.filename)
                 post_process(doc)
                 DocEvent.objects.create(type="approved_slides", doc=doc, rev=doc.rev, by=request.user.person, desc="Slides approved")
 
@@ -4994,11 +4998,14 @@ def approve_proposed_slides(request, slidesubmission_id, num):
                 # in a SlideSubmission object without a file.  Handle
                 # this case and keep processing the 'disapprove' even if
                 # the filename doesn't exist.
-                try:
-                    if submission.filename != None and submission.filename != '':
+
+                if submission.filename != None and submission.filename != '':
+                    try:
                         os.unlink(submission.staged_filepath())
-                except (FileNotFoundError, IsADirectoryError):
-                    pass
+                    except (FileNotFoundError, IsADirectoryError):
+                        pass
+                    remove_from_storage("staging", submission.filename)
+
                 acronym = submission.session.group.acronym
                 submission.status = SlideSubmissionStatusName.objects.get(slug='rejected')
                 submission.save()
diff --git a/ietf/nomcom/models.py b/ietf/nomcom/models.py
index 2ed1124c5c..c206e467bd 100644
--- a/ietf/nomcom/models.py
+++ b/ietf/nomcom/models.py
@@ -42,6 +42,7 @@ class ReminderDates(models.Model):
 
 
 class NomCom(models.Model):
+    # TODO-BLOBSTORE: migrate this to a database field instead of a FileField and update code accordingly
     public_key = models.FileField(storage=NoLocationMigrationFileSystemStorage(location=settings.NOMCOM_PUBLIC_KEYS_DIR),
                                   upload_to=upload_path_handler, blank=True, null=True)
 
diff --git a/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py b/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py
new file mode 100644
index 0000000000..f34382fa70
--- /dev/null
+++ b/ietf/person/migrations/0004_alter_person_photo_alter_person_photo_thumb.py
@@ -0,0 +1,38 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+import ietf.utils.storage
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("person", "0003_alter_personalapikey_endpoint"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="person",
+            name="photo",
+            field=models.ImageField(
+                blank=True,
+                default=None,
+                storage=ietf.utils.storage.BlobShadowFileSystemStorage(
+                    kind="", location=None
+                ),
+                upload_to="photo",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="person",
+            name="photo_thumb",
+            field=models.ImageField(
+                blank=True,
+                default=None,
+                storage=ietf.utils.storage.BlobShadowFileSystemStorage(
+                    kind="", location=None
+                ),
+                upload_to="photo",
+            ),
+        ),
+    ]
diff --git a/ietf/person/models.py b/ietf/person/models.py
index 85989acfc1..93364478ae 100644
--- a/ietf/person/models.py
+++ b/ietf/person/models.py
@@ -29,7 +29,7 @@
 from ietf.name.models import ExtResourceName
 from ietf.person.name import name_parts, initials, plain_name
 from ietf.utils.mail import send_mail_preformatted
-from ietf.utils.storage import NoLocationMigrationFileSystemStorage
+from ietf.utils.storage import BlobShadowFileSystemStorage
 from ietf.utils.mail import formataddr
 from ietf.person.name import unidecode_name
 from ietf.utils import log
@@ -60,8 +60,18 @@ class Person(models.Model):
     pronouns_selectable = jsonfield.JSONCharField("Pronouns", max_length=120, blank=True, null=True, default=list )
     pronouns_freetext = models.CharField(" ", max_length=30, null=True, blank=True, help_text="Optionally provide your personal pronouns. These will be displayed on your public profile page and alongside your name in Meetecho and, in future, other systems. Select any number of the checkboxes OR provide a custom string up to 30 characters.")
     biography = models.TextField(blank=True, help_text="Short biography for use on leadership pages. Use plain text or reStructuredText markup.")
-    photo = models.ImageField(storage=NoLocationMigrationFileSystemStorage(), upload_to=settings.PHOTOS_DIRNAME, blank=True, default=None)
-    photo_thumb = models.ImageField(storage=NoLocationMigrationFileSystemStorage(), upload_to=settings.PHOTOS_DIRNAME, blank=True, default=None)
+    photo = models.ImageField(
+        storage=BlobShadowFileSystemStorage(kind="photo"),
+        upload_to=settings.PHOTOS_DIRNAME,
+        blank=True,
+        default=None,
+    )
+    photo_thumb = models.ImageField(
+        storage=BlobShadowFileSystemStorage(kind="photo"),
+        upload_to=settings.PHOTOS_DIRNAME,
+        blank=True,
+        default=None,
+    )
     name_from_draft = models.CharField("Full Name (from submission)", null=True, max_length=255, editable=False, help_text="Name as found in an Internet-Draft submission.")
 
     def __str__(self):
diff --git a/ietf/settings.py b/ietf/settings.py
index 125127ba15..faee42237c 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -183,6 +183,12 @@
 # Server-side static.ietf.org URL (used in pdfized)
 STATIC_IETF_ORG_INTERNAL = STATIC_IETF_ORG
 
+ENABLE_BLOBSTORAGE = True
+
+BLOBSTORAGE_MAX_ATTEMPTS = 1
+BLOBSTORAGE_CONNECT_TIMEOUT = 2
+BLOBSTORAGE_READ_TIMEOUT = 2
+
 WSGI_APPLICATION = "ietf.wsgi.application"
 
 AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', )
@@ -736,6 +742,38 @@ def skip_unreadable_post(record):
     "schedule_name": r"(?P<name>[A-Za-z0-9-:_]+)",
 }
 
+STORAGES: dict[str, Any] = {
+    "default": {"BACKEND": "django.core.files.storage.FileSystemStorage"},
+    "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"},
+}
+
+# settings_local will need to configure storages for these names
+MORE_STORAGE_NAMES: list[str] = [
+    "bofreq",
+    "charter",
+    "conflrev",
+    "active-draft",
+    "draft",
+    "slides",
+    "minutes",
+    "agenda",
+    "bluesheets",
+    "procmaterials",
+    "narrativeminutes",
+    "statement",
+    "statchg",
+    "liai-att",
+    "chatlog",
+    "polls",
+    "staging",
+    "bibxml-ids",
+    "indexes",
+    "floorplan",
+    "meetinghostlogo",
+    "photo",
+    "review",
+]
+
 # Override this in settings_local.py if needed
 # *_PATH variables ends with a slash/ .
 
diff --git a/ietf/settings_test.py b/ietf/settings_test.py
index 94ca22c71b..fe77152d42 100755
--- a/ietf/settings_test.py
+++ b/ietf/settings_test.py
@@ -14,7 +14,8 @@
 import shutil
 import tempfile
 from ietf.settings import *                                          # pyflakes:ignore
-from ietf.settings import TEST_CODE_COVERAGE_CHECKER
+from ietf.settings import STORAGES, TEST_CODE_COVERAGE_CHECKER, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
+import botocore.config
 import debug                            # pyflakes:ignore
 debug.debug = True
 
@@ -105,3 +106,30 @@ def tempdir_with_cleanup(**kwargs):
         'level': 'INFO',
     },
 }
+
+# Configure storages for the blob store - use env settings if present. See the --no-manage-blobstore test option.
+_blob_store_endpoint_url = os.environ.get("DATATRACKER_BLOB_STORE_ENDPOINT_URL", "http://blobstore:9000")
+_blob_store_access_key = os.environ.get("DATATRACKER_BLOB_STORE_ACCESS_KEY", "minio_root")
+_blob_store_secret_key = os.environ.get("DATATRACKER_BLOB_STORE_SECRET_KEY", "minio_pass")
+_blob_store_bucket_prefix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "test-")
+_blob_store_enable_profiling = (
+    os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true"
+)
+for storagename in MORE_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
+        "OPTIONS": dict(
+            endpoint_url=_blob_store_endpoint_url,
+            access_key=_blob_store_access_key,
+            secret_key=_blob_store_secret_key,
+            security_token=None,
+            client_config=botocore.config.Config(
+                signature_version="s3v4",
+                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
+                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
+                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
+            ),
+            bucket_name=f"{_blob_store_bucket_prefix}{storagename}",
+            ietf_log_blob_timing=_blob_store_enable_profiling,
+        ),
+    }
diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index 6a56839177..9a993480cd 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -31,6 +31,7 @@
                                 ReviewFactory, WgRfcFactory)
 from ietf.doc.models import ( Document, DocEvent, State,
     BallotPositionDocEvent, DocumentAuthor, SubmissionDocEvent )
+from ietf.doc.storage_utils import exists_in_storage, retrieve_str, store_str
 from ietf.doc.utils import create_ballot_if_not_open, can_edit_docextresources, update_action_holders
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.group.models import Group
@@ -53,6 +54,7 @@
 from ietf.utils import tool_version
 from ietf.utils.accesstoken import generate_access_token
 from ietf.utils.mail import outbox, get_payload_text
+from ietf.utils.test_runner import TestBlobstoreManager
 from ietf.utils.test_utils import login_testing_unauthorized, TestCase
 from ietf.utils.timezone import date_today
 from ietf.utils.draft import PlaintextDraft
@@ -355,6 +357,7 @@ def verify_bibxml_ids_creation(self, draft):
 
     def submit_new_wg(self, formats):
         # submit new -> supply submitter info -> approve
+        TestBlobstoreManager().emptyTestBlobstores()
         GroupFactory(type_id='wg',acronym='ames')
         mars = GroupFactory(type_id='wg', acronym='mars')
         RoleFactory(name_id='chair', group=mars, person__user__username='marschairman')
@@ -428,6 +431,13 @@ def submit_new_wg(self, formats):
         self.assertTrue(draft.latest_event(type="added_suggested_replaces"))
         self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))))
         self.assertTrue(os.path.exists(os.path.join(self.repository_dir, "%s-%s.txt" % (name, rev))))
+        check_ext = ["xml", "txt", "html"] if "xml" in formats else ["txt"]
+        for ext in check_ext:
+            basename=f"{name}-{rev}.{ext}"
+            extname=f"{ext}/{basename}"
+            self.assertFalse(exists_in_storage("staging", basename))
+            self.assertTrue(exists_in_storage("active-draft", extname))
+            self.assertTrue(exists_in_storage("draft", extname))            
         self.assertEqual(draft.type_id, "draft")
         self.assertEqual(draft.stream_id, "ietf")
         self.assertTrue(draft.expires >= timezone.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE - 1))
@@ -535,6 +545,7 @@ def test_submit_new_wg_as_author_bad_submitter(self):
 
     def submit_new_concluded_wg_as_author(self, group_state_id='conclude'):
         """A new concluded WG submission by a logged-in author needs AD approval"""
+        TestBlobstoreManager().emptyTestBlobstores()
         mars = GroupFactory(type_id='wg', acronym='mars', state_id=group_state_id)
         draft = WgDraftFactory(group=mars)
         setup_default_community_list_for_group(draft.group)
@@ -580,6 +591,7 @@ def test_submit_new_wg_with_extresources(self):
 
     def submit_existing(self, formats, change_authors=True, group_type='wg', stream_type='ietf'):
         # submit new revision of existing -> supply submitter info -> prev authors confirm
+        TestBlobstoreManager().emptyTestBlobstores()
 
         def _assert_authors_are_action_holders(draft, expect=True):
             for author in draft.authors():
@@ -771,6 +783,13 @@ def inspect_docevents(docevents, event_delta, event_type, be_in_desc, by_name):
         self.assertTrue(os.path.exists(os.path.join(self.archive_dir, "%s-%s.txt" % (name, old_rev))))
         self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))))
         self.assertTrue(os.path.exists(os.path.join(self.repository_dir, "%s-%s.txt" % (name, rev))))
+        check_ext = ["xml", "txt", "html"] if "xml" in formats else ["txt"]
+        for ext in check_ext:
+            basename=f"{name}-{rev}.{ext}"
+            extname=f"{ext}/{basename}"
+            self.assertFalse(exists_in_storage("staging", basename))
+            self.assertTrue(exists_in_storage("active-draft", extname))
+            self.assertTrue(exists_in_storage("draft", extname))  
         self.assertEqual(draft.type_id, "draft")
         if stream_type == 'ietf':
             self.assertEqual(draft.stream_id, "ietf")
@@ -909,6 +928,7 @@ def test_submit_existing_iab_with_extresources(self):
 
     def submit_new_individual(self, formats):
         # submit new -> supply submitter info -> confirm
+        TestBlobstoreManager().emptyTestBlobstores()
 
         name = "draft-authorname-testing-tests"
         rev = "00"
@@ -971,7 +991,13 @@ def submit_new_individual(self, formats):
                 self.assertTrue(variant_path.samefile(variant_ftp_path))
                 variant_all_archive_path = Path(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR) / variant_path.name
                 self.assertTrue(variant_path.samefile(variant_all_archive_path))
-
+        check_ext = ["xml", "txt", "html"] if "xml" in formats else ["txt"]
+        for ext in check_ext:
+            basename=f"{name}-{rev}.{ext}"
+            extname=f"{ext}/{basename}"
+            self.assertFalse(exists_in_storage("staging", basename))
+            self.assertTrue(exists_in_storage("active-draft", extname))
+            self.assertTrue(exists_in_storage("draft", extname))  
 
 
     def test_submit_new_individual_txt(self):
@@ -988,6 +1014,7 @@ def test_submit_new_individual_txt_xml(self):
         self.submit_new_individual(["txt", "xml"])
 
     def submit_new_draft_no_org_or_address(self, formats):
+        TestBlobstoreManager().emptyTestBlobstores()
         name = 'draft-testing-no-org-or-address'
 
         author = PersonFactory()
@@ -1078,6 +1105,7 @@ def _assert_extresource_change_event(self, doc, is_present=True):
             self.assertIsNone(event, 'External resource change event was unexpectedly created')
 
     def submit_new_draft_with_extresources(self, group):
+        TestBlobstoreManager().emptyTestBlobstores()
         name = 'draft-testing-with-extresources'
 
         status_url, author = self.do_submission(name, rev='00', group=group)
@@ -1107,6 +1135,7 @@ def test_submit_new_individual_with_extresources(self):
 
     def submit_new_individual_logged_in(self, formats):
         # submit new -> supply submitter info -> done
+        TestBlobstoreManager().emptyTestBlobstores()
 
         name = "draft-authorname-testing-logged-in"
         rev = "00"
@@ -1250,6 +1279,7 @@ def submit_existing_with_extresources(self, group_type, stream_type='ietf'):
         
         Unlike some other tests in this module, does not confirm draft if this would be required.
         """
+        TestBlobstoreManager().emptyTestBlobstores()
         orig_draft: Document = DocumentFactory(  # type: ignore[annotation-unchecked]
             type_id='draft',
             group=GroupFactory(type_id=group_type) if group_type else None,
@@ -1290,6 +1320,7 @@ def test_submit_update_individual_with_extresources(self):
 
     def submit_new_individual_replacing_wg(self, logged_in=False, group_state_id='active', notify_ad=False):
         """Chair of an active WG should be notified if individual draft is proposed to replace a WG draft"""
+        TestBlobstoreManager().emptyTestBlobstores()
         name = "draft-authorname-testing-tests"
         rev = "00"
         group = None
@@ -1416,6 +1447,7 @@ def test_cancel_submission(self):
         # cancel
         r = self.client.post(status_url, dict(action=action))
         self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))))
+        self.assertFalse(exists_in_storage("staging",f"{name}-{rev}.txt"))
 
     def test_edit_submission_and_force_post(self):
         # submit -> edit
@@ -1605,16 +1637,21 @@ def test_submit_all_file_types(self):
         self.assertEqual(Submission.objects.filter(name=name).count(), 1)
 
         self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))))
+        self.assertTrue(exists_in_storage("staging",f"{name}-{rev}.txt"))
         fd = io.open(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev)))
         txt_contents = fd.read()
         fd.close()
         self.assertTrue(name in txt_contents)
         self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))))
+        self.assertTrue(exists_in_storage("staging",f"{name}-{rev}.txt"))
         fd = io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev)))
         xml_contents = fd.read()
         fd.close()
         self.assertTrue(name in xml_contents)
         self.assertTrue('<?xml version="1.0" encoding="UTF-8"?>' in xml_contents)
+        xml_contents = retrieve_str("staging", f"{name}-{rev}.xml")
+        self.assertTrue(name in xml_contents)
+        self.assertTrue('<?xml version="1.0" encoding="UTF-8"?>' in xml_contents)       
 
     def test_expire_submissions(self):
         s = Submission.objects.create(name="draft-ietf-mars-foo",
@@ -1901,6 +1938,7 @@ def do_wg_approval_auth_test(self, state, chair_can_approve=False):
         
         Assumes approval allowed by AD and secretary and, optionally, chair of WG
         """
+        TestBlobstoreManager().emptyTestBlobstores()
         class _SubmissionFactory:
             """Helper class to generate fresh submissions"""
             def __init__(self, author, state):
@@ -2750,6 +2788,7 @@ class AsyncSubmissionTests(BaseSubmitTestCase):
     """Tests of async submission-related tasks"""
     def test_process_and_accept_uploaded_submission(self):
         """process_and_accept_uploaded_submission should properly process a submission"""
+        TestBlobstoreManager().emptyTestBlobstores()
         _today = date_today()
         xml, author = submission_file('draft-somebody-test-00', 'draft-somebody-test-00.xml', None, 'test_submission.xml')
         xml_data = xml.read()
@@ -2765,10 +2804,13 @@ def test_process_and_accept_uploaded_submission(self):
         xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
         with xml_path.open('w') as f:
             f.write(xml_data)
+        store_str("staging", "draft-somebody-test-00.xml", xml_data)
         txt_path = xml_path.with_suffix('.txt')
         self.assertFalse(txt_path.exists())
         html_path = xml_path.with_suffix('.html')
         self.assertFalse(html_path.exists())
+        for ext in ["txt", "html"]:
+            self.assertFalse(exists_in_storage("staging",f"draft-somebody-test-00.{ext}"))
         process_and_accept_uploaded_submission(submission)
 
         submission = Submission.objects.get(pk=submission.pk)  # refresh
@@ -2784,6 +2826,8 @@ def test_process_and_accept_uploaded_submission(self):
         # at least test that these were created
         self.assertTrue(txt_path.exists())
         self.assertTrue(html_path.exists())
+        for ext in ["txt", "html"]:
+            self.assertTrue(exists_in_storage("staging", f"draft-somebody-test-00.{ext}"))
         self.assertEqual(submission.file_size, os.stat(txt_path).st_size)
         self.assertIn('Completed submission validation checks', submission.submissionevent_set.last().desc)
 
@@ -2798,6 +2842,7 @@ def test_process_and_accept_uploaded_submission_invalid(self):
         txt.close()
 
         # submitter is not an author
+        TestBlobstoreManager().emptyTestBlobstores()
         submitter = PersonFactory()
         submission = SubmissionFactory(
             name='draft-somebody-test',
@@ -2809,12 +2854,14 @@ def test_process_and_accept_uploaded_submission_invalid(self):
         xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
         with xml_path.open('w') as f:
             f.write(xml_data)
+        store_str("staging", "draft-somebody-test-00.xml", xml_data)
         process_and_accept_uploaded_submission(submission)
         submission = Submission.objects.get(pk=submission.pk)  # refresh
         self.assertEqual(submission.state_id, 'cancel')
         self.assertIn('not one of the document authors', submission.submissionevent_set.last().desc)
 
         # author has no email address in XML
+        TestBlobstoreManager().emptyTestBlobstores()
         submission = SubmissionFactory(
             name='draft-somebody-test',
             rev='00',
@@ -2825,12 +2872,14 @@ def test_process_and_accept_uploaded_submission_invalid(self):
         xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
         with xml_path.open('w') as f:
             f.write(re.sub(r'<email>.*</email>', '', xml_data))
+        store_str("staging", "draft-somebody-test-00.xml", re.sub(r'<email>.*</email>', '', xml_data))
         process_and_accept_uploaded_submission(submission)
         submission = Submission.objects.get(pk=submission.pk)  # refresh
         self.assertEqual(submission.state_id, 'cancel')
         self.assertIn('Email address not found for all authors', submission.submissionevent_set.last().desc)
 
         # no title
+        TestBlobstoreManager().emptyTestBlobstores()
         submission = SubmissionFactory(
             name='draft-somebody-test',
             rev='00',
@@ -2841,12 +2890,14 @@ def test_process_and_accept_uploaded_submission_invalid(self):
         xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
         with xml_path.open('w') as f:
             f.write(re.sub(r'<title>.*</title>', '<title></title>', xml_data))
+        store_str("staging", "draft-somebody-test-00.xml", re.sub(r'<title>.*</title>', '<title></title>', xml_data))
         process_and_accept_uploaded_submission(submission)
         submission = Submission.objects.get(pk=submission.pk)  # refresh
         self.assertEqual(submission.state_id, 'cancel')
         self.assertIn('Could not extract a valid title', submission.submissionevent_set.last().desc)
 
         # draft name mismatch
+        TestBlobstoreManager().emptyTestBlobstores()
         submission = SubmissionFactory(
             name='draft-different-name',
             rev='00',
@@ -2857,12 +2908,14 @@ def test_process_and_accept_uploaded_submission_invalid(self):
         xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-different-name-00.xml'
         with xml_path.open('w') as f:
             f.write(xml_data)
+        store_str("staging", "draft-different-name-00.xml", xml_data)
         process_and_accept_uploaded_submission(submission)
         submission = Submission.objects.get(pk=submission.pk)  # refresh
         self.assertEqual(submission.state_id, 'cancel')
         self.assertIn('Submission rejected: XML Internet-Draft filename', submission.submissionevent_set.last().desc)
 
         # rev mismatch
+        TestBlobstoreManager().emptyTestBlobstores()
         submission = SubmissionFactory(
             name='draft-somebody-test',
             rev='01',
@@ -2873,12 +2926,14 @@ def test_process_and_accept_uploaded_submission_invalid(self):
         xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-01.xml'
         with xml_path.open('w') as f:
             f.write(xml_data)
+        store_str("staging", "draft-somebody-test-01.xml", xml_data)
         process_and_accept_uploaded_submission(submission)
         submission = Submission.objects.get(pk=submission.pk)  # refresh
         self.assertEqual(submission.state_id, 'cancel')
         self.assertIn('Submission rejected: XML Internet-Draft revision', submission.submissionevent_set.last().desc)
 
         # not xml
+        TestBlobstoreManager().emptyTestBlobstores()
         submission = SubmissionFactory(
             name='draft-somebody-test',
             rev='00',
@@ -2889,12 +2944,14 @@ def test_process_and_accept_uploaded_submission_invalid(self):
         txt_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.txt'
         with txt_path.open('w') as f:
             f.write(txt_data)
+        store_str("staging", "draft-somebody-test-00.txt", txt_data)
         process_and_accept_uploaded_submission(submission)
         submission = Submission.objects.get(pk=submission.pk)  # refresh
         self.assertEqual(submission.state_id, 'cancel')
         self.assertIn('Only XML Internet-Draft submissions', submission.submissionevent_set.last().desc)
 
         # wrong state
+        TestBlobstoreManager().emptyTestBlobstores()
         submission = SubmissionFactory(
             name='draft-somebody-test',
             rev='00',
@@ -2903,8 +2960,9 @@ def test_process_and_accept_uploaded_submission_invalid(self):
             state_id='uploaded',
         )
         xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
-        with xml_path.open('w') as f:
+        with xml_path.open('w') as f: # Why is this state being written if the thing that uses it is mocked out?
             f.write(xml_data)
+        store_str("staging", "draft-somebody-test-00.xml", xml_data)
         with mock.patch('ietf.submit.utils.process_submission_xml') as mock_proc_xml:
             process_and_accept_uploaded_submission(submission)
         submission = Submission.objects.get(pk=submission.pk)  # refresh
@@ -2912,6 +2970,7 @@ def test_process_and_accept_uploaded_submission_invalid(self):
         self.assertEqual(submission.state_id, 'uploaded', 'State should not be changed')
 
         # failed checker
+        TestBlobstoreManager().emptyTestBlobstores()
         submission = SubmissionFactory(
             name='draft-somebody-test',
             rev='00',
@@ -2922,6 +2981,7 @@ def test_process_and_accept_uploaded_submission_invalid(self):
         xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
         with xml_path.open('w') as f:
             f.write(xml_data)
+        store_str("staging", "draft-somebody-test-00.xml", xml_data)
         with mock.patch(
                 'ietf.submit.utils.apply_checkers',
                 side_effect = lambda _, __: submission.checks.create(
@@ -2958,6 +3018,7 @@ def test_process_and_accept_uploaded_submission_task_ignores_invalid_id(self, mo
         self.assertEqual(mock_method.call_count, 0)
 
     def test_process_submission_xml(self):
+        TestBlobstoreManager().emptyTestBlobstores()
         xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / "draft-somebody-test-00.xml"
         xml, _ = submission_file(
             "draft-somebody-test-00",
@@ -2968,6 +3029,7 @@ def test_process_submission_xml(self):
         )
         xml_contents = xml.read()
         xml_path.write_text(xml_contents)
+        store_str("staging", "draft-somebody-test-00.xml", xml_contents)
         output = process_submission_xml("draft-somebody-test", "00")
         self.assertEqual(output["filename"], "draft-somebody-test")
         self.assertEqual(output["rev"], "00")
@@ -2983,23 +3045,32 @@ def test_process_submission_xml(self):
         self.assertEqual(output["xml_version"], "3")
 
         # Should behave on missing or partial <date> elements
+        TestBlobstoreManager().emptyTestBlobstores()
         xml_path.write_text(re.sub(r"<date.+>", "", xml_contents))  # strip <date...> entirely
+        store_str("staging", "draft-somebody-test-00.xml", re.sub(r"<date.+>", "", xml_contents))
         output = process_submission_xml("draft-somebody-test", "00")
         self.assertEqual(output["document_date"], None)
 
+        TestBlobstoreManager().emptyTestBlobstores()
         xml_path.write_text(re.sub(r"<date year=.+ month", "<date month", xml_contents))  # remove year
+        store_str("staging", "draft-somebody-test-00.xml", re.sub(r"<date year=.+ month", "<date month", xml_contents))
         output = process_submission_xml("draft-somebody-test", "00")
         self.assertEqual(output["document_date"], date_today())
 
+        TestBlobstoreManager().emptyTestBlobstores()
         xml_path.write_text(re.sub(r"(<date.+) month=.+day=(.+>)", r"\1 day=\2", xml_contents))  # remove month
+        store_str("staging", "draft-somebody-test-00.xml", re.sub(r"(<date.+) month=.+day=(.+>)", r"\1 day=\2", xml_contents))
         output = process_submission_xml("draft-somebody-test", "00")
         self.assertEqual(output["document_date"], date_today())
 
+        TestBlobstoreManager().emptyTestBlobstores()
         xml_path.write_text(re.sub(r"<date(.+) day=.+>", r"<date\1>", xml_contents))  # remove day
+        store_str("staging", "draft-somebody-test-00.xml", re.sub(r"<date(.+) day=.+>", r"<date\1>", xml_contents))
         output = process_submission_xml("draft-somebody-test", "00")
         self.assertEqual(output["document_date"], date_today())
 
         # name mismatch
+        TestBlobstoreManager().emptyTestBlobstores()
         xml, _ = submission_file(
             "draft-somebody-wrong-name-00",  # name that appears in the file
             "draft-somebody-test-00.xml",
@@ -3008,10 +3079,13 @@ def test_process_submission_xml(self):
             title="Correct Draft Title",
         )
         xml_path.write_text(xml.read())
+        xml.seek(0)
+        store_str("staging", "draft-somebody-test-00.xml", xml.read())
         with self.assertRaisesMessage(SubmissionError, "disagrees with submission filename"):
             process_submission_xml("draft-somebody-test", "00")
 
         # rev mismatch
+        TestBlobstoreManager().emptyTestBlobstores()
         xml, _ = submission_file(
             "draft-somebody-test-01",  # name that appears in the file
             "draft-somebody-test-00.xml",
@@ -3020,10 +3094,13 @@ def test_process_submission_xml(self):
             title="Correct Draft Title",
         )
         xml_path.write_text(xml.read())
+        xml.seek(0)
+        store_str("staging", "draft-somebody-test-00.xml", xml.read())
         with self.assertRaisesMessage(SubmissionError, "disagrees with submission revision"):
             process_submission_xml("draft-somebody-test", "00")
 
         # missing title
+        TestBlobstoreManager().emptyTestBlobstores()
         xml, _ = submission_file(
             "draft-somebody-test-00",  # name that appears in the file
             "draft-somebody-test-00.xml",
@@ -3032,10 +3109,13 @@ def test_process_submission_xml(self):
             title="",
         )
         xml_path.write_text(xml.read())
+        xml.seek(0)
+        store_str("staging", "draft-somebody-test-00.xml", xml.read())
         with self.assertRaisesMessage(SubmissionError, "Could not extract a valid title"):
             process_submission_xml("draft-somebody-test", "00")
 
     def test_process_submission_text(self):
+        TestBlobstoreManager().emptyTestBlobstores()
         txt_path = Path(settings.IDSUBMIT_STAGING_PATH) / "draft-somebody-test-00.txt"
         txt, _ = submission_file(
             "draft-somebody-test-00",
@@ -3045,6 +3125,8 @@ def test_process_submission_text(self):
             title="Correct Draft Title",
         )
         txt_path.write_text(txt.read())
+        txt.seek(0)
+        store_str("staging", "draft-somebody-test-00.txt", txt.read())
         output = process_submission_text("draft-somebody-test", "00")
         self.assertEqual(output["filename"], "draft-somebody-test")
         self.assertEqual(output["rev"], "00")
@@ -3060,6 +3142,7 @@ def test_process_submission_text(self):
         self.assertIsNone(output["xml_version"])
 
         # name mismatch
+        TestBlobstoreManager().emptyTestBlobstores()
         txt, _ = submission_file(
             "draft-somebody-wrong-name-00",  # name that appears in the file
             "draft-somebody-test-00.txt",
@@ -3069,11 +3152,14 @@ def test_process_submission_text(self):
         )
         with txt_path.open('w') as fd:
             fd.write(txt.read())
+        txt.seek(0)
+        store_str("staging", "draft-somebody-test-00.txt", txt.read())
         txt.close()
         with self.assertRaisesMessage(SubmissionError, 'disagrees with submission filename'):
             process_submission_text("draft-somebody-test", "00")
 
         # rev mismatch
+        TestBlobstoreManager().emptyTestBlobstores()
         txt, _ = submission_file(
             "draft-somebody-test-01",  # name that appears in the file
             "draft-somebody-test-00.txt",
@@ -3083,6 +3169,8 @@ def test_process_submission_text(self):
         )
         with txt_path.open('w') as fd:
             fd.write(txt.read())
+        txt.seek(0)
+        store_str("staging", "draft-somebody-test-00.txt", txt.read())
         txt.close()
         with self.assertRaisesMessage(SubmissionError, 'disagrees with submission revision'):
             process_submission_text("draft-somebody-test", "00")
@@ -3221,6 +3309,7 @@ def test_find_submission_filenames(self):
         path = Path(self.staging_dir)
         for ext in ['txt', 'xml', 'pdf', 'md']:
             (path / f'{draft.name}-{draft.rev}.{ext}').touch()
+            store_str("staging", f"{draft.name}-{draft.rev}.{ext}", "")
         files = find_submission_filenames(draft)
         self.assertCountEqual(
             files,
@@ -3280,6 +3369,7 @@ def test_validate_submission_rev(self):
         new_wg_doc = WgDraftFactory(rev='01', relations=[('replaces',old_wg_doc)])
         path = Path(self.archive_dir) / f'{new_wg_doc.name}-{new_wg_doc.rev}.txt'
         path.touch()
+        store_str("staging", f"{new_wg_doc.name}-{new_wg_doc.rev}.txt", "")
 
         bad_revs = (None, '', '2', 'aa', '00', '01', '100', '002', u'öö')
         for rev in bad_revs:
diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index f19f2384c8..3e06f45c54 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -36,6 +36,7 @@
     DocumentAuthor, AddedMessageEvent )
 from ietf.doc.models import NewRevisionDocEvent
 from ietf.doc.models import RelatedDocument, DocRelationshipName, DocExtResource
+from ietf.doc.storage_utils import remove_from_storage, retrieve_bytes, store_bytes, store_file, store_str
 from ietf.doc.utils import (add_state_change_event, rebuild_reference_relations,
     set_replaces_for_document, prettify_std_name, update_doc_extresources, 
     can_edit_docextresources, update_documentauthors, update_action_holders,
@@ -455,6 +456,7 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc):
         from ietf.doc.expire import move_draft_files_to_archive
         move_draft_files_to_archive(draft, prev_rev)
 
+    submission.draft = draft
     move_files_to_repository(submission)
     submission.state = DraftSubmissionStateName.objects.get(slug="posted")
     log.log(f"{submission.name}: moved files")
@@ -488,7 +490,6 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc):
     if new_possibly_replaces:
         send_review_possibly_replaces_request(request, draft, submitter_info)
 
-    submission.draft = draft
     submission.save()
 
     create_submission_event(request, submission, approved_subm_desc)
@@ -498,6 +499,7 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc):
     ref_rev_file_name = os.path.join(os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids'), 'reference.I-D.%s-%s.xml' % (draft.name, draft.rev ))
     with io.open(ref_rev_file_name, "w", encoding='utf-8') as f:
         f.write(ref_text)
+    store_str("bibxml-ids", f"reference.I-D.{draft.name}-{draft.rev}.txt", ref_text) # TODO-BLOBSTORE verify with test
 
     log.log(f"{submission.name}: done")
     
@@ -666,6 +668,12 @@ def move_files_to_repository(submission):
             ftp_dest = Path(settings.FTP_DIR) / "internet-drafts" / dest.name
             os.link(dest, all_archive_dest)
             os.link(dest, ftp_dest)
+            # Shadow what's happening to the fs in the blobstores. When the stores become
+            # authoritative, the source and dest checks will need to apply to the stores instead.
+            content_bytes = retrieve_bytes("staging", fname)
+            store_bytes("active-draft", f"{ext}/{fname}", content_bytes)
+            submission.draft.store_bytes(f"{ext}/{fname}", content_bytes)
+            remove_from_storage("staging", fname)
         elif dest.exists():
             log.log("Intended to move '%s' to '%s', but found source missing while destination exists.")
         elif f".{ext}" in submission.file_types.split(','):
@@ -678,6 +686,7 @@ def remove_staging_files(name, rev):
     exts = [f'.{ext}' for ext in settings.IDSUBMIT_FILE_TYPES]
     for ext in exts:
         basename.with_suffix(ext).unlink(missing_ok=True)
+        remove_from_storage("staging", basename.with_suffix(ext).name, warn_if_missing=False)
 
 
 def remove_submission_files(submission):
@@ -766,6 +775,8 @@ def save_files(form):
             for chunk in f.chunks():
                 destination.write(chunk)
         log.log("saved file %s" % name)
+        f.seek(0)
+        store_file("staging", f"{form.filename}-{form.revision}.{ext}", f)
     return file_name
 
 
@@ -988,6 +999,10 @@ def render_missing_formats(submission):
                 xml_version,
             )
         )
+        # When the blobstores become autoritative - the guard at the
+        # containing if statement needs to be based on the store
+        with Path(txt_path).open("rb") as f:
+            store_file("staging", f"{submission.name}-{submission.rev}.txt", f)
 
     # --- Convert to html ---
     html_path = staging_path(submission.name, submission.rev, '.html')
@@ -1010,6 +1025,8 @@ def render_missing_formats(submission):
             xml_version,
         )
     )
+    with Path(html_path).open("rb") as f:
+        store_file("staging", f"{submission.name}-{submission.rev}.html", f)
 
 
 def accept_submission(submission: Submission, request: Optional[HttpRequest] = None, autopost=False):
@@ -1361,6 +1378,7 @@ def process_and_validate_submission(submission):
     except SubmissionError:
         raise  # pass SubmissionErrors up the stack
     except Exception as err:
+        # (this is a good point to just `raise err` when diagnosing Submission test failures)
         # convert other exceptions into SubmissionErrors
         log.log(f'Unexpected exception while processing submission {submission.pk}.')
         log.log(traceback.format_exc())
diff --git a/ietf/utils/storage.py b/ietf/utils/storage.py
index 0aa02cab86..9f41f3d50f 100644
--- a/ietf/utils/storage.py
+++ b/ietf/utils/storage.py
@@ -1,8 +1,56 @@
+# Copyright The IETF Trust 2020-2025, All Rights Reserved
+"""Django Storage classes"""
+from pathlib import Path
+
+from django.conf import settings
 from django.core.files.storage import FileSystemStorage
+from ietf.doc.storage_utils import store_file
+from .log import log
+
 
 class NoLocationMigrationFileSystemStorage(FileSystemStorage):
 
-    def deconstruct(obj):               # pylint: disable=no-self-argument
-        path, args, kwargs = FileSystemStorage.deconstruct(obj)
-        kwargs["location"] = None
-        return (path, args, kwargs)
+    def deconstruct(self):
+        path, args, kwargs = super().deconstruct()
+        kwargs["location"] = None  # don't record location in migrations
+        return path, args, kwargs
+
+
+class BlobShadowFileSystemStorage(NoLocationMigrationFileSystemStorage):
+    """FileSystemStorage that shadows writes to the blob store as well
+
+    Strips directories from the filename when naming the blob.
+    """
+
+    def __init__(
+        self,
+        *,  # disallow positional arguments
+        kind: str,
+        location=None,
+        base_url=None,
+        file_permissions_mode=None,
+        directory_permissions_mode=None,
+    ):
+        self.kind = kind
+        super().__init__(
+            location, base_url, file_permissions_mode, directory_permissions_mode
+        )
+
+    def save(self, name, content, max_length=None):
+        # Write content to the filesystem - this deals with chunks, etc...
+        saved_name = super().save(name, content, max_length)
+
+        if settings.ENABLE_BLOBSTORAGE:
+            # Retrieve the content and write to the blob store
+            blob_name = Path(saved_name).name  # strips path
+            try:
+                with self.open(saved_name, "rb") as f:
+                    store_file(self.kind, blob_name, f, allow_overwrite=True)
+            except Exception as err:
+                log(f"Failed to shadow {saved_name} at {self.kind}:{blob_name}: {err}")
+        return saved_name  # includes the path!
+
+    def deconstruct(self):
+        path, args, kwargs = super().deconstruct()
+        kwargs["kind"] = ""  # don't record "kind" in migrations
+        return path, args, kwargs
diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py
index 49d53e1e1d..3c89a2d01c 100644
--- a/ietf/utils/test_runner.py
+++ b/ietf/utils/test_runner.py
@@ -48,6 +48,8 @@
 import subprocess
 import tempfile
 import copy
+import boto3
+import botocore.config
 import factory.random
 import urllib3
 import warnings
@@ -85,6 +87,8 @@
 from ietf.utils.test_smtpserver import SMTPTestServerDriver
 from ietf.utils.test_utils import TestCase
 
+from mypy_boto3_s3.service_resource import Bucket
+
 
 loaded_templates = set()
 visited_urls = set()
@@ -722,9 +726,25 @@ def add_arguments(cls, parser):
         parser.add_argument('--rerun-until-failure',
             action='store_true', dest='rerun', default=False,
             help='Run the indicated tests in a loop until a failure occurs. ' )
-
-    def __init__(self, ignore_lower_coverage=False, skip_coverage=False, save_version_coverage=None, html_report=None, permit_mixed_migrations=None, show_logging=None, validate_html=None, validate_html_harder=None, rerun=None, **kwargs):
-        #
+        parser.add_argument('--no-manage-blobstore', action='store_false', dest='manage_blobstore',
+                            help='Disable creating/deleting test buckets in the blob store.'
+                                 'When this argument is used, a set of buckets with "test-" prefixed to their '
+                                 'names must already exist.')
+
+    def __init__(
+        self,
+        ignore_lower_coverage=False,
+        skip_coverage=False,
+        save_version_coverage=None,
+        html_report=None,
+        permit_mixed_migrations=None,
+        show_logging=None,
+        validate_html=None,
+        validate_html_harder=None,
+        rerun=None,
+        manage_blobstore=True,
+        **kwargs
+    ):    #
         self.ignore_lower_coverage = ignore_lower_coverage
         self.check_coverage = not skip_coverage
         self.save_version_coverage = save_version_coverage
@@ -752,6 +772,8 @@ def __init__(self, ignore_lower_coverage=False, skip_coverage=False, save_versio
         # contains parent classes to later subclasses, the parent classes will determine the ordering, so use the most
         # specific classes necessary to get the right ordering:
         self.reorder_by = (PyFlakesTestCase, MyPyTest,) + self.reorder_by + (StaticLiveServerTestCase, TemplateTagTest, CoverageTest,)
+        #self.buckets = set()
+        self.blobstoremanager = TestBlobstoreManager() if manage_blobstore else None
 
     def setup_test_environment(self, **kwargs):
         global template_coverage_collection
@@ -936,6 +958,9 @@ def setup_test_environment(self, **kwargs):
                 print(" (extra pedantically)")
                 self.vnu = start_vnu_server()
 
+        if self.blobstoremanager is not None:
+            self.blobstoremanager.createTestBlobstores()
+        
         super(IetfTestRunner, self).setup_test_environment(**kwargs)
 
     def teardown_test_environment(self, **kwargs):
@@ -966,6 +991,9 @@ def teardown_test_environment(self, **kwargs):
             if self.vnu:
                 self.vnu.terminate()
 
+        if self.blobstoremanager is not None:
+            self.blobstoremanager.destroyTestBlobstores()
+
         super(IetfTestRunner, self).teardown_test_environment(**kwargs)
 
     def validate(self, testcase):
@@ -1220,3 +1248,39 @@ def tearDown(self):
         for k, v in self.replaced_settings.items():
             setattr(settings, k, v)
         super().tearDown()
+
+class TestBlobstoreManager():
+    # N.B. buckets and blobstore are intentional Class-level attributes
+    buckets: set[Bucket] = set()
+
+    blobstore = boto3.resource("s3",
+        endpoint_url="http://blobstore:9000",
+        aws_access_key_id="minio_root",
+        aws_secret_access_key="minio_pass",
+        aws_session_token=None,
+        config = botocore.config.Config(signature_version="s3v4"),
+        #config=botocore.config.Config(signature_version=botocore.UNSIGNED),
+        verify=False
+    )
+
+    def createTestBlobstores(self):
+        for storagename in settings.MORE_STORAGE_NAMES:
+            bucketname = f"test-{storagename}"
+            try:
+                bucket = self.blobstore.create_bucket(Bucket=bucketname)
+                self.buckets.add(bucket)
+            except self.blobstore.meta.client.exceptions.BucketAlreadyOwnedByYou:
+                bucket = self.blobstore.Bucket(bucketname)
+                self.buckets.add(bucket)
+
+    def destroyTestBlobstores(self):
+        self.emptyTestBlobstores(destroy=True)
+
+    def emptyTestBlobstores(self, destroy=False):
+        # debug.show('f"Asked to empty test blobstores with destroy={destroy}"')
+        for bucket in self.buckets:
+            bucket.objects.delete()
+            if destroy:
+                bucket.delete()
+        if destroy:
+            self.buckets = set()
diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index f266ffcd62..912607f466 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -6,7 +6,9 @@
 import json
 
 from ietf import __release_hash__
-from ietf.settings import *                                          # pyflakes:ignore
+from ietf.settings import *  # pyflakes:ignore
+from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
+import botocore.config
 
 
 def _multiline_to_list(s):
@@ -29,7 +31,7 @@ def _multiline_to_list(s):
 if _SECRET_KEY is not None:
     SECRET_KEY = _SECRET_KEY
 else:
-    raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set")    
+    raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set")
 
 _NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None)
 if _NOMCOM_APP_SECRET_B64 is not None:
@@ -41,7 +43,7 @@ def _multiline_to_list(s):
 if _IANA_SYNC_PASSWORD is not None:
     IANA_SYNC_PASSWORD = _IANA_SYNC_PASSWORD
 else:
-    raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set")    
+    raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set")
 
 _RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None)
 if _RFC_EDITOR_SYNC_PASSWORD is not None:
@@ -59,25 +61,25 @@ def _multiline_to_list(s):
 if _GITHUB_BACKUP_API_KEY is not None:
     GITHUB_BACKUP_API_KEY = _GITHUB_BACKUP_API_KEY
 else:
-    raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set")    
+    raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set")
 
 _API_KEY_TYPE = os.environ.get("DATATRACKER_API_KEY_TYPE", None)
 if _API_KEY_TYPE is not None:
     API_KEY_TYPE = _API_KEY_TYPE
 else:
-    raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set")    
+    raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set")
 
 _API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None)
 if _API_PUBLIC_KEY_PEM_B64 is not None:
     API_PUBLIC_KEY_PEM = b64decode(_API_PUBLIC_KEY_PEM_B64)
 else:
-    raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set")    
+    raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set")
 
 _API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None)
 if _API_PRIVATE_KEY_PEM_B64 is not None:
     API_PRIVATE_KEY_PEM = b64decode(_API_PRIVATE_KEY_PEM_B64)
 else:
-    raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set")    
+    raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set")
 
 # Set DEBUG if DATATRACKER_DEBUG env var is the word "true"
 DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true"
@@ -102,7 +104,9 @@ def _multiline_to_list(s):
 # Configure persistent connections. A setting of 0 is Django's default.
 _conn_max_age = os.environ.get("DATATRACKER_DB_CONN_MAX_AGE", "0")
 # A string "none" means unlimited age.
-DATABASES["default"]["CONN_MAX_AGE"] = None if _conn_max_age.lower() == "none" else int(_conn_max_age)
+DATABASES["default"]["CONN_MAX_AGE"] = (
+    None if _conn_max_age.lower() == "none" else int(_conn_max_age)
+)
 # Enable connection health checks if DATATRACKER_DB_CONN_HEALTH_CHECK is the string "true"
 _conn_health_checks = bool(
     os.environ.get("DATATRACKER_DB_CONN_HEALTH_CHECKS", "false").lower() == "true"
@@ -114,9 +118,11 @@ def _multiline_to_list(s):
 if _admins_str is not None:
     ADMINS = [parseaddr(admin) for admin in _multiline_to_list(_admins_str)]
 else:
-    raise RuntimeError("DATATRACKER_ADMINS must be set")    
+    raise RuntimeError("DATATRACKER_ADMINS must be set")
 
-USING_DEBUG_EMAIL_SERVER = os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true"
+USING_DEBUG_EMAIL_SERVER = (
+    os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true"
+)
 EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost")
 EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025"))
 
@@ -126,7 +132,7 @@ def _multiline_to_list(s):
 CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format(
     host=os.environ.get("RABBITMQ_HOSTNAME", "dt-rabbitmq"),
     password=_celery_password,
-    queue=os.environ.get("RABBITMQ_QUEUE", "dt")
+    queue=os.environ.get("RABBITMQ_QUEUE", "dt"),
 )
 
 IANA_SYNC_USERNAME = "ietfsync"
@@ -140,10 +146,10 @@ def _multiline_to_list(s):
     raise RuntimeError("DATATRACKER_REGISTRATION_API_KEY must be set")
 STATS_REGISTRATION_ATTENDEES_JSON_URL = f"https://registration.ietf.org/{{number}}/attendees/?apikey={_registration_api_key}"
 
-#FIRST_CUTOFF_DAYS = 12
-#SECOND_CUTOFF_DAYS = 12
-#SUBMISSION_CUTOFF_DAYS = 26
-#SUBMISSION_CORRECTION_DAYS = 57
+# FIRST_CUTOFF_DAYS = 12
+# SECOND_CUTOFF_DAYS = 12
+# SUBMISSION_CUTOFF_DAYS = 26
+# SUBMISSION_CORRECTION_DAYS = 57
 MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26
 MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 54
 
@@ -155,7 +161,7 @@ def _multiline_to_list(s):
 if _MEETECHO_CLIENT_ID is not None and _MEETECHO_CLIENT_SECRET is not None:
     MEETECHO_API_CONFIG = {
         "api_base": os.environ.get(
-            "DATATRACKER_MEETECHO_API_BASE", 
+            "DATATRACKER_MEETECHO_API_BASE",
             "https://meetings.conf.meetecho.com/api/v1/",
         ),
         "client_id": _MEETECHO_CLIENT_ID,
@@ -173,7 +179,9 @@ def _multiline_to_list(s):
         raise RuntimeError(
             "Only one of DATATRACKER_APP_API_TOKENS_JSON and DATATRACKER_APP_API_TOKENS_JSON_B64 may be set"
         )
-    _APP_API_TOKENS_JSON = b64decode(os.environ.get("DATATRACKER_APP_API_TOKENS_JSON_B64"))
+    _APP_API_TOKENS_JSON = b64decode(
+        os.environ.get("DATATRACKER_APP_API_TOKENS_JSON_B64")
+    )
 else:
     _APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None)
 
@@ -189,7 +197,9 @@ def _multiline_to_list(s):
 
 # Leave DATATRACKER_MATOMO_SITE_ID unset to disable Matomo reporting
 if "DATATRACKER_MATOMO_SITE_ID" in os.environ:
-    MATOMO_DOMAIN_PATH = os.environ.get("DATATRACKER_MATOMO_DOMAIN_PATH", "analytics.ietf.org")
+    MATOMO_DOMAIN_PATH = os.environ.get(
+        "DATATRACKER_MATOMO_DOMAIN_PATH", "analytics.ietf.org"
+    )
     MATOMO_SITE_ID = os.environ.get("DATATRACKER_MATOMO_SITE_ID")
     MATOMO_DISABLE_COOKIES = True
 
@@ -197,9 +207,13 @@ def _multiline_to_list(s):
 _SCOUT_KEY = os.environ.get("DATATRACKER_SCOUT_KEY", None)
 if _SCOUT_KEY is not None:
     if SERVER_MODE == "production":
-        PROD_PRE_APPS = ["scout_apm.django", ]
+        PROD_PRE_APPS = [
+            "scout_apm.django",
+        ]
     else:
-        DEV_PRE_APPS = ["scout_apm.django", ]
+        DEV_PRE_APPS = [
+            "scout_apm.django",
+        ]
     SCOUT_MONITOR = True
     SCOUT_KEY = _SCOUT_KEY
     SCOUT_NAME = os.environ.get("DATATRACKER_SCOUT_NAME", "Datatracker")
@@ -216,16 +230,17 @@ def _multiline_to_list(s):
 STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None)
 if STATIC_URL is None:
     from ietf import __version__
+
     STATIC_URL = f"https://static.ietf.org/dt/{__version__}/"
 
 # Set these to the same as "production" in settings.py, whether production mode or not
 MEDIA_ROOT = "/a/www/www6s/lib/dt/media/"
-MEDIA_URL  = "https://www.ietf.org/lib/dt/media/"
+MEDIA_URL = "https://www.ietf.org/lib/dt/media/"
 PHOTOS_DIRNAME = "photo"
 PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME
 
 # Normally only set for debug, but needed until we have a real FS
-DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json')
+DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, "static/dist-neue/manifest.json")
 
 # Binaries that are different in the docker image
 DE_GFM_BINARY = "/usr/local/bin/de-gfm"
@@ -235,6 +250,7 @@ def _multiline_to_list(s):
 MEMCACHED_HOST = os.environ.get("DT_MEMCACHED_SERVICE_HOST", "127.0.0.1")
 MEMCACHED_PORT = os.environ.get("DT_MEMCACHED_SERVICE_PORT", "11211")
 from ietf import __version__
+
 CACHES = {
     "default": {
         "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
@@ -285,3 +301,46 @@ def _multiline_to_list(s):
 
 # Console logs as JSON instead of plain when running in k8s
 LOGGING["handlers"]["console"]["formatter"] = "json"
+
+# Configure storages for the blob store
+_blob_store_endpoint_url = os.environ.get("DATATRACKER_BLOB_STORE_ENDPOINT_URL")
+_blob_store_access_key = os.environ.get("DATATRACKER_BLOB_STORE_ACCESS_KEY")
+_blob_store_secret_key = os.environ.get("DATATRACKER_BLOB_STORE_SECRET_KEY")
+if None in (_blob_store_endpoint_url, _blob_store_access_key, _blob_store_secret_key):
+    raise RuntimeError(
+        "All of DATATRACKER_BLOB_STORE_ENDPOINT_URL, DATATRACKER_BLOB_STORE_ACCESS_KEY, "
+        "and DATATRACKER_BLOB_STORE_SECRET_KEY must be set"
+    )
+_blob_store_bucket_prefix = os.environ.get(
+    "DATATRACKER_BLOB_STORE_BUCKET_PREFIX", ""
+)
+_blob_store_enable_profiling = (
+    os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true"
+)
+_blob_store_max_attempts = (
+    os.environ.get("DATATRACKER_BLOB_STORE_MAX_ATTEMPTS", BLOBSTORAGE_MAX_ATTEMPTS)
+)
+_blob_store_connect_timeout = (
+    os.environ.get("DATATRACKER_BLOB_STORE_CONNECT_TIMEOUT", BLOBSTORAGE_CONNECT_TIMEOUT)
+)
+_blob_store_read_timeout = (
+    os.environ.get("DATATRACKER_BLOB_STORE_READ_TIMEOUT", BLOBSTORAGE_READ_TIMEOUT)
+)
+for storage_name in MORE_STORAGE_NAMES:
+    STORAGES[storage_name] = {
+        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
+        "OPTIONS": dict(
+            endpoint_url=_blob_store_endpoint_url,
+            access_key=_blob_store_access_key,
+            secret_key=_blob_store_secret_key,
+            security_token=None,
+            client_config=botocore.config.Config(
+                signature_version="s3v4",
+                connect_timeout=_blob_store_connect_timeout,
+                read_timeout=_blob_store_read_timeout,
+                retries={"total_max_attempts": _blob_store_max_attempts},
+            ),
+            bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(),
+            ietf_log_blob_timing=_blob_store_enable_profiling,
+        ),
+    }
diff --git a/requirements.txt b/requirements.txt
index 073a6bfa0a..66a785e929 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,6 +6,9 @@ beautifulsoup4>=4.11.1    # Only used in tests
 bibtexparser>=1.2.0    # Only used in tests
 bleach>=6
 types-bleach>=6
+boto3>=1.35,<1.36
+boto3-stubs[s3]>=1.35,<1.36
+botocore>=1.35,<1.36
 celery>=5.2.6
 coverage>=4.5.4,<5.0    # Coverage 5.x moves from a json database to SQLite.  Moving to 5.x will require substantial rewrites in ietf.utils.test_runner and ietf.release.views
 defusedxml>=0.7.1    # for TastyPie when using xml; not a declared dependency
@@ -21,6 +24,7 @@ django-markup>=1.5    # Limited use - need to reconcile against direct use of ma
 django-oidc-provider==0.8.2    # 0.8.3 changes logout flow and claim return
 django-referrer-policy>=1.0
 django-simple-history>=3.0.0
+django-storages>=1.14.4
 django-stubs>=4.2.7,<5    # The django-stubs version used determines the the mypy version indicated below
 django-tastypie>=0.14.7,<0.15.0    # Version must be locked in sync with version of Django
 django-vite>=2.0.2,<3
@@ -75,7 +79,7 @@ tblib>=1.7.0    # So that the django test runner provides tracebacks
 tlds>=2022042700    # Used to teach bleach about which TLDs currently exist
 tqdm>=4.64.0
 Unidecode>=1.3.4
-urllib3>=2
+urllib3>=1.26,<2
 weasyprint>=59
 xml2rfc[pdf]>=3.23.0
 xym>=0.6,<1.0

From be7ad9acabca649cd77374e3aea574dfdcebc7d8 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 19 Feb 2025 21:42:07 -0400
Subject: [PATCH 260/656] ci: cast env vars to correct types (#8558)

---
 k8s/settings_local.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 912607f466..a0e197b7f5 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -317,13 +317,13 @@ def _multiline_to_list(s):
 _blob_store_enable_profiling = (
     os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true"
 )
-_blob_store_max_attempts = (
+_blob_store_max_attempts = int(
     os.environ.get("DATATRACKER_BLOB_STORE_MAX_ATTEMPTS", BLOBSTORAGE_MAX_ATTEMPTS)
 )
-_blob_store_connect_timeout = (
+_blob_store_connect_timeout = float(
     os.environ.get("DATATRACKER_BLOB_STORE_CONNECT_TIMEOUT", BLOBSTORAGE_CONNECT_TIMEOUT)
 )
-_blob_store_read_timeout = (
+_blob_store_read_timeout = float(
     os.environ.get("DATATRACKER_BLOB_STORE_READ_TIMEOUT", BLOBSTORAGE_READ_TIMEOUT)
 )
 for storage_name in MORE_STORAGE_NAMES:

From 2f8b9c3cfa157ed14d19b372c877eaed3beb6391 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 20 Feb 2025 13:12:43 -0400
Subject: [PATCH 261/656] fix: ignore exceptions from blobstore ops (#8565)

* fix: ignore exceptions from to blobstore ops

* fix: log repr(err) instead of just err
---
 ietf/doc/storage_utils.py | 67 +++++++++++++++++++++++++++------------
 ietf/utils/storage.py     |  6 ++--
 2 files changed, 49 insertions(+), 24 deletions(-)

diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py
index 4f0516339a..1bc2aa293c 100644
--- a/ietf/doc/storage_utils.py
+++ b/ietf/doc/storage_utils.py
@@ -8,6 +8,8 @@
 from django.core.files.base import ContentFile, File
 from django.core.files.storage import storages
 
+from ietf.utils.log import log
+
 
 # TODO-BLOBSTORE (Future, maybe after leaving 3.9) : add a return type
 def _get_storage(kind: str):
@@ -22,16 +24,22 @@ def _get_storage(kind: str):
 
 def exists_in_storage(kind: str, name: str) -> bool:
     if settings.ENABLE_BLOBSTORAGE:
-        store = _get_storage(kind)
-        return store.exists_in_storage(kind, name)
+        try:
+            store = _get_storage(kind)
+            return store.exists_in_storage(kind, name)
+        except Exception as err:
+            log(f"Blobstore Error: Failed to test existence of {kind}:{name}: {repr(err)}")
     else:
         return False
 
 
 def remove_from_storage(kind: str, name: str, warn_if_missing: bool = True) -> None:
     if settings.ENABLE_BLOBSTORAGE:
-        store = _get_storage(kind)
-        store.remove_from_storage(kind, name, warn_if_missing)
+        try:
+            store = _get_storage(kind)
+            store.remove_from_storage(kind, name, warn_if_missing)
+        except Exception as err:
+            log(f"Blobstore Error: Failed to remove {kind}:{name}: {repr(err)}")
     return None
 
 
@@ -46,8 +54,11 @@ def store_file(
 ) -> None:
     # debug.show('f"asked to store {name} into {kind}"')
     if settings.ENABLE_BLOBSTORAGE:
-        store = _get_storage(kind)
-        store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev)
+        try:
+            store = _get_storage(kind)
+            store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev)
+        except Exception as err:
+            log(f"Blobstore Error: Failed to store file {kind}:{name}: {repr(err)}")
     return None
 
 
@@ -60,7 +71,11 @@ def store_bytes(
     doc_rev: Optional[str] = None,
 ) -> None:
     if settings.ENABLE_BLOBSTORAGE:
-        store_file(kind, name, ContentFile(content), allow_overwrite)
+        try:
+            store_file(kind, name, ContentFile(content), allow_overwrite)
+        except Exception as err:
+            # n.b., not likely to get an exception here because store_file or store_bytes will catch it
+            log(f"Blobstore Error: Failed to store bytes to {kind}:{name}: {repr(err)}")
     return None
 
 
@@ -73,8 +88,12 @@ def store_str(
     doc_rev: Optional[str] = None,
 ) -> None:
     if settings.ENABLE_BLOBSTORAGE:
-        content_bytes = content.encode("utf-8")
-        store_bytes(kind, name, content_bytes, allow_overwrite)
+        try:
+            content_bytes = content.encode("utf-8")
+            store_bytes(kind, name, content_bytes, allow_overwrite)
+        except Exception as err:
+            # n.b., not likely to get an exception here because store_file or store_bytes will catch it
+            log(f"Blobstore Error: Failed to store string to {kind}:{name}: {repr(err)}")
     return None
 
 
@@ -82,22 +101,28 @@ def retrieve_bytes(kind: str, name: str) -> bytes:
     from ietf.doc.storage_backends import maybe_log_timing
     content = b""
     if settings.ENABLE_BLOBSTORAGE:
-        store = _get_storage(kind)
-        with store.open(name) as f:
-            with maybe_log_timing(
-                hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing,
-                "read",
-                bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "",
-                name=name,
-            ):
-                content = f.read()
+        try:
+            store = _get_storage(kind)
+            with store.open(name) as f:
+                with maybe_log_timing(
+                    hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing,
+                    "read",
+                    bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "",
+                    name=name,
+                ):
+                    content = f.read()
+        except Exception as err:
+            log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}")
     return content
 
 
 def retrieve_str(kind: str, name: str) -> str:
     content = ""
     if settings.ENABLE_BLOBSTORAGE:
-        content_bytes = retrieve_bytes(kind, name)
-        # TODO-BLOBSTORE: try to decode all the different ways doc.text() does
-        content = content_bytes.decode("utf-8")
+        try:
+            content_bytes = retrieve_bytes(kind, name)
+            # TODO-BLOBSTORE: try to decode all the different ways doc.text() does
+            content = content_bytes.decode("utf-8")
+        except Exception as err:
+            log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}")
     return content
diff --git a/ietf/utils/storage.py b/ietf/utils/storage.py
index 9f41f3d50f..42fcf884aa 100644
--- a/ietf/utils/storage.py
+++ b/ietf/utils/storage.py
@@ -41,13 +41,13 @@ def save(self, name, content, max_length=None):
         saved_name = super().save(name, content, max_length)
 
         if settings.ENABLE_BLOBSTORAGE:
-            # Retrieve the content and write to the blob store
-            blob_name = Path(saved_name).name  # strips path
             try:
+                # Retrieve the content and write to the blob store
+                blob_name = Path(saved_name).name  # strips path
                 with self.open(saved_name, "rb") as f:
                     store_file(self.kind, blob_name, f, allow_overwrite=True)
             except Exception as err:
-                log(f"Failed to shadow {saved_name} at {self.kind}:{blob_name}: {err}")
+                log(f"Blobstore Error: Failed to shadow {saved_name} at {self.kind}:{blob_name}: {repr(err)}")
         return saved_name  # includes the path!
 
     def deconstruct(self):

From aeba63bb41a02bab3f4ef043d5ac8d5e38e24b80 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 20 Feb 2025 13:13:44 -0600
Subject: [PATCH 262/656] chore: ensure proper return type (#8566)

* chore: ensure proper return type

* test: comment out (intentionally) failing assertion

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 ietf/doc/storage_utils.py       | 3 +--
 ietf/doc/tests_status_change.py | 5 +++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py
index 1bc2aa293c..012efc9071 100644
--- a/ietf/doc/storage_utils.py
+++ b/ietf/doc/storage_utils.py
@@ -29,8 +29,7 @@ def exists_in_storage(kind: str, name: str) -> bool:
             return store.exists_in_storage(kind, name)
         except Exception as err:
             log(f"Blobstore Error: Failed to test existence of {kind}:{name}: {repr(err)}")
-    else:
-        return False
+    return False
 
 
 def remove_from_storage(kind: str, name: str, warn_if_missing: bool = True) -> None:
diff --git a/ietf/doc/tests_status_change.py b/ietf/doc/tests_status_change.py
index cbdc1a049a..da1a4f1906 100644
--- a/ietf/doc/tests_status_change.py
+++ b/ietf/doc/tests_status_change.py
@@ -564,8 +564,9 @@ def test_initial_submission(self):
         ftp_filepath = Path(settings.FTP_DIR) / "status-changes" / basename
         self.assertFalse(filepath.exists())
         self.assertFalse(ftp_filepath.exists())
-        with self.assertRaises(FileNotFoundError):
-            retrieve_str("statchg",basename)
+        # TODO-BLOBSTORE: next assert is disabled because we currently suppress all exceptions 
+        # with self.assertRaises(FileNotFoundError):
+        #     retrieve_str("statchg",basename)
         r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1"))
         self.assertEqual(r.status_code,302)
         doc = Document.objects.get(name='status-change-imaginary-mid-review')

From 041fa83d21ce7d8320e09a2a08f9c0b7577f6fba Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 20 Feb 2025 16:45:22 -0400
Subject: [PATCH 263/656] chore: handle errors in app-configure-blobstore.py
 (#8567)

---
 docker/scripts/app-configure-blobstore.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py
index 7b5ce962eb..ae87bf1afe 100755
--- a/docker/scripts/app-configure-blobstore.py
+++ b/docker/scripts/app-configure-blobstore.py
@@ -2,6 +2,8 @@
 # Copyright The IETF Trust 2024, All Rights Reserved
 
 import boto3
+import botocore.config
+import botocore.exceptions
 import os
 import sys
 
@@ -16,13 +18,19 @@ def init_blobstore():
         aws_secret_access_key=os.environ.get("BLOB_STORE_SECRET_KEY", "minio_pass"),
         aws_session_token=None,
         config=botocore.config.Config(signature_version="s3v4"),
-        verify=False,
     )
     for bucketname in MORE_STORAGE_NAMES:
-        blobstore.create_bucket(
-            Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip()
-        )
-
+        try:
+            blobstore.create_bucket(
+                Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip()
+            )
+        except botocore.exceptions.ClientError as err:
+            if err.response["Error"]["Code"] == "BucketAlreadyExists":
+                print(f"Bucket {bucketname} already exists")
+            else:
+                print(f"Error creating {bucketname}: {err.response['Error']['Code']}")
+        else:
+            print(f"Bucket {bucketname} created")
 
 if __name__ == "__main__":
     sys.exit(init_blobstore())

From fb310e5ce209ca3ce1b5e557e8beb795d9b75a02 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 21 Feb 2025 11:49:16 -0400
Subject: [PATCH 264/656] feat: useful error when submission has inconsistent
 date (#8576)

* chore: handle errors in app-configure-blobstore.py

* feat: sensible error for inconsistent <date>
---
 ietf/submit/utils.py   |  9 +++++++--
 ietf/utils/xmldraft.py | 15 ++++++++++++++-
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index 3e06f45c54..61274c0116 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -58,7 +58,7 @@
 from ietf.utils.mail import is_valid_email
 from ietf.utils.text import parse_unicode, normalize_text
 from ietf.utils.timezone import date_today
-from ietf.utils.xmldraft import XMLDraft
+from ietf.utils.xmldraft import InvalidMetadataError, XMLDraft
 from ietf.person.name import unidecode_name
 
 
@@ -1201,6 +1201,11 @@ def process_submission_xml(filename, revision):
     if not title:
         raise SubmissionError("Could not extract a valid title from the XML")
     
+    try:
+        document_date = xml_draft.get_creation_date()
+    except InvalidMetadataError as err:
+        raise SubmissionError(str(err)) from err
+
     return {
         "filename": xml_draft.filename,
         "rev": xml_draft.revision,
@@ -1210,7 +1215,7 @@ def process_submission_xml(filename, revision):
             for auth in xml_draft.get_author_list()
         ],
         "abstract": None,  # not supported from XML
-        "document_date": xml_draft.get_creation_date(),
+        "document_date": document_date,
         "pages": None,  # not supported from XML
         "words": None,  # not supported from XML
         "first_two_pages": None,  # not supported from XML
diff --git a/ietf/utils/xmldraft.py b/ietf/utils/xmldraft.py
index c39c4d0a06..73baf917d8 100644
--- a/ietf/utils/xmldraft.py
+++ b/ietf/utils/xmldraft.py
@@ -159,7 +159,16 @@ def parse_creation_date(date_elt):
                 day = today.day
             else:
                 day = 15
-        return datetime.date(year, month, day)
+        try:
+            creation_date = datetime.date(year, month, day)
+        except Exception:
+            raise InvalidMetadataError(
+                "The <date> element in the <front> section specified an incomplete date "
+                "that was not consistent with today's date. If you specify only a year, "
+                "it must be the four-digit current year. To use today's date, omit the "
+                "date tag or use <date/>."
+            )
+        return creation_date
 
     def get_creation_date(self):
         return self.parse_creation_date(self.xmlroot.find("front/date"))
@@ -269,3 +278,7 @@ def parser_msgs(self):
 class InvalidXMLError(Exception):
     """File is not valid XML"""
     pass
+
+
+class InvalidMetadataError(Exception):
+    """XML is well-formed but has invalid metadata"""

From 183cd995aa289632f194bff8b4b9c9b32a042200 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 21 Feb 2025 14:17:56 -0400
Subject: [PATCH 265/656] fix: maintain original TZID letter case (#8577)

---
 ietf/meeting/tests_views.py       | 15 ++++++++++++++-
 ietf/templates/meeting/agenda.ics |  2 +-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 848c9b7723..519f5f7c2d 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -385,7 +385,20 @@ def test_meeting_agenda(self):
         assert_ical_response_is_valid(self, r)
         self.assertContains(r, "BEGIN:VTIMEZONE")
         self.assertContains(r, "END:VTIMEZONE")
-
+        self.assertContains(r, meeting.time_zone, msg_prefix="time_zone should appear in its original case")
+        self.assertNotEqual(
+            meeting.time_zone,
+            meeting.time_zone.lower(),
+            "meeting needs a mixed-case tz for this test",
+        )
+        self.assertNotContains(r, meeting.time_zone.lower(), msg_prefix="time_zone should not be lower-cased")
+        self.assertNotEqual(
+            meeting.time_zone,
+            meeting.time_zone.upper(),
+            "meeting needs a mixed-case tz for this test",
+        )
+        self.assertNotContains(r, meeting.time_zone.upper(), msg_prefix="time_zone should not be upper-cased")
+    
         # iCal, single group
         r = self.client.get(ical_url + "?show=" + session.group.parent.acronym.upper())
         assert_ical_response_is_valid(self, r)
diff --git a/ietf/templates/meeting/agenda.ics b/ietf/templates/meeting/agenda.ics
index 8bc8222bbe..eb83dd479a 100644
--- a/ietf/templates/meeting/agenda.ics
+++ b/ietf/templates/meeting/agenda.ics
@@ -1,4 +1,4 @@
-{% load humanize tz %}{% autoescape off %}{% timezone schedule.meeting.tz %}{% with tzname=schedule.meeting.time_zone|lower %}{% load ietf_filters textfilters %}{% load cache %}{% cache 1800 ietf_meeting_agenda_ics schedule.meeting.number request.path request.GET %}BEGIN:VCALENDAR
+{% load humanize tz %}{% autoescape off %}{% timezone schedule.meeting.tz %}{% with tzname=schedule.meeting.time_zone %}{% load ietf_filters textfilters %}{% load cache %}{% cache 1800 ietf_meeting_agenda_ics schedule.meeting.number request.path request.GET %}BEGIN:VCALENDAR
 VERSION:2.0
 METHOD:PUBLISH
 PRODID:-//IETF//datatracker.ietf.org ical agenda//EN

From a9a8f9ba0116aea7480e83a2dfc133196cf9280f Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 21 Feb 2025 20:08:12 -0400
Subject: [PATCH 266/656] chore(deps): remove pydyf pin, update weasyprint req
 (#8580)

* chore(deps): remove pydyf pin, update weasyprint req

* chore(deps): drop pdf extra from xml2rfc dep

This should come back when we use xml2rfc for PDF generation
---
 requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 66a785e929..d8b6e0742f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -57,7 +57,7 @@ oic>=1.3    # Used only by tests
 Pillow>=9.1.0
 psycopg2>=2.9.6
 pyang>=2.5.3
-pydyf>0.8.0,<0.10.0 # until weasyprint adjusts for 0.10.0 and later
+pydyf>0.8.0
 pyflakes>=2.4.0
 pyopenssl>=22.0.0    # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency
 pyquery>=1.4.3
@@ -80,6 +80,6 @@ tlds>=2022042700    # Used to teach bleach about which TLDs currently exist
 tqdm>=4.64.0
 Unidecode>=1.3.4
 urllib3>=1.26,<2
-weasyprint>=59
-xml2rfc[pdf]>=3.23.0
+weasyprint>=64.1
+xml2rfc>=3.23.0
 xym>=0.6,<1.0

From cb8ef96f364c3e76d6726f12889d534ab1b49a66 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 3 Mar 2025 11:21:39 -0400
Subject: [PATCH 267/656] fix: more submission date feedback; refactor xml2rfc
 log capture (#8621)

* feat: catch and report any <date> parsing error

* refactor: error handling in a more testable way

* fix: no bare `except`

* test: exception cases for test_parse_creation_date

* fix: explicitly reject non-numeric day/year

* test: suppress xml2rfc output in test

* refactor: context manager to capture xml2rfc output

* refactor: more capture_xml2rfc_output usage

* fix: capture_xml2rfc_output exception handling
---
 ietf/submit/utils.py   | 162 ++++++++++++++++++++---------------------
 ietf/utils/tests.py    | 107 ++++++++++++++++++++++++++-
 ietf/utils/xmldraft.py |  66 ++++++++++++-----
 3 files changed, 232 insertions(+), 103 deletions(-)

diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index 61274c0116..a0c7dd8511 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -58,7 +58,7 @@
 from ietf.utils.mail import is_valid_email
 from ietf.utils.text import parse_unicode, normalize_text
 from ietf.utils.timezone import date_today
-from ietf.utils.xmldraft import InvalidMetadataError, XMLDraft
+from ietf.utils.xmldraft import InvalidMetadataError, XMLDraft, capture_xml2rfc_output
 from ietf.person.name import unidecode_name
 
 
@@ -926,105 +926,101 @@ def render_missing_formats(submission):
     If a txt file already exists, leaves it in place. Overwrites an existing html file
     if there is one.
     """
-    # Capture stdio/stdout from xml2rfc
-    xml2rfc_stdout = io.StringIO()
-    xml2rfc_stderr = io.StringIO()
-    xml2rfc.log.write_out = xml2rfc_stdout
-    xml2rfc.log.write_err = xml2rfc_stderr
-    xml_path = staging_path(submission.name, submission.rev, '.xml')
-    parser = xml2rfc.XmlRfcParser(str(xml_path), quiet=True)
-    try:
-        # --- Parse the xml ---
-        xmltree = parser.parse(remove_comments=False)
-    except Exception as err:
-        raise XmlRfcError(
-            "Error parsing XML",
-            xml2rfc_stdout=xml2rfc_stdout.getvalue(),
-            xml2rfc_stderr=xml2rfc_stderr.getvalue(),
-        ) from err
-    # If we have v2, run it through v2v3. Keep track of the submitted version, though.
-    xmlroot = xmltree.getroot()
-    xml_version = xmlroot.get('version', '2')
-    if xml_version == '2':
-        v2v3 = xml2rfc.V2v3XmlWriter(xmltree)
+    with capture_xml2rfc_output() as xml2rfc_logs:
+        xml_path = staging_path(submission.name, submission.rev, '.xml')
+        parser = xml2rfc.XmlRfcParser(str(xml_path), quiet=True)
         try:
-            xmltree.tree = v2v3.convert2to3()
+            # --- Parse the xml ---
+            xmltree = parser.parse(remove_comments=False)
         except Exception as err:
             raise XmlRfcError(
-                "Error converting v2 XML to v3",
-                xml2rfc_stdout=xml2rfc_stdout.getvalue(),
-                xml2rfc_stderr=xml2rfc_stderr.getvalue(),
+                "Error parsing XML",
+                xml2rfc_stdout=xml2rfc_logs["stdout"].getvalue(),
+                xml2rfc_stderr=xml2rfc_logs["stderr"].getvalue(),
             ) from err
-
-    # --- Prep the xml ---
-    today = date_today()
-    prep = xml2rfc.PrepToolWriter(xmltree, quiet=True, liberal=True, keep_pis=[xml2rfc.V3_PI_TARGET])
-    prep.options.accept_prepped = True
-    prep.options.date = today
-    try:
-        xmltree.tree = prep.prep()
-    except RfcWriterError:
-        raise XmlRfcError(
-            f"Error during xml2rfc prep: {prep.errors}",
-            xml2rfc_stdout=xml2rfc_stdout.getvalue(),
-            xml2rfc_stderr=xml2rfc_stderr.getvalue(),
-        )
-    except Exception as err:
-        raise XmlRfcError(
-            "Unexpected error during xml2rfc prep",
-            xml2rfc_stdout=xml2rfc_stdout.getvalue(),
-            xml2rfc_stderr=xml2rfc_stderr.getvalue(),
-        ) from err
-
-    # --- Convert to txt ---
-    txt_path = staging_path(submission.name, submission.rev, '.txt')
-    if not txt_path.exists():
-        writer = xml2rfc.TextWriter(xmltree, quiet=True)
-        writer.options.accept_prepped = True
+        # If we have v2, run it through v2v3. Keep track of the submitted version, though.
+        xmlroot = xmltree.getroot()
+        xml_version = xmlroot.get('version', '2')
+        if xml_version == '2':
+            v2v3 = xml2rfc.V2v3XmlWriter(xmltree)
+            try:
+                xmltree.tree = v2v3.convert2to3()
+            except Exception as err:
+                raise XmlRfcError(
+                    "Error converting v2 XML to v3",
+                    xml2rfc_stdout=xml2rfc_logs["stdout"].getvalue(),
+                    xml2rfc_stderr=xml2rfc_logs["stderr"].getvalue(),
+                ) from err
+    
+        # --- Prep the xml ---
+        today = date_today()
+        prep = xml2rfc.PrepToolWriter(xmltree, quiet=True, liberal=True, keep_pis=[xml2rfc.V3_PI_TARGET])
+        prep.options.accept_prepped = True
+        prep.options.date = today
+        try:
+            xmltree.tree = prep.prep()
+        except RfcWriterError:
+            raise XmlRfcError(
+                f"Error during xml2rfc prep: {prep.errors}",
+                xml2rfc_stdout=xml2rfc_logs["stdout"].getvalue(),
+                xml2rfc_stderr=xml2rfc_logs["stderr"].getvalue(),
+            )
+        except Exception as err:
+            raise XmlRfcError(
+                "Unexpected error during xml2rfc prep",
+                xml2rfc_stdout=xml2rfc_logs["stdout"].getvalue(),
+                xml2rfc_stderr=xml2rfc_logs["stderr"].getvalue(),
+            ) from err
+    
+        # --- Convert to txt ---
+        txt_path = staging_path(submission.name, submission.rev, '.txt')
+        if not txt_path.exists():
+            writer = xml2rfc.TextWriter(xmltree, quiet=True)
+            writer.options.accept_prepped = True
+            writer.options.date = today
+            try:
+                writer.write(txt_path)
+            except Exception as err:
+                raise XmlRfcError(
+                    "Error generating text format from XML",
+                    xml2rfc_stdout=xml2rfc_logs["stdout"].getvalue(),
+                    xml2rfc_stderr=xml2rfc_logs["stderr"].getvalue(),
+                ) from err
+            log.log(
+                'In %s: xml2rfc %s generated %s from %s (version %s)' % (
+                    str(xml_path.parent),
+                    xml2rfc.__version__,
+                    txt_path.name,
+                    xml_path.name,
+                    xml_version,
+                )
+            )
+            # When the blobstores become autoritative - the guard at the
+            # containing if statement needs to be based on the store
+            with Path(txt_path).open("rb") as f:
+                store_file("staging", f"{submission.name}-{submission.rev}.txt", f)
+    
+        # --- Convert to html ---
+        html_path = staging_path(submission.name, submission.rev, '.html')
+        writer = xml2rfc.HtmlWriter(xmltree, quiet=True)
         writer.options.date = today
         try:
-            writer.write(txt_path)
+            writer.write(str(html_path))
         except Exception as err:
             raise XmlRfcError(
-                "Error generating text format from XML",
-            xml2rfc_stdout=xml2rfc_stdout.getvalue(),
-            xml2rfc_stderr=xml2rfc_stderr.getvalue(),
+                "Error generating HTML format from XML",
+                xml2rfc_stdout=xml2rfc_logs["stdout"].getvalue(),
+                xml2rfc_stderr=xml2rfc_logs["stderr"].getvalue(),
             ) from err
         log.log(
             'In %s: xml2rfc %s generated %s from %s (version %s)' % (
                 str(xml_path.parent),
                 xml2rfc.__version__,
-                txt_path.name,
+                html_path.name,
                 xml_path.name,
                 xml_version,
             )
         )
-        # When the blobstores become autoritative - the guard at the
-        # containing if statement needs to be based on the store
-        with Path(txt_path).open("rb") as f:
-            store_file("staging", f"{submission.name}-{submission.rev}.txt", f)
-
-    # --- Convert to html ---
-    html_path = staging_path(submission.name, submission.rev, '.html')
-    writer = xml2rfc.HtmlWriter(xmltree, quiet=True)
-    writer.options.date = today
-    try:
-        writer.write(str(html_path))
-    except Exception as err:
-        raise XmlRfcError(
-            "Error generating HTML format from XML",
-            xml2rfc_stdout=xml2rfc_stdout.getvalue(),
-            xml2rfc_stderr=xml2rfc_stderr.getvalue(),
-        ) from err
-    log.log(
-        'In %s: xml2rfc %s generated %s from %s (version %s)' % (
-            str(xml_path.parent),
-            xml2rfc.__version__,
-            html_path.name,
-            xml_path.name,
-            xml_version,
-        )
-    )
     with Path(html_path).open("rb") as f:
         store_file("staging", f"{submission.name}-{submission.rev}.html", f)
 
diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py
index 0a1986a608..2dd861cd11 100644
--- a/ietf/utils/tests.py
+++ b/ietf/utils/tests.py
@@ -23,6 +23,8 @@
 from importlib import import_module
 from textwrap import dedent
 from tempfile import mkdtemp
+from xml2rfc import log as xml2rfc_log
+from xml2rfc.util.date import extract_date as xml2rfc_extract_date
 
 from django.apps import apps
 from django.contrib.auth.models import User
@@ -57,7 +59,7 @@
 from ietf.utils.test_utils import TestCase, unicontent
 from ietf.utils.text import parse_unicode
 from ietf.utils.timezone import timezone_not_near_midnight
-from ietf.utils.xmldraft import XMLDraft
+from ietf.utils.xmldraft import XMLDraft, InvalidMetadataError, capture_xml2rfc_output
 
 class SendingMail(TestCase):
 
@@ -544,7 +546,7 @@ def test_get_refs_v2(self):
     def test_parse_creation_date(self):
         # override date_today to avoid skew when test runs around midnight
         today = datetime.date.today()
-        with patch("ietf.utils.xmldraft.date_today", return_value=today):
+        with capture_xml2rfc_output(), patch("ietf.utils.xmldraft.date_today", return_value=today):
             # Note: using a dict as a stand-in for XML elements, which rely on the get() method
             self.assertEqual(
                 XMLDraft.parse_creation_date({"year": "2022", "month": "11", "day": "24"}),
@@ -590,6 +592,74 @@ def test_parse_creation_date(self):
                 ),
                 datetime.date(today.year, 1 if today.month != 1 else 2, 15),
             )
+            # Some exeception-inducing conditions
+            with self.assertRaises(
+                InvalidMetadataError,
+                msg="raise an InvalidMetadataError if a year-only date is not current",
+            ):
+                XMLDraft.parse_creation_date(
+                    {
+                        "year": str(today.year - 1),
+                        "month": "",
+                        "day": "",
+                    }
+                )
+            with self.assertRaises(
+                InvalidMetadataError,
+                msg="raise an InvalidMetadataError for a non-numeric year"
+            ):
+                XMLDraft.parse_creation_date(
+                    {
+                        "year": "two thousand twenty-five",
+                        "month": "2",
+                        "day": "28",
+                    }
+                )
+            with self.assertRaises(
+                InvalidMetadataError,
+                msg="raise an InvalidMetadataError for an invalid month"
+            ):
+                XMLDraft.parse_creation_date(
+                    {
+                        "year": "2024",
+                        "month": "13",
+                        "day": "28",
+                    }
+                )
+            with self.assertRaises(
+                InvalidMetadataError,
+                msg="raise an InvalidMetadataError for a misspelled month"
+            ):
+                XMLDraft.parse_creation_date(
+                    {
+                        "year": "2024",
+                        "month": "Oktobur",
+                        "day": "28",
+                    }
+                )
+            with self.assertRaises(
+                InvalidMetadataError,
+                msg="raise an InvalidMetadataError for an invalid day"
+            ):
+                XMLDraft.parse_creation_date(
+                    {
+                        "year": "2024",
+                        "month": "feb",
+                        "day": "31",
+                    }
+                )
+            with self.assertRaises(
+                InvalidMetadataError,
+                msg="raise an InvalidMetadataError for a non-numeric day"
+            ):
+                XMLDraft.parse_creation_date(
+                    {
+                        "year": "2024",
+                        "month": "feb",
+                        "day": "twenty-four",
+                    }
+                )
+
 
     def test_parse_docname(self):
         with self.assertRaises(ValueError) as cm:
@@ -671,6 +741,39 @@ def test_render_author_name(self):
             "J. Q.",
         )
 
+    def test_capture_xml2rfc_output(self):
+        """capture_xml2rfc_output reroutes and captures xml2rfc logs"""
+        orig_write_out = xml2rfc_log.write_out
+        orig_write_err = xml2rfc_log.write_err
+        with capture_xml2rfc_output() as outer_log_streams:  # ensure no output
+            # such meta! very Inception!
+            with capture_xml2rfc_output() as inner_log_streams:
+                # arbitrary xml2rfc method that triggers a log, nothing special otherwise
+                xml2rfc_extract_date({"year": "fish"}, datetime.date(2025,3,1))
+            self.assertNotEqual(inner_log_streams, outer_log_streams)
+            self.assertEqual(xml2rfc_log.write_out, outer_log_streams["stdout"], "out stream should be restored")
+            self.assertEqual(xml2rfc_log.write_err, outer_log_streams["stderr"], "err stream should be restored")
+        self.assertEqual(xml2rfc_log.write_out, orig_write_out, "original out stream should be restored")
+        self.assertEqual(xml2rfc_log.write_err, orig_write_err, "original err stream should be restored")
+
+        # don't happen to get any output on stdout and not paranoid enough to force some, just test stderr
+        self.assertGreater(len(inner_log_streams["stderr"].getvalue()), 0, "want output on inner streams")
+        self.assertEqual(len(outer_log_streams["stdout"].getvalue()), 0, "no output on outer streams")
+        self.assertEqual(len(outer_log_streams["stderr"].getvalue()), 0, "no output on outer streams")
+
+    def test_capture_xml2rfc_output_exception_handling(self):
+        """capture_xml2rfc_output restores streams after an exception"""
+        orig_write_out = xml2rfc_log.write_out
+        orig_write_err = xml2rfc_log.write_err
+        with capture_xml2rfc_output() as outer_log_streams:  # ensure no output
+            with self.assertRaises(RuntimeError), capture_xml2rfc_output() as inner_log_streams:
+                raise RuntimeError("nooo")
+            self.assertNotEqual(inner_log_streams, outer_log_streams)
+            self.assertEqual(xml2rfc_log.write_out, outer_log_streams["stdout"], "out stream should be restored")
+            self.assertEqual(xml2rfc_log.write_err, outer_log_streams["stderr"], "err stream should be restored")
+        self.assertEqual(xml2rfc_log.write_out, orig_write_out, "original out stream should be restored")
+        self.assertEqual(xml2rfc_log.write_err, orig_write_err, "original err stream should be restored")
+
 
 class NameTests(TestCase):
 
diff --git a/ietf/utils/xmldraft.py b/ietf/utils/xmldraft.py
index 73baf917d8..3ac9a269c7 100644
--- a/ietf/utils/xmldraft.py
+++ b/ietf/utils/xmldraft.py
@@ -7,7 +7,7 @@
 
 import debug  # pyflakes: ignore
 
-from contextlib import ExitStack
+from contextlib import contextmanager
 from lxml.etree import XMLSyntaxError
 from xml2rfc.util.date import augment_date, extract_date
 from ietf.utils.timezone import date_today
@@ -15,6 +15,21 @@
 from .draft import Draft
 
 
+@contextmanager
+def capture_xml2rfc_output():
+    orig_write_out = xml2rfc.log.write_out
+    orig_write_err = xml2rfc.log.write_err
+    parser_out = io.StringIO()
+    parser_err = io.StringIO()
+    xml2rfc.log.write_out = parser_out
+    xml2rfc.log.write_err = parser_err
+    try:
+        yield {"stdout": parser_out, "stderr": parser_err}
+    finally:
+        xml2rfc.log.write_out = orig_write_out
+        xml2rfc.log.write_err = orig_write_err
+
+
 class XMLDraft(Draft):
     """Draft from XML source
 
@@ -38,27 +53,18 @@ def parse_xml(filename):
         Converts to xml2rfc v3 schema, then returns the root of the v3 tree and the original
         xml version.
         """
-        orig_write_out = xml2rfc.log.write_out
-        orig_write_err = xml2rfc.log.write_err
-        parser_out = io.StringIO()
-        parser_err = io.StringIO()
-
-        with ExitStack() as stack:
-            @stack.callback
-            def cleanup():  # called when context exited, even if there's an exception
-                xml2rfc.log.write_out = orig_write_out
-                xml2rfc.log.write_err = orig_write_err
-
-            xml2rfc.log.write_out = parser_out
-            xml2rfc.log.write_err = parser_err
 
+        with capture_xml2rfc_output() as parser_logs:
             parser = xml2rfc.XmlRfcParser(filename, quiet=True)
             try:
                 tree = parser.parse()
             except XMLSyntaxError:
                 raise InvalidXMLError()
             except Exception as e:
-                raise XMLParseError(parser_out.getvalue(), parser_err.getvalue()) from e
+                raise XMLParseError(
+                    parser_logs["stdout"].getvalue(),
+                    parser_logs["stderr"].getvalue(),
+                ) from e
 
             xml_version = tree.getroot().get('version', '2')
             if xml_version == '2':
@@ -147,10 +153,31 @@ def get_title(self):
     def parse_creation_date(date_elt):
         if date_elt is None:
             return None
+
         today = date_today()
-        # ths mimics handling of date elements in the xml2rfc text/html writers
-        year, month, day = extract_date(date_elt, today)
-        year, month, day = augment_date(year, month, day, today)
+
+        # Outright reject non-numeric year / day (xml2rfc's extract_date does not do this)
+        # (n.b., "year" can be non-numeric in a <reference> section per RFC 7991)
+        year = date_elt.get("year")
+        day = date_elt.get("day")
+        non_numeric_year = year and not year.isdigit()
+        non_numeric_day = day and not day.isdigit()
+        if non_numeric_day or non_numeric_year:
+            raise InvalidMetadataError(
+                "Unable to parse the <date> element in the <front> section: "
+                "year and day must be numeric values if specified."
+            )
+
+        try:
+            # ths mimics handling of date elements in the xml2rfc text/html writers
+            year, month, day = extract_date(date_elt, today)
+            year, month, day = augment_date(year, month, day, today)
+        except Exception as err:
+            # Give a generic error if anything goes wrong so far...
+            raise InvalidMetadataError(
+                "Unable to parse the <date> element in the <front> section."
+            ) from err
+
         if not day:
             # Must choose a day for a datetime.date. Per RFC 7991 sect 2.17, we use
             # today's date if it is consistent with the rest of the date. Otherwise,
@@ -159,9 +186,12 @@ def parse_creation_date(date_elt):
                 day = today.day
             else:
                 day = 15
+
         try:
             creation_date = datetime.date(year, month, day)
         except Exception:
+            # If everything went well, we should have had a valid datetime, but we didn't.
+            # The parsing _worked_ but not in a way that we can go forward with.
             raise InvalidMetadataError(
                 "The <date> element in the <front> section specified an incomplete date "
                 "that was not consistent with today's date. If you specify only a year, "

From 232a861f8ae52e1026d59d7088f07211acd166a5 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 3 Mar 2025 14:51:14 -0400
Subject: [PATCH 268/656] chore: config gunicorn secure_scheme_headers (#8632)

* chore: config gunicorn secure_scheme_headers

* chore: typo in comment
---
 dev/build/gunicorn.conf.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py
index 6666a0d37d..032d95ee0d 100644
--- a/dev/build/gunicorn.conf.py
+++ b/dev/build/gunicorn.conf.py
@@ -1,5 +1,11 @@
 # Copyright The IETF Trust 2024, All Rights Reserved
 
+# Configure security scheme headers for forwarded requests. Cloudflare sets X-Forwarded-Proto 
+# for us. Don't trust any of the other similar headers. Only trust the header if it's coming
+# from localhost, as all legitimate traffic will reach gunicorn via co-located nginx.
+secure_scheme_headers = {"X-FORWARDED-PROTO": "https"}
+forwarded_allow_ips = "127.0.0.1, ::1"  # this is the default 
+
 # Log as JSON on stdout (to distinguish from Django's logs on stderr)
 #
 # This is applied as an update to gunicorn's glogging.CONFIG_DEFAULTS.

From 554182ef8ab33947ca8d9ee904a5d5472d3c57f8 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 4 Mar 2025 11:42:04 -0600
Subject: [PATCH 269/656] feat: run the docker container as dev (#8606)

* feat: run the docker container as dev

* fix: $@ -> $*

Old bug, but might as well fix it now

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 dev/celery/docker-init.sh | 13 ++++++---
 docker-compose.yml        |  4 ++-
 docker/celery.Dockerfile  | 60 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+), 5 deletions(-)
 create mode 100644 docker/celery.Dockerfile

diff --git a/dev/celery/docker-init.sh b/dev/celery/docker-init.sh
index 4fd1f1294f..9940dfd7d0 100755
--- a/dev/celery/docker-init.sh
+++ b/dev/celery/docker-init.sh
@@ -49,11 +49,16 @@ if [[ -n "${CELERY_GID}" ]]; then
 fi
 
 run_as_celery_uid () {
-  SU_OPTS=()
-  if [[ -n "${CELERY_GROUP}" ]]; then
-    SU_OPTS+=("-g" "${CELERY_GROUP}")
+  IAM=$(whoami)
+  if [ "${IAM}" = "${CELERY_USERNAME:-root}" ]; then
+    SU_OPTS=()
+    if [[ -n "${CELERY_GROUP}" ]]; then
+      SU_OPTS+=("-g" "${CELERY_GROUP}")
+    fi
+    su "${SU_OPTS[@]}" "${CELERY_USERNAME:-root}" -s /bin/sh -c "$*"
+  else
+    /bin/sh -c "$*"
   fi
-  su "${SU_OPTS[@]}" "${CELERY_USERNAME:-root}" -s /bin/sh -c "$@"
 }
 
 log_term_timing_msgs () {
diff --git a/docker-compose.yml b/docker-compose.yml
index 30ce8ba4d2..9910c02a99 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -67,7 +67,9 @@ services:
         restart: unless-stopped
 
     celery:
-        image: ghcr.io/ietf-tools/datatracker-celery:latest
+        build:
+            context: .
+            dockerfile: docker/celery.Dockerfile
         init: true
         environment:
             CELERY_APP: ietf
diff --git a/docker/celery.Dockerfile b/docker/celery.Dockerfile
new file mode 100644
index 0000000000..e44200398c
--- /dev/null
+++ b/docker/celery.Dockerfile
@@ -0,0 +1,60 @@
+FROM ghcr.io/ietf-tools/datatracker-celery:latest
+LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install needed packages and setup non-root user.
+ARG USERNAME=dev
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh
+RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh
+
+# Add Postgresql Apt Repository to get 14    
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
+RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \
+    # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
+    && apt-get purge -y imagemagick imagemagick-6-common \
+    # Install common packages, non-root user
+    # Syntax: ./docker-setup-debian.sh [install zsh flag] [username] [user UID] [user GID] [upgrade packages flag] [install Oh My Zsh! flag] [Add non-free packages]
+    && bash /tmp/library-scripts/docker-setup-debian.sh "true" "${USERNAME}" "${USER_UID}" "${USER_GID}" "false" "true" "true"
+
+# Setup default python tools in a venv via pipx to avoid conflicts
+ENV PIPX_HOME=/usr/local/py-utils \
+    PIPX_BIN_DIR=/usr/local/py-utils/bin
+ENV PATH=${PATH}:${PIPX_BIN_DIR}
+COPY docker/scripts/app-setup-python.sh /tmp/library-scripts/docker-setup-python.sh
+RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-python.sh && chmod +x /tmp/library-scripts/docker-setup-python.sh
+RUN bash /tmp/library-scripts/docker-setup-python.sh "none" "/usr/local" "${PIPX_HOME}" "${USERNAME}"
+
+# Remove library scripts for final image
+RUN rm -rf /tmp/library-scripts
+
+# Copy the startup file
+COPY dev/celery/docker-init.sh /docker-init.sh
+RUN sed -i 's/\r$//' /docker-init.sh && \
+    chmod +x /docker-init.sh
+
+ENTRYPOINT [ "/docker-init.sh" ]
+
+# Fix user UID / GID to match host
+RUN groupmod --gid $USER_GID $USERNAME \
+    && usermod --uid $USER_UID --gid $USER_GID $USERNAME \
+    && chown -R $USER_UID:$USER_GID /home/$USERNAME \
+    || exit 0
+
+# Switch to local dev user
+USER dev:dev
+
+# Install current datatracker python dependencies
+COPY requirements.txt /tmp/pip-tmp/
+RUN pip3 --disable-pip-version-check --no-cache-dir install --user --no-warn-script-location -r /tmp/pip-tmp/requirements.txt
+RUN pip3 --disable-pip-version-check --no-cache-dir install --user --no-warn-script-location watchdog[watchmedo]
+
+RUN sudo rm -rf /tmp/pip-tmp
+
+VOLUME [ "/assets" ]
+

From cf6340443f7437ac23ff65c981741335a0911363 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Sat, 8 Mar 2025 00:56:14 -0500
Subject: [PATCH 270/656] docs: Update README.md

---
 README.md | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 0ece0eb03b..abebb7ca02 100644
--- a/README.md
+++ b/README.md
@@ -44,6 +44,7 @@
 
 This project is following the standard **Git Feature Workflow** development model. Learn about all the various steps of the development workflow, from creating a fork to submitting a pull request, in the [Contributing](https://github.com/ietf-tools/.github/blob/main/CONTRIBUTING.md) guide.
 
+> [!TIP]  
 > Make sure to read the [Styleguides](https://github.com/ietf-tools/.github/blob/main/CONTRIBUTING.md#styleguides) section to ensure a cohesive code format across the project.
 
 You can submit bug reports, enhancement and new feature requests in the [discussions](https://github.com/ietf-tools/datatracker/discussions) area. Accepted tickets will be converted to issues.
@@ -52,7 +53,8 @@ You can submit bug reports, enhancement and new feature requests in the [discuss
 
 Click the <kbd>Fork</kbd> button in the top-right corner of the repository to create a personal copy that you can work on.
 
-> Note that some GitHub Actions might be enabled by default in your fork. You should disable them by going to **Settings** > **Actions** > **General** and selecting **Disable actions** (then Save).
+> [!NOTE]  
+> Some GitHub Actions might be enabled by default in your fork. You should disable them by going to **Settings** > **Actions** > **General** and selecting **Disable actions** (then Save).
 
 #### Git Cloning Tips
 
@@ -104,7 +106,8 @@ Read the [Docker Dev Environment](docker/README.md) guide to get started.
 
 Nightly database dumps of the datatracker are available as Docker images: `ghcr.io/ietf-tools/datatracker-db:latest`  
 
-> Note that to update the database in your dev environment to the latest version, you should run the `docker/cleandb` script.
+> [!TIP]  
+> In order to update the database in your dev environment to the latest version, you should run the `docker/cleandb` script.
 
 ### Blob storage for dev/test
 
@@ -248,6 +251,7 @@ From a datatracker container, run the command:
 ./ietf/manage.py test --settings=settings_test
 ```
 
+> [!TIP]  
 > You can limit the run to specific tests using the `--pattern` argument.
 
 ### Frontend Tests
@@ -257,11 +261,13 @@ Frontend tests are done via Playwright. There're 2 different type of tests:
 - Tests that test Vue pages / components and run natively without any external dependency.
 - Tests that require a running datatracker instance to test against (usually legacy views).
 
+> [!IMPORTANT]  
 > Make sure you have Node.js 16.x or later installed on your machine.
 
 #### Run Vue Tests
 
-> :warning: All commands below **MUST** be run from the `./playwright` directory, unless noted otherwise.
+> [!WARNING]  
+> All commands below **MUST** be run from the `./playwright` directory, unless noted otherwise.
 
 1. Run **once** to install dependencies on your system:
     ```sh
@@ -294,7 +300,8 @@ Frontend tests are done via Playwright. There're 2 different type of tests:
 
 First, you need to start a datatracker instance (dev or prod), ideally from a docker container, exposing the 8000 port.
 
-> :warning: All commands below **MUST** be run from the `./playwright` directory.
+> [!WARNING]  
+> All commands below **MUST** be run from the `./playwright` directory.
 
 1. Run **once** to install dependencies on your system:
 ```sh

From cf21c4129a3d083980297dcaa82b5fd58bf447f6 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Sat, 8 Mar 2025 00:59:26 -0500
Subject: [PATCH 271/656] docs: Update README.md

---
 docker/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index 14fcc38995..f2161a173f 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -4,11 +4,12 @@
 
 1. [Set up Docker](https://docs.docker.com/get-started/) on your preferred platform. On Windows, it is highly recommended to use the [WSL 2 *(Windows Subsystem for Linux)*](https://docs.docker.com/desktop/windows/wsl/) backend.
 
+> [!IMPORTANT]  
 > See the [IETF Tools Windows Dev guide](https://github.com/ietf-tools/.github/blob/main/docs/windows-dev.md) on how to get started when using Windows.
 
 2. On Linux, you must [install Docker Compose manually](https://docs.docker.com/compose/install/linux/#install-the-plugin-manually) and not install Docker Desktop. On Mac and Windows install Docker Desktop which already includes Docker Compose.
 
-2. If you have a copy of the datatracker code checked out already, simply `cd` to the top-level directory.
+3. If you have a copy of the datatracker code checked out already, simply `cd` to the top-level directory.
 
    If not, check out a datatracker branch as usual. We'll check out `main` below, but you can use any branch:
 
@@ -18,7 +19,7 @@
     git checkout main
     ```
 
-3. Follow the instructions for your preferred editor:
+4. Follow the instructions for your preferred editor:
     - [Visual Studio Code](#using-visual-studio-code)
     - [Other Editors / Generic](#using-other-editors--generic)
 
@@ -189,7 +190,6 @@ The content of the source files will be copied into the target `.ics` files. Mak
 
 Because including all assets in the image would significantly increase the file size, they are not included by default. You can however fetch them by running the **Fetch assets via rsync** task in VS Code or run manually the script `docker/scripts/app-rsync-extras.sh`
 
-
 ### Linux file permissions leaking to the host system
 
 If on the host filesystem you have permissions that look like this,

From 06158c05c77b89434197a626d7cfcf04b8b1e14a Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Mon, 10 Mar 2025 16:28:20 -0400
Subject: [PATCH 272/656] chore: Remove deprecated version from
 docker-compose.extend.yml

---
 .devcontainer/docker-compose.extend.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml
index 286eefb29c..a92f42bc6d 100644
--- a/.devcontainer/docker-compose.extend.yml
+++ b/.devcontainer/docker-compose.extend.yml
@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
     app:
         environment:

From 887ec11f3916d19da04dd939a0aa2edd697f91fd Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 15 Mar 2025 00:43:17 -0400
Subject: [PATCH 273/656] chore(deps): bump ncipollo/release-action from 1.15.0
 to 1.16.0 (#8594)

Bumps [ncipollo/release-action](https://github.com/ncipollo/release-action) from 1.15.0 to 1.16.0.
- [Release notes](https://github.com/ncipollo/release-action/releases)
- [Commits](https://github.com/ncipollo/release-action/compare/v1.15.0...v1.16.0)

---
updated-dependencies:
- dependency-name: ncipollo/release-action
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index a14ea73e9b..9f621e16b7 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -97,7 +97,7 @@ jobs:
         echo "IS_RELEASE=true" >> $GITHUB_ENV
         
     - name: Create Draft Release
-      uses: ncipollo/release-action@v1.15.0
+      uses: ncipollo/release-action@v1.16.0
       if: ${{ github.ref_name == 'release' }}
       with:
         prerelease: true
@@ -316,7 +316,7 @@ jobs:
         histCoveragePath: historical-coverage.json
         
     - name: Create Release
-      uses: ncipollo/release-action@v1.15.0
+      uses: ncipollo/release-action@v1.16.0
       if: ${{ env.SHOULD_DEPLOY == 'true' }}
       with:
         allowUpdates: true
@@ -329,7 +329,7 @@ jobs:
         token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Update Baseline Coverage
-      uses: ncipollo/release-action@v1.15.0
+      uses: ncipollo/release-action@v1.16.0
       if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }}
       with:
         allowUpdates: true

From e56c6cae272cb482a6d33f1f7e24c7ba8a48361d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 15 Mar 2025 00:43:47 -0400
Subject: [PATCH 274/656] chore(deps): bump actions/download-artifact from
 4.1.8 to 4.1.9 (#8628)

Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.1.8 to 4.1.9.
- [Release notes](https://github.com/actions/download-artifact/releases)
- [Commits](https://github.com/actions/download-artifact/compare/v4.1.8...v4.1.9)

---
updated-dependencies:
- dependency-name: actions/download-artifact
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 9f621e16b7..123bd5c65a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -185,7 +185,7 @@ jobs:
 
     - name: Download a Coverage Results
       if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
-      uses: actions/download-artifact@v4.1.8
+      uses: actions/download-artifact@v4.1.9
       with:
         name: coverage
 
@@ -292,7 +292,7 @@ jobs:
 
     - name: Download Coverage Results
       if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
-      uses: actions/download-artifact@v4.1.8
+      uses: actions/download-artifact@v4.1.9
       with:
         name: coverage
 
@@ -407,7 +407,7 @@ jobs:
     - uses: actions/checkout@v4
         
     - name: Download a Release Artifact
-      uses: actions/download-artifact@v4.1.8
+      uses: actions/download-artifact@v4.1.9
       with:
         name: release-${{ env.PKG_VERSION }}
         

From 9db109f692d8592c5407ea82d2c2790eb6006263 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 15 Mar 2025 00:44:18 -0400
Subject: [PATCH 275/656] chore(deps): bump appleboy/ssh-action from 1.2.0 to
 1.2.2 (#8650)

Bumps [appleboy/ssh-action](https://github.com/appleboy/ssh-action) from 1.2.0 to 1.2.2.
- [Release notes](https://github.com/appleboy/ssh-action/releases)
- [Changelog](https://github.com/appleboy/ssh-action/blob/master/.goreleaser.yaml)
- [Commits](https://github.com/appleboy/ssh-action/compare/7eaf76671a0d7eec5d98ee897acda4f968735a17...2ead5e36573f08b82fbfce1504f1a4b05a647c6f)

---
updated-dependencies:
- dependency-name: appleboy/ssh-action
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/tests-az.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml
index 6d53a121aa..8553563a19 100644
--- a/.github/workflows/tests-az.yml
+++ b/.github/workflows/tests-az.yml
@@ -38,7 +38,7 @@ jobs:
         ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts
 
     - name: Remote SSH into VM
-      uses: appleboy/ssh-action@7eaf76671a0d7eec5d98ee897acda4f968735a17
+      uses: appleboy/ssh-action@2ead5e36573f08b82fbfce1504f1a4b05a647c6f
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:

From 227b44bfa25036e7d4fea86a72405beb4e53e2ce Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 15 Mar 2025 00:45:42 -0400
Subject: [PATCH 276/656] chore(deps): bump nanoid (#8649)

Bumps the npm group with 1 update in the /dev/deploy-to-container directory: [nanoid](https://github.com/ai/nanoid).


Updates `nanoid` from 5.0.9 to 5.1.3
- [Release notes](https://github.com/ai/nanoid/releases)
- [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md)
- [Commits](https://github.com/ai/nanoid/compare/5.0.9...5.1.3)

---
updated-dependencies:
- dependency-name: nanoid
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/deploy-to-container/package-lock.json | 15 ++++++++-------
 dev/deploy-to-container/package.json      |  2 +-
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json
index 03327083b1..c4f675527e 100644
--- a/dev/deploy-to-container/package-lock.json
+++ b/dev/deploy-to-container/package-lock.json
@@ -8,7 +8,7 @@
       "dependencies": {
         "dockerode": "^4.0.4",
         "fs-extra": "^11.3.0",
-        "nanoid": "5.0.9",
+        "nanoid": "5.1.3",
         "nanoid-dictionary": "5.0.0-beta.1",
         "slugify": "1.6.6",
         "tar": "^7.4.3",
@@ -668,15 +668,16 @@
       "optional": true
     },
     "node_modules/nanoid": {
-      "version": "5.0.9",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.9.tgz",
-      "integrity": "sha512-Aooyr6MXU6HpvvWXKoVoXwKMs/KyVakWwg7xQfv5/S/RIgJMy0Ifa45H9qqYy7pTCszrHzP21Uk4PZq2HpEM8Q==",
+      "version": "5.1.3",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.3.tgz",
+      "integrity": "sha512-zAbEOEr7u2CbxwoMRlz/pNSpRP0FdAU4pRaYunCdEezWohXFs+a0Xw7RfkKaezMsmSM1vttcLthJtwRnVtOfHQ==",
       "funding": [
         {
           "type": "github",
           "url": "https://github.com/sponsors/ai"
         }
       ],
+      "license": "MIT",
       "bin": {
         "nanoid": "bin/nanoid.js"
       },
@@ -1612,9 +1613,9 @@
       "optional": true
     },
     "nanoid": {
-      "version": "5.0.9",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.9.tgz",
-      "integrity": "sha512-Aooyr6MXU6HpvvWXKoVoXwKMs/KyVakWwg7xQfv5/S/RIgJMy0Ifa45H9qqYy7pTCszrHzP21Uk4PZq2HpEM8Q=="
+      "version": "5.1.3",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.3.tgz",
+      "integrity": "sha512-zAbEOEr7u2CbxwoMRlz/pNSpRP0FdAU4pRaYunCdEezWohXFs+a0Xw7RfkKaezMsmSM1vttcLthJtwRnVtOfHQ=="
     },
     "nanoid-dictionary": {
       "version": "5.0.0-beta.1",
diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json
index 1f54745ebf..8772c568c8 100644
--- a/dev/deploy-to-container/package.json
+++ b/dev/deploy-to-container/package.json
@@ -4,7 +4,7 @@
   "dependencies": {
     "dockerode": "^4.0.4",
     "fs-extra": "^11.3.0",
-    "nanoid": "5.0.9",
+    "nanoid": "5.1.3",
     "nanoid-dictionary": "5.0.0-beta.1",
     "slugify": "1.6.6",
     "tar": "^7.4.3",

From 968820de34bdf6d0c1fd69b921ed490da725b941 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Sat, 15 Mar 2025 14:45:04 +0700
Subject: [PATCH 277/656] feat: celery task + admin to resend Messages (#8661)

* feat: Message re-send task

* feat: admin action to queue redelivery

* feat: MessageAdmin list_filters

* feat: show sent status

* feat: better date filtering

* chore: remove send-by-date task

Adds complexity and risk - the improved Messages admin lets us do most of what it did without the opportunity for accidentally resending huge ranges

* chore: fill in empty docstring

* style: black

* fix: unused import

* feat: better logging

* chore: mypy lint

* test: test retry_send_messages_by_pk_task

* test: test retry_send_messages
---
 ietf/message/admin.py          | 79 +++++++++++++++++++++++++++++++---
 ietf/message/tasks.py          | 24 ++++++++++-
 ietf/message/tests.py          | 59 +++++++++++++++++++++++--
 ietf/message/utils.py          | 36 ++++++++++++++--
 ietf/settings.py               |  1 +
 ietf/templates/admin/base.html |  1 +
 requirements.txt               |  1 +
 7 files changed, 187 insertions(+), 14 deletions(-)

diff --git a/ietf/message/admin.py b/ietf/message/admin.py
index c2564c04b9..250e1eb596 100644
--- a/ietf/message/admin.py
+++ b/ietf/message/admin.py
@@ -1,32 +1,99 @@
-from django.contrib import admin
+# Copyright The IETF Trust 2012-2025, All Rights Reserved
+from django.contrib import admin, messages
+from django.db.models import QuerySet
+from rangefilter.filters import DateRangeQuickSelectListFilterBuilder
 
 from ietf.message.models import Message, MessageAttachment, SendQueue, AnnouncementFrom
+from ietf.message.tasks import retry_send_messages_by_pk_task
+
+
+class MessageSentStatusListFilter(admin.SimpleListFilter):
+    """Filter Messages by whether or not they were sent"""
+
+    title = "status"
+    parameter_name = "status"
+
+    def lookups(self, request, model_admin):
+        return [
+            ("sent", "Sent"),
+            ("unsent", "Not sent"),
+        ]
+
+    def queryset(self, request, queryset):
+        if self.value() == "unsent":
+            return queryset.filter(sent__isnull=True)
+        elif self.value() == "sent":
+            return queryset.filter(sent__isnull=False)
+
 
 class MessageAdmin(admin.ModelAdmin):
-    list_display = ["subject", "by", "time", "groups"]
+    list_display = ["sent_status", "subject", "by", "time", "groups"]
     search_fields = ["subject", "body"]
     raw_id_fields = ["by", "related_groups", "related_docs"]
+    list_filter = [
+        MessageSentStatusListFilter,
+        ("time", DateRangeQuickSelectListFilterBuilder()),
+    ]
     ordering = ["-time"]
+    actions = ["retry_send"]
 
     def groups(self, instance):
         return ", ".join(g.acronym for g in instance.related_groups.all())
+
+    @admin.display(description="Sent", boolean=True)
+    def sent_status(self, instance):
+        return instance.sent is not None
+
+    @admin.action(description="Send selected messages if unsent")
+    def retry_send(self, request, queryset: QuerySet[Message]):
+        try:
+            retry_send_messages_by_pk_task.delay(
+                message_pks=list(queryset.values_list("pk", flat=True)),
+                resend=False,
+            )
+        except Exception as err:
+            self.message_user(
+                request,
+                f"Error: {repr(err)}",
+                messages.ERROR,
+            )
+        else:
+            self.message_user(request, "Messages queued for delivery", messages.SUCCESS)
+
+
 admin.site.register(Message, MessageAdmin)
 
+
 class MessageAttachmentAdmin(admin.ModelAdmin):
-    list_display = ['id', 'message', 'filename', 'removed',]
-    raw_id_fields = ['message']
+    list_display = [
+        "id",
+        "message",
+        "filename",
+        "removed",
+    ]
+    raw_id_fields = ["message"]
+
+
 admin.site.register(MessageAttachment, MessageAttachmentAdmin)
 
+
 class SendQueueAdmin(admin.ModelAdmin):
     list_display = ["time", "by", "message", "send_at", "sent_at"]
     list_filter = ["time", "send_at", "sent_at"]
     search_fields = ["message__body"]
     raw_id_fields = ["by", "message"]
     ordering = ["-time"]
+
+
 admin.site.register(SendQueue, SendQueueAdmin)
 
+
 class AnnouncementFromAdmin(admin.ModelAdmin):
-    list_display = ['name', 'group', 'address', ]
-admin.site.register(AnnouncementFrom, AnnouncementFromAdmin)
+    list_display = [
+        "name",
+        "group",
+        "address",
+    ]
 
 
+admin.site.register(AnnouncementFrom, AnnouncementFromAdmin)
diff --git a/ietf/message/tasks.py b/ietf/message/tasks.py
index efd776b9d8..1fdff7bea4 100644
--- a/ietf/message/tasks.py
+++ b/ietf/message/tasks.py
@@ -5,8 +5,8 @@
 from celery import shared_task
 from smtplib import SMTPException
 
-from ietf.message.utils import send_scheduled_message_from_send_queue
-from ietf.message.models import SendQueue
+from ietf.message.utils import send_scheduled_message_from_send_queue, retry_send_messages
+from ietf.message.models import SendQueue, Message
 from ietf.utils import log
 from ietf.utils.mail import log_smtp_exception, send_error_email
 
@@ -25,3 +25,23 @@ def send_scheduled_mail_task():
         except SMTPException as e:
             log_smtp_exception(e)
             send_error_email(e)
+
+
+@shared_task
+def retry_send_messages_by_pk_task(message_pks: list, resend=False):
+    """Task to retry sending Messages by PK
+    
+    Sends Messages whose PK is included in the list.
+    Only previously unsent messages are sent unless `resend` is true.
+    """
+    log.log(
+        "retry_send_messages_by_pk_task: "
+        "retrying send of Message PKs [{}] (resend={})".format(
+            ", ".join(str(pk) for pk in message_pks),
+            resend,
+        )
+    )
+    retry_send_messages(
+        messages=Message.objects.filter(pk__in=message_pks),
+        resend=resend,
+    )
diff --git a/ietf/message/tests.py b/ietf/message/tests.py
index 7fbd29167c..a677d5477e 100644
--- a/ietf/message/tests.py
+++ b/ietf/message/tests.py
@@ -11,10 +11,10 @@
 import debug                            # pyflakes:ignore
 
 from ietf.group.factories import GroupFactory
-from ietf.message.factories import SendQueueFactory
+from ietf.message.factories import MessageFactory, SendQueueFactory
 from ietf.message.models import Message, SendQueue
-from ietf.message.tasks import send_scheduled_mail_task
-from ietf.message.utils import send_scheduled_message_from_send_queue
+from ietf.message.tasks import send_scheduled_mail_task, retry_send_messages_by_pk_task
+from ietf.message.utils import send_scheduled_message_from_send_queue, retry_send_messages
 from ietf.person.models import Person
 from ietf.utils.mail import outbox, send_mail_text, send_mail_message, get_payload_text
 from ietf.utils.test_utils import TestCase
@@ -133,6 +133,44 @@ def test_send_mime_announcement(self):
         self.assertTrue(SendQueue.objects.get(id=q.id).sent_at)
 
 
+class UtilsTests(TestCase):
+    @mock.patch("ietf.message.utils.send_mail_message")
+    def test_retry_send_messages(self, mock_send_mail_message):
+        sent_message = MessageFactory(sent=timezone.now())
+        unsent_messages = MessageFactory.create_batch(2, sent=None)
+        
+        # Send the sent message and one of the unsent messages
+        retry_send_messages(
+            Message.objects.filter(pk__in=[
+                sent_message.pk,
+                unsent_messages[0].pk,
+            ]),
+            resend=False,
+        )
+        self.assertEqual(mock_send_mail_message.call_count, 1)
+        self.assertEqual(
+            mock_send_mail_message.call_args.args[1],
+            unsent_messages[0],
+        )
+        
+        mock_send_mail_message.reset_mock()
+        # Once again, send the sent message and one of the unsent messages 
+        # (we can use the same one because our mock prevented it from having
+        # its status updated to sent)
+        retry_send_messages(
+            Message.objects.filter(pk__in=[
+                sent_message.pk,
+                unsent_messages[0].pk,
+            ]),
+            resend=True,
+        )
+        self.assertEqual(mock_send_mail_message.call_count, 2)
+        self.assertCountEqual(
+            [call_args.args[1] for call_args in mock_send_mail_message.call_args_list],
+            [sent_message, unsent_messages[0]],
+        )
+
+
 class TaskTests(TestCase):
     @mock.patch("ietf.message.tasks.log_smtp_exception")
     @mock.patch("ietf.message.tasks.send_scheduled_message_from_send_queue")
@@ -150,3 +188,18 @@ def test_send_scheduled_mail_task(self, mock_send_message, mock_log_smtp_excepti
         self.assertEqual(mock_send_message.call_count, 1)
         self.assertEqual(mock_send_message.call_args[0], (not_yet_sent,))
         self.assertTrue(mock_log_smtp_exception.called)
+
+    @mock.patch("ietf.message.tasks.retry_send_messages")
+    def test_retry_send_messages_by_pk_task(self, mock_retry_send):
+        msgs = MessageFactory.create_batch(3)
+        MessageFactory()  # an extra message that won't be resent
+
+        retry_send_messages_by_pk_task([msg.pk for msg in msgs], resend=False)
+        called_with_messages = mock_retry_send.call_args.kwargs["messages"]
+        self.assertCountEqual(msgs, called_with_messages)
+        self.assertFalse(mock_retry_send.call_args.kwargs["resend"])
+
+        retry_send_messages_by_pk_task([msg.pk for msg in msgs], resend=True)
+        called_with_messages = mock_retry_send.call_args.kwargs["messages"]
+        self.assertCountEqual(msgs, called_with_messages)
+        self.assertTrue(mock_retry_send.call_args.kwargs["resend"])
diff --git a/ietf/message/utils.py b/ietf/message/utils.py
index 2601eccab8..74448ca7c9 100644
--- a/ietf/message/utils.py
+++ b/ietf/message/utils.py
@@ -1,13 +1,17 @@
 # Copyright The IETF Trust 2012-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
 
+import email
+import email.utils
+import re
+import smtplib
 
-import re, email
-
+from django.db.models import QuerySet
 from django.utils import timezone
 from django.utils.encoding import force_str
 
-from ietf.utils.mail import send_mail_text, send_mail_mime
+from ietf.utils import log
+from ietf.utils.mail import send_mail_text, send_mail_mime, send_mail_message
 from ietf.message.models import Message
 
 first_dot_on_line_re = re.compile(r'^\.', re.MULTILINE)
@@ -58,3 +62,29 @@ def send_scheduled_message_from_send_queue(queue_item):
 
     queue_item.message.sent = queue_item.sent_at
     queue_item.message.save()
+
+
+def retry_send_messages(messages: QuerySet[Message], resend=False):
+    """Attempt delivery of Messages"""
+    if not resend:
+        # only include sent messages on explicit request
+        for already_sent in messages.filter(sent__isnull=False):
+            assert already_sent.sent is not None  # appease mypy type checking
+            log.log(
+                f"retry_send_messages: skipping {already_sent.pk} "
+                f"(already sent {already_sent.sent.isoformat(timespec='milliseconds')})"
+            )
+        messages = messages.filter(sent__isnull=True)
+    for msg in messages:
+        to = ",".join(a[1] for a in email.utils.getaddresses([msg.to]))
+        try:
+            send_mail_message(None, msg)
+            log.log(
+                f'retry_send_messages: '
+                f'sent {msg.pk} {msg.frm} -> {to} "{msg.subject.strip()}"'
+            )
+        except smtplib.SMTPException as e:
+            log.log(
+                f'retry_send_messages: '
+                f'Failure {e}:  {msg.pk}  {msg.frm} -> {to}  "{msg.subject.strip()}"'
+            )
diff --git a/ietf/settings.py b/ietf/settings.py
index faee42237c..1fe5f48229 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -465,6 +465,7 @@ def skip_unreadable_post(record):
     'drf_spectacular',
     'drf_standardized_errors',
     'rest_framework',
+    'rangefilter',
     'simple_history',
     'tastypie',
     'widget_tweaks',
diff --git a/ietf/templates/admin/base.html b/ietf/templates/admin/base.html
index 9ca7377a54..d48891dfc4 100644
--- a/ietf/templates/admin/base.html
+++ b/ietf/templates/admin/base.html
@@ -20,6 +20,7 @@
                 --header-color: var(--bs-secondary);
                 --breadcrumbs-fg: var(--bs-secondary);
                 --breadcrumbs-link-fg: var(--link-fg);
+                .calendar caption { background-color: var(--secondary);}
             }
             span.text-danger { color: var(--bs-danger); }
         </style>
diff --git a/requirements.txt b/requirements.txt
index d8b6e0742f..8bd906c220 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,6 +13,7 @@ celery>=5.2.6
 coverage>=4.5.4,<5.0    # Coverage 5.x moves from a json database to SQLite.  Moving to 5.x will require substantial rewrites in ietf.utils.test_runner and ietf.release.views
 defusedxml>=0.7.1    # for TastyPie when using xml; not a declared dependency
 Django>4.2,<5
+django-admin-rangefilter>=0.13.2
 django-analytical>=3.1.0
 django-bootstrap5>=21.3
 django-celery-beat>=2.3.0

From 48211414dfc992e4a51f0d2de9183eab5c44dad0 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Sat, 15 Mar 2025 07:58:06 +0000
Subject: [PATCH 278/656] ci: update base image target version to 20250315T0745

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 852ba43c80..2b02a091c5 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250128T1728
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250315T0745
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index fbc9426744..e6f490b168 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250128T1728
+20250315T0745

From 200d2bd164224694c0524e734ea6e46b62f3998a Mon Sep 17 00:00:00 2001
From: Russ Housley <housley@vigilsec.com>
Date: Sat, 15 Mar 2025 04:23:35 -0400
Subject: [PATCH 279/656] fix(review_info template): display correct date for
 I-D to be reviewed

---
 ietf/templates/doc/review/request_info.html | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/ietf/templates/doc/review/request_info.html b/ietf/templates/doc/review/request_info.html
index ee46916b43..9ad126d59e 100644
--- a/ietf/templates/doc/review/request_info.html
+++ b/ietf/templates/doc/review/request_info.html
@@ -96,7 +96,7 @@
                 </td>
             </tr>
         {% endif %}
-        {% if doc.time %}
+        {% if review_req.doc.time %}
             <tr>
                 <th scope="row">
                 </th>
@@ -104,7 +104,10 @@
                     I-D last updated
                 </th>
                 <td>
-                    {{ doc.time|date:"Y-m-d" }}
+                    {{ review_req.doc.time|date:"Y-m-d" }}
+                    {% if review_req.doc.pub_date %}
+                        <span class="text-body-secondary">(Latest revision {{ review_req.doc.pub_date|date:"Y-m-d" }})</span>
+                    {% endif %}
                 </td>
             </tr>
         {% endif %}

From 603938a9b6438ce78caea13272e61c0e15546478 Mon Sep 17 00:00:00 2001
From: Rich Salz <rsalz@akamai.com>
Date: Sat, 15 Mar 2025 05:19:12 -0400
Subject: [PATCH 280/656] chore: Add IETF to "Last Call Expired" email Subject
 (#8675)

No need to change the test.
Fixes: #8526
---
 ietf/doc/mails.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/doc/mails.py b/ietf/doc/mails.py
index c1e2074bc0..ddecbb6b54 100644
--- a/ietf/doc/mails.py
+++ b/ietf/doc/mails.py
@@ -568,7 +568,7 @@ def email_last_call_expired(doc):
     send_mail(None,
               addrs.to,
               "DraftTracker Mail System <iesg-secretary@ietf.org>",
-              "Last Call Expired: %s" % doc.file_tag(),
+              "IETF Last Call Expired: %s" % doc.file_tag(),
               "doc/mail/change_notice.txt",
               dict(text=text,
                    doc=doc,

From 9eb5b2fa8e25c68ec927b82008d61c8e38be354c Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Sat, 15 Mar 2025 05:23:16 -0400
Subject: [PATCH 281/656] fix(agenda): always render session row if rendering a
 new date row when filtering (#8672)

---
 client/agenda/AgendaScheduleList.vue | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index 905677b4da..ab0f6e0184 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -253,6 +253,7 @@ const meetingEvents = computed(() => {
 
     // -> Add date row
     const itemDate = DateTime.fromISO(item.adjustedStartDate)
+    let willRenderDateRow = false
     if (itemDate.toISODate() !== acc.lastDate) {
       acc.result.push({
         id: item.id,
@@ -262,12 +263,13 @@ const meetingEvents = computed(() => {
         date: itemDate.toLocaleString(DateTime.DATE_HUGE),
         cssClasses: 'agenda-table-display-day'
       })
+      willRenderDateRow = true
     }
     acc.lastDate = itemDate.toISODate()
 
     // -> Add session header row
     const typeName = `${item.type}-${item.slotName}`
-    if (item.type === 'regular' && acc.lastTypeName !== typeName) {
+    if (item.type === 'regular' && (acc.lastTypeName !== typeName || willRenderDateRow)) {
       acc.result.push({
         key: `sesshd-${item.id}`,
         displayType: 'session-head',

From 4bf1b938724abb775a5e9532740934dfa9a4dc40 Mon Sep 17 00:00:00 2001
From: Russ Housley <housley@vigilsec.com>
Date: Sat, 15 Mar 2025 05:38:23 -0400
Subject: [PATCH 282/656] fix: Set Review Type Name for IETF Last Call (#8678)

---
 ietf/name/fixtures/names.json               |  4 ++--
 ietf/name/migrations/0015_last_call_name.py | 22 +++++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)
 create mode 100644 ietf/name/migrations/0015_last_call_name.py

diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json
index 8f1262b4cf..96273fbc35 100644
--- a/ietf/name/fixtures/names.json
+++ b/ietf/name/fixtures/names.json
@@ -2628,7 +2628,7 @@
       "used": true
     },
     "model": "doc.state",
-    "pk": 182
+    "pk": 181
   },
   {
     "fields": {
@@ -13210,7 +13210,7 @@
   {
     "fields": {
       "desc": "",
-      "name": "Last Call",
+      "name": "IETF Last Call",
       "order": 2,
       "used": true
     },
diff --git a/ietf/name/migrations/0015_last_call_name.py b/ietf/name/migrations/0015_last_call_name.py
new file mode 100644
index 0000000000..ac210a274f
--- /dev/null
+++ b/ietf/name/migrations/0015_last_call_name.py
@@ -0,0 +1,22 @@
+# Copyright 2025, IETF Trust
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    ReviewTypeName = apps.get_model("name", "ReviewTypeName")
+    ReviewTypeName.objects.filter(slug="lc").update(name="IETF Last Call")
+
+def reverse(apps, schema_editor):
+    ReviewTypeName = apps.get_model("name", "ReviewTypeName")
+    ReviewTypeName.objects.filter(slug="lc").update(name="Last Call")
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("name", "0014_change_legacy_stream_desc"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse)
+    ]

From 44bb285836e032acb15d38febc82f4495a6dd9e8 Mon Sep 17 00:00:00 2001
From: Jim Fenton <fenton@bluepopcorn.net>
Date: Sun, 16 Mar 2025 14:37:56 +0700
Subject: [PATCH 283/656] fix: use correct variable to decide to show meeting
 notes (#8674)

* fix: use correct variable to decide to show session notes

* Correct number of lines for selftests

* fix: adjust test to match

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/meeting/tests_views.py                   | 28 +++++++++++--------
 .../meeting/session_details_panel.html        |  2 +-
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 519f5f7c2d..59d7e49f7f 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -424,37 +424,41 @@ def test_meeting_agenda(self):
         self.assertEqual(r.status_code, 200)
 
     def test_session_recordings_via_factories(self):
-        session = SessionFactory(meeting__type_id="ietf", meeting__date=date_today()-datetime.timedelta(days=180))
+        session = SessionFactory(meeting__type_id="ietf", meeting__date=date_today()-datetime.timedelta(days=180), meeting__number=str(random.randint(108,150)))
         self.assertEqual(session.meetecho_recording_name, "")
         self.assertEqual(len(session.recordings()), 0)
         url = urlreverse("ietf.meeting.views.session_details", kwargs=dict(num=session.meeting.number, acronym=session.group.acronym))
         r = self.client.get(url)
         q = PyQuery(r.content)
         # debug.show("q(f'#notes_and_recordings_{session.pk}')")
-        self.assertEqual(len(q(f"#notes_and_recordings_{session.pk} tr")), 1)
-        link = q(f"#notes_and_recordings_{session.pk} tr a")
-        self.assertEqual(len(link), 1)
-        self.assertEqual(link[0].attrib['href'], str(session.session_recording_url()))
+        self.assertEqual(len(q(f"#notes_and_recordings_{session.pk} tr")), 2)
+        links = q(f"#notes_and_recordings_{session.pk} tr a")
+        self.assertEqual(len(links), 2)
+        self.assertEqual(links[0].attrib['href'], str(session.notes_url()))
+        self.assertEqual(links[1].attrib['href'], str(session.session_recording_url()))
 
         session.meetecho_recording_name = 'my_test_session_name'
         session.save()
         r = self.client.get(url)
         q = PyQuery(r.content)
-        self.assertEqual(len(q(f"#notes_and_recordings_{session.pk} tr")), 1)
+        self.assertEqual(len(q(f"#notes_and_recordings_{session.pk} tr")), 2)
         links = q(f"#notes_and_recordings_{session.pk} tr a")
-        self.assertEqual(len(links), 1)
-        self.assertEqual(links[0].attrib['href'], session.session_recording_url())
+        self.assertEqual(len(links), 2)
+        self.assertEqual(links[0].attrib['href'], str(session.notes_url()))
+        self.assertEqual(links[1].attrib['href'], str(session.session_recording_url()))
 
         new_recording_url = "https://www.youtube.com/watch?v=jNQXAC9IVRw"
         new_recording_title = "Me at the zoo"
         create_recording(session, new_recording_url, new_recording_title)
         r = self.client.get(url)
         q = PyQuery(r.content)
-        self.assertEqual(len(q(f"#notes_and_recordings_{session.pk} tr")), 2)
+        self.assertEqual(len(q(f"#notes_and_recordings_{session.pk} tr")), 3)
         links = q(f"#notes_and_recordings_{session.pk} tr a")
-        self.assertEqual(len(links), 2)
-        self.assertEqual(links[0].attrib['href'], new_recording_url)
-        self.assertIn(new_recording_title, links[0].text_content())
+        self.assertEqual(len(links), 3)
+        self.assertEqual(links[0].attrib['href'], str(session.notes_url()))
+        self.assertEqual(links[1].attrib['href'], new_recording_url)
+        self.assertIn(new_recording_title, links[1].text_content())
+        self.assertEqual(links[2].attrib['href'], str(session.session_recording_url()))
         #debug.show("q(f'#notes_and_recordings_{session_pk}')")
 
     def test_delete_recordings(self):
diff --git a/ietf/templates/meeting/session_details_panel.html b/ietf/templates/meeting/session_details_panel.html
index 9b7a192f05..87d9e3d672 100644
--- a/ietf/templates/meeting/session_details_panel.html
+++ b/ietf/templates/meeting/session_details_panel.html
@@ -310,7 +310,7 @@ <h3 class="mt-4">Notes and recordings</h3>
         <table class="table table-sm table-striped meeting-tools"
                id="notes_and_recordings_{{ session.pk }}">
             <tbody>
-            {% if session.uses_notes %}
+            {% if meeting.uses_notes %}
                 <tr>
                     <td>
                         <a href="{{ session.notes_url }}">

From 2cb2ad15ba8500acb79b8dc62fd1903ad127a385 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 17 Mar 2025 17:09:38 -0400
Subject: [PATCH 284/656] chore(deps): bump nanoid in /dev/deploy-to-container
 in the npm group (#8689)

Bumps the npm group in /dev/deploy-to-container with 1 update: [nanoid](https://github.com/ai/nanoid).


Updates `nanoid` from 5.1.3 to 5.1.4
- [Release notes](https://github.com/ai/nanoid/releases)
- [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md)
- [Commits](https://github.com/ai/nanoid/compare/5.1.3...5.1.4)

---
updated-dependencies:
- dependency-name: nanoid
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/deploy-to-container/package-lock.json | 14 +++++++-------
 dev/deploy-to-container/package.json      |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json
index c4f675527e..f4ce2c367c 100644
--- a/dev/deploy-to-container/package-lock.json
+++ b/dev/deploy-to-container/package-lock.json
@@ -8,7 +8,7 @@
       "dependencies": {
         "dockerode": "^4.0.4",
         "fs-extra": "^11.3.0",
-        "nanoid": "5.1.3",
+        "nanoid": "5.1.4",
         "nanoid-dictionary": "5.0.0-beta.1",
         "slugify": "1.6.6",
         "tar": "^7.4.3",
@@ -668,9 +668,9 @@
       "optional": true
     },
     "node_modules/nanoid": {
-      "version": "5.1.3",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.3.tgz",
-      "integrity": "sha512-zAbEOEr7u2CbxwoMRlz/pNSpRP0FdAU4pRaYunCdEezWohXFs+a0Xw7RfkKaezMsmSM1vttcLthJtwRnVtOfHQ==",
+      "version": "5.1.4",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.4.tgz",
+      "integrity": "sha512-GTFcMIDgR7tqji/LpSY8rtg464VnJl/j6ypoehYnuGb+Y8qZUdtKB8WVCXon0UEZgFDbuUxpIl//6FHLHgXSNA==",
       "funding": [
         {
           "type": "github",
@@ -1613,9 +1613,9 @@
       "optional": true
     },
     "nanoid": {
-      "version": "5.1.3",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.3.tgz",
-      "integrity": "sha512-zAbEOEr7u2CbxwoMRlz/pNSpRP0FdAU4pRaYunCdEezWohXFs+a0Xw7RfkKaezMsmSM1vttcLthJtwRnVtOfHQ=="
+      "version": "5.1.4",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.4.tgz",
+      "integrity": "sha512-GTFcMIDgR7tqji/LpSY8rtg464VnJl/j6ypoehYnuGb+Y8qZUdtKB8WVCXon0UEZgFDbuUxpIl//6FHLHgXSNA=="
     },
     "nanoid-dictionary": {
       "version": "5.0.0-beta.1",
diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json
index 8772c568c8..eb8cb9012a 100644
--- a/dev/deploy-to-container/package.json
+++ b/dev/deploy-to-container/package.json
@@ -4,7 +4,7 @@
   "dependencies": {
     "dockerode": "^4.0.4",
     "fs-extra": "^11.3.0",
-    "nanoid": "5.1.3",
+    "nanoid": "5.1.4",
     "nanoid-dictionary": "5.0.0-beta.1",
     "slugify": "1.6.6",
     "tar": "^7.4.3",

From 8af0dc457f4dda714046b6151a722e7d20b1a55d Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 18 Mar 2025 08:30:52 +0700
Subject: [PATCH 285/656] fix: add doc to raw_id_fields for
 SlideSubmissionAdmin (#8687)

---
 ietf/meeting/admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/meeting/admin.py b/ietf/meeting/admin.py
index e975dd38a6..b7e56c7b77 100644
--- a/ietf/meeting/admin.py
+++ b/ietf/meeting/admin.py
@@ -189,7 +189,7 @@ class ImportantDateAdmin(admin.ModelAdmin):
 class SlideSubmissionAdmin(admin.ModelAdmin):
     model = SlideSubmission
     list_display = ['session', 'submitter', 'title']
-    raw_id_fields = ['submitter', 'session']
+    raw_id_fields = ['submitter', 'session', 'doc']
 
 admin.site.register(SlideSubmission, SlideSubmissionAdmin)
 

From 563fcee3a80d5036c9c490a794f31a78bd7a8e90 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 18 Mar 2025 14:53:38 +0700
Subject: [PATCH 286/656] fix: approve slides link for all pending submissions
 (#8692)

* test: slide approval links shown for all sessions

* fix: suggest pending slides for all sessions

* refactor: flatten logic

* refactor: avoid extra exists() query

* test: update test comment

* test: only pending slides suggested for approval

* style: Black

* test: temp path override for SessionDetailsTests
---
 ietf/meeting/tests_views.py                 | 80 +++++++++++++++++++++
 ietf/meeting/views.py                       | 14 ++--
 ietf/templates/meeting/session_details.html | 34 +++++----
 3 files changed, 104 insertions(+), 24 deletions(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 59d7e49f7f..111584cc56 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -4527,6 +4527,7 @@ def test_persistent_enabled_timeslot_types(self):
 
 
 class SessionDetailsTests(TestCase):
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['SLIDE_STAGING_PATH']
 
     def test_session_details(self):
 
@@ -4659,6 +4660,85 @@ def test_add_session_drafts(self):
         q = PyQuery(r.content)
         self.assertEqual(1,len(q(".alert-warning:contains('may affect published proceedings')")))
 
+    def test_proposed_slides_for_approval(self):
+        # This test overlaps somewhat with MaterialsTests of proposed slides handling. The focus
+        # here is on the display of slides, not the approval action.
+        group = GroupFactory()
+        meeting = MeetingFactory(
+            type_id="ietf", date=date_today() + datetime.timedelta(days=10)
+        )
+        sessions = SessionFactory.create_batch(
+            2,
+            group=group,
+            meeting=meeting,
+        )
+    
+        # slides submission _not_ in the `pending` state
+        do_not_show = [
+            SlideSubmissionFactory(
+                session=sessions[0],
+                title="already approved",
+                status_id="approved",
+            ),
+            SlideSubmissionFactory(
+                session=sessions[1],
+                title="already rejected",
+                status_id="rejected",
+            ),
+        ]
+    
+        # pending submissions
+        first_session_pending = SlideSubmissionFactory(
+            session=sessions[0], title="first session title"
+        )
+        second_session_pending = SlideSubmissionFactory(
+            session=sessions[1], title="second session title"
+        )
+    
+        # and their approval URLs
+        def _approval_url(slidesub):
+            return urlreverse(
+                "ietf.meeting.views.approve_proposed_slides",
+                kwargs={"slidesubmission_id": slidesub.pk, "num": meeting.number},
+            )
+    
+        first_approval_url = _approval_url(first_session_pending)
+        second_approval_url = _approval_url(second_session_pending)
+        do_not_show_urls = [_approval_url(ss) for ss in do_not_show]
+    
+        # Retrieve the URL as a group chair
+        url = urlreverse(
+            "ietf.meeting.views.session_details",
+            kwargs={
+                "num": meeting.number,
+                "acronym": group.acronym,
+            },
+        )
+        chair = RoleFactory(group=group, name_id="chair").person
+        self.client.login(
+            username=chair.user.username, password=f"{chair.user.username}+password"
+        )
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        pq = PyQuery(r.content)
+        self.assertEqual(
+            len(pq(f'a[href="{first_approval_url}"]')),
+            1,
+            "first session proposed slides should be linked for approval",
+        )
+        self.assertEqual(
+            len(pq(f'a[href="{second_approval_url}"]')),
+            1,
+            "second session proposed slides should be linked for approval",
+        )
+        for no_show_url in do_not_show_urls:
+            self.assertEqual(
+                len(pq(f'a[href="{no_show_url}"]')),
+                0,
+                "second session proposed slides should be linked for approval",
+            )
+        
+
 class EditScheduleListTests(TestCase):
     def setUp(self):
         super().setUp()
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 3fa605ed7e..d02ae40902 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -2509,12 +2509,14 @@ def session_details(request, num, acronym):
     scheduled_sessions = [s for s in sessions if s.current_status == 'sched']
     unscheduled_sessions = [s for s in sessions if s.current_status != 'sched']
 
-    pending_suggestions = None
-    if request.user.is_authenticated:
-        if can_manage:
-            pending_suggestions = session.slidesubmission_set.filter(status__slug='pending')
-        else:
-            pending_suggestions = session.slidesubmission_set.filter(status__slug='pending', submitter=request.user.person)
+    # Start with all the pending suggestions for all the group's sessions
+    pending_suggestions = SlideSubmission.objects.filter(session__in=sessions, status__slug='pending')
+    if can_manage:
+        pass  # keep the full set
+    elif hasattr(request.user, "person"):
+        pending_suggestions = pending_suggestions.filter(submitter=request.user.person)
+    else:
+        pending_suggestions = SlideSubmission.objects.none()
 
     return render(request, "meeting/session_details.html",
                   { 'scheduled_sessions':scheduled_sessions ,
diff --git a/ietf/templates/meeting/session_details.html b/ietf/templates/meeting/session_details.html
index 571715b4b4..55fa3d3857 100644
--- a/ietf/templates/meeting/session_details.html
+++ b/ietf/templates/meeting/session_details.html
@@ -31,30 +31,28 @@ <h2 class="mt-3">Scheduled Sessions</h2>
     {% include 'meeting/session_details_panel.html' with sessions=scheduled_sessions %}
     <h2 class="mt-3">Unscheduled Sessions</h2>
     {% include 'meeting/session_details_panel.html' with sessions=unscheduled_sessions %}
-    {% if pending_suggestions %}
-        <h2 class="mt-3">
+    {% for s in pending_suggestions %}
+        {% if forloop.first %}<h2 class="mt-3">
             {% if can_manage_materials %}
                 Proposed slides awaiting your approval
             {% else %}
                 Your proposed slides awaiting chair approval
             {% endif %}
         </h2>
-        <div class="proposedslidelist">
-            {% for s in pending_suggestions %}
-                {% if can_manage_materials %}
-                    <p>
-                        <a href="{% url "ietf.meeting.views.approve_proposed_slides" slidesubmission_id=s.pk num=s.session.meeting.number %}">
-                            {{ s.submitter }} - {{ s.title }} ({{ s.time }})
-                        </a>
-                    </p>
-                {% else %}
-                    <p>
-                        {{ s.title }} ({{ s.time }})
-                    </p>
-                {% endif %}
-            {% endfor %}
-        </div>
-    {% endif %}
+        <div class="proposedslidelist">{% endif %}
+            {% if can_manage_materials %}
+                <p>
+                    <a href="{% url "ietf.meeting.views.approve_proposed_slides" slidesubmission_id=s.pk num=s.session.meeting.number %}">
+                        {{ s.submitter }} - {{ s.title }} ({{ s.time }})
+                    </a>
+                </p>
+            {% else %}
+                <p>
+                    {{ s.title }} ({{ s.time }})
+                </p>
+            {% endif %}
+        {% if forloop.last %}</div>{% endif %}
+    {% endfor %}
 {% endblock %}
 {% block js %}
     <script src="{% static 'ietf/js/list.js' %}"></script>

From 73abdcc29798850a86ddeabdd65b824adcc62eb6 Mon Sep 17 00:00:00 2001
From: Rudi Matz <rudi@staff.ietf.org>
Date: Tue, 18 Mar 2025 22:25:10 -0400
Subject: [PATCH 287/656] feat(agenda): add preliminary date when no agenda
 available (#8690)

* feat(meetings): add preliminary agenda date when no agenda available

* test: adapt test for additional field for preliminary agenda date

---------

Co-authored-by: Rudi Matz <rudi@staff.ieft.org>
---
 client/agenda/AgendaScheduleList.vue | 1 +
 ietf/meeting/tests_views.py          | 5 +++--
 ietf/meeting/views.py                | 8 ++++++--
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index ab0f6e0184..369025d5da 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -15,6 +15,7 @@
           td(:colspan='pickerModeActive ? 6 : 5')
             i.bi.bi-exclamation-triangle.me-2
             span(v-if='agendaStore.searchVisible && agendaStore.searchText') No event matching your search query.
+            span(v-else-if='agendaStore.meeting.prelimAgendaDate') A preliminary agenda is expected to be released on {{ agendaStore.meeting.prelimAgendaDate }}
             span(v-else) Nothing to display
         tr(
           v-for='item of meetingEvents'
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 111584cc56..0f91986f77 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -233,6 +233,7 @@ def test_meeting_agenda(self):
         session.save()
         slot = TimeSlot.objects.get(sessionassignments__session=session,sessionassignments__schedule=meeting.schedule)
         meeting.timeslot_set.filter(type_id="break").update(show_location=False)
+        meeting.importantdate_set.create(name_id='prelimagenda',date=date_today() + datetime.timedelta(days=20))
         #
         self.write_materials_files(meeting, session)
         #
@@ -262,7 +263,8 @@ def test_meeting_agenda(self):
                     "updated": generated_data.get("meeting").get("updated"),  # Just expect the value to exist
                     "timezone": meeting.time_zone,
                     "infoNote": meeting.agenda_info_note,
-                    "warningNote": meeting.agenda_warning_note
+                    "warningNote": meeting.agenda_warning_note,
+                    "prelimAgendaDate": (date_today() + datetime.timedelta(days=20)).isoformat()
                 },
                 "categories": generated_data.get("categories"),  # Just expect the value to exist
                 "isCurrentMeeting": True,
@@ -9341,4 +9343,3 @@ def test_bluesheet_data(self):
                 {"name": attended_with_affil.person.plain_name(), "affiliation": "Somewhere"},
             ]
         )
-
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index d02ae40902..6a73059d92 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -59,7 +59,7 @@
 from ietf.ietfauth.utils import role_required, has_role, user_is_person
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, SessionPresentation, TimeSlot, SlideSubmission, Attended
-from ietf.meeting.models import SessionStatusName, SchedulingEvent, SchedTimeSessAssignment, Room, TimeSlotTypeName
+from ietf.meeting.models import ImportantDate, SessionStatusName, SchedulingEvent, SchedTimeSessAssignment, Room, TimeSlotTypeName
 from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm, ImportMinutesForm,
                                  TimeSlotCreateForm, TimeSlotEditForm, SessionCancelForm, SessionEditForm )
 from ietf.meeting.helpers import get_person_by_email, get_schedule_by_name
@@ -1709,6 +1709,9 @@ def generate_agenda_data(num=None, force_refresh=False):
     # Get Floor Plans
     floors = FloorPlan.objects.filter(meeting=meeting).order_by('order')
     
+    # Get Preliminary Agenda Date
+    prelimAgendaDate = ImportantDate.objects.filter(name_id="prelimagenda", meeting=meeting).first()
+
     result = {
         "meeting": {
             "number": schedule.meeting.number,
@@ -1718,7 +1721,8 @@ def generate_agenda_data(num=None, force_refresh=False):
             "updated": updated,
             "timezone": meeting.time_zone,
             "infoNote": schedule.meeting.agenda_info_note,
-            "warningNote": schedule.meeting.agenda_warning_note
+            "warningNote": schedule.meeting.agenda_warning_note,
+            "prelimAgendaDate": prelimAgendaDate.date.isoformat() if prelimAgendaDate else ""
         },
         "categories": filter_organizer.get_filter_categories(),
         "isCurrentMeeting": is_current_meeting,

From 3356505dc89e2a372f54c3ed1da472c3a11bc6de Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Wed, 19 Mar 2025 15:28:17 +1300
Subject: [PATCH 288/656] fix(agenda): Agenda 'now' fallback to next event if
 there is no current event (#8693)

* fix: agenda now fallback to next event if there is no current event

* chore: agenda goto now PR feedback

* chore: simplifying 'agenda goto now' next event logic

* chore: simplifying 'agenda goto now' nextEvent var
---
 client/agenda/AgendaMobileBar.vue    |  8 ++++----
 client/agenda/AgendaQuickAccess.vue  |  8 ++++----
 client/agenda/AgendaScheduleList.vue | 16 ++++++++++------
 client/agenda/store.js               | 22 ++++++++++++++++++++++
 4 files changed, 40 insertions(+), 14 deletions(-)

diff --git a/client/agenda/AgendaMobileBar.vue b/client/agenda/AgendaMobileBar.vue
index 63611e21c2..43480bedd3 100644
--- a/client/agenda/AgendaMobileBar.vue
+++ b/client/agenda/AgendaMobileBar.vue
@@ -124,11 +124,11 @@ const downloadIcsOptions = [
 
 function jumpToDay (dayId) {
   if (dayId === 'now') {
-    const lastEventId = agendaStore.findCurrentEventId()
-    if (lastEventId) {
-      document.getElementById(`agenda-rowid-${lastEventId}`)?.scrollIntoView(true)
+    const nowEventId = agendaStore.findNowEventId()
+    if (nowEventId) {
+      document.getElementById(`agenda-rowid-${nowEventId}`)?.scrollIntoView(true)
     } else {
-      message.warning('There is no event happening right now.')
+      message.warning('There is no event happening right now or in the future.')
     }
   } else {
     document.getElementById(dayId)?.scrollIntoView(true)
diff --git a/client/agenda/AgendaQuickAccess.vue b/client/agenda/AgendaQuickAccess.vue
index b226d09c60..c9412f6663 100644
--- a/client/agenda/AgendaQuickAccess.vue
+++ b/client/agenda/AgendaQuickAccess.vue
@@ -204,12 +204,12 @@ function scrollToDay (daySlug, ev) {
 }
 
 function scrollToNow (ev) {
-  const lastEventId = agendaStore.findCurrentEventId()
+  const nowEventId = agendaStore.findNowEventId()
 
-  if (lastEventId) {
-    document.getElementById(`agenda-rowid-${lastEventId}`)?.scrollIntoView(true)
+  if (nowEventId) {
+    document.getElementById(`agenda-rowid-${nowEventId}`)?.scrollIntoView(true)
   } else {
-    message.warning('There is no event happening right now.')
+    message.warning('There is no event happening right now or in the future.')
   }
 }
 
diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index 369025d5da..5a971c25cf 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -591,10 +591,10 @@ function renderLinkLabel (opt) {
 
 function recalculateRedLine () {
   state.currentMinute = DateTime.local().minute
-  const lastEventId = agendaStore.findCurrentEventId()
+  const currentEventId = agendaStore.findCurrentEventId()
 
-  if (lastEventId) {
-    state.redhandOffset = document.getElementById(`agenda-rowid-${lastEventId}`)?.offsetTop || 0
+  if (currentEventId) {
+    state.redhandOffset = document.getElementById(`agenda-rowid-${currentEventId}`)?.offsetTop || 0
   } else {
     state.redhandOffset = 0
   }
@@ -615,9 +615,13 @@ function recalculateRedLine () {
       return
     }
     unsubscribe() // we only need to scroll once, so unsubscribe from future updates
-    if(window.location.hash === "#now") {
-      const lastEventId = agendaStore.findCurrentEventId()
-      document.getElementById(`agenda-rowid-${lastEventId}`)?.scrollIntoView(true)
+    if (window.location.hash === "#now") {
+      const nowEventId = agendaStore.findNowEvent()
+      if (nowEventId) {
+        document.getElementById(`agenda-rowid-${nowEventId}`)?.scrollIntoView(true)
+      } else {
+        message.warning('There is no event happening right now or in the future.')
+      }
     } else if(window.location.hash.startsWith(`#${daySlugPrefix}`)) {
       document.getElementById(window.location.hash.substring(1))?.scrollIntoView(true)
     }
diff --git a/client/agenda/store.js b/client/agenda/store.js
index 71c1219725..359c5fbf05 100644
--- a/client/agenda/store.js
+++ b/client/agenda/store.js
@@ -230,6 +230,28 @@ export const useAgendaStore = defineStore('agenda', {
 
       return lastEvent.id || null
     },
+    findNowEventId () {
+      const currentEventId = this.findCurrentEventId()
+      
+      if (currentEventId) {
+        return currentEventId
+      }
+
+      // if there isn't a current event then instead find the next event
+
+      const current = (this.nowDebugDiff ? DateTime.local().minus(this.nowDebugDiff) : DateTime.local()).setZone(this.timezone)
+
+      // -> Find next event after current time
+      let nextEventId = undefined
+      for(const sh of this.scheduleAdjusted) {
+        if (sh.adjustedStart > current) {
+          nextEventId = sh.id
+          break
+        }
+      }
+
+      return nextEventId || null
+    },
     hideLoadingScreen () {
       // -> Hide loading screen
       const loadingRef = document.querySelector('#app-loading')

From c3a296fdb559b015f8e4d457f56fea137e10cf37 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 19 Mar 2025 04:56:59 -0400
Subject: [PATCH 289/656] fix(agenda): handle calendar view events that spread
 across multiple days (#8685)

---
 client/agenda/AgendaScheduleCalendar.vue | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/client/agenda/AgendaScheduleCalendar.vue b/client/agenda/AgendaScheduleCalendar.vue
index 9b56b7f5a7..6701ddabd7 100644
--- a/client/agenda/AgendaScheduleCalendar.vue
+++ b/client/agenda/AgendaScheduleCalendar.vue
@@ -187,6 +187,7 @@ function refreshData () {
   let earliestDate = DateTime.fromISO('2200-01-01')
   let latestDate = DateTime.fromISO('1990-01-01')
   let nowDate = DateTime.now()
+  let hasCrossDayEvents = false
 
   calendarOptions.events = agendaStore.scheduleAdjusted.map(ev => {
     // -> Determine boundaries
@@ -202,6 +203,9 @@ function refreshData () {
     if (ev.adjustedEnd < latestDate) {
       latestDate = ev.adjustedEnd
     }
+    if (ev.adjustedStart.day !== ev.adjustedEnd.day) {
+      hasCrossDayEvents = true
+    }
     // -> Build event object
     return {
       id: ev.id,
@@ -214,8 +218,8 @@ function refreshData () {
   })
 
   // -> Display settings
-  calendarOptions.slotMinTime = `${earliestHour.toString().padStart(2, '0')}:00:00`
-  calendarOptions.slotMaxTime = `${latestHour.toString().padStart(2, '0')}:00:00`
+  calendarOptions.slotMinTime = hasCrossDayEvents ? '00:00:00' : `${earliestHour.toString().padStart(2, '0')}:00:00`
+  calendarOptions.slotMaxTime = hasCrossDayEvents ? '23:59:59' : `${latestHour.toString().padStart(2, '0')}:00:00`
   calendarOptions.validRange.start = earliestDate.minus({ days: 1 }).toISODate()
   calendarOptions.validRange.end = latestDate.plus({ days: 1 }).toISODate()
   // calendarOptions.scrollTime = `${earliestHour.toString().padStart(2, '0')}:00:00`

From fde8136df53b3b9d2666674e16e15a7edf7e0ed1 Mon Sep 17 00:00:00 2001
From: Rich Salz <rsalz@akamai.com>
Date: Wed, 19 Mar 2025 05:06:47 -0400
Subject: [PATCH 290/656] chore: Add draft name to review-completed email
 (#8676)

* chore: Add draft name to review-completed email

Fixes: #7866

* fix: typo

* fix: show title and add test

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/doc/tests_review.py                   | 5 ++++-
 ietf/templates/review/completed_review.txt | 4 +++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py
index e93bc02181..13ddbc22ba 100644
--- a/ietf/doc/tests_review.py
+++ b/ietf/doc/tests_review.py
@@ -902,7 +902,10 @@ def test_complete_review_enter_content(self):
 
         self.assertEqual(len(outbox), 1)
         self.assertIn(assignment.review_request.team.list_email, outbox[0]["To"])
-        self.assertIn("This is a review", get_payload_text(outbox[0]))
+        payload = get_payload_text(outbox[0])
+        self.assertIn("This is a review", payload)
+        self.assertIn(f"Document: {assignment.review_request.doc.name}", payload)
+        self.assertIn(f"Title: {assignment.review_request.doc.title}", payload)
 
         self.assertIn(settings.MAILING_LIST_ARCHIVE_URL, assignment.review.external_url)
 
diff --git a/ietf/templates/review/completed_review.txt b/ietf/templates/review/completed_review.txt
index bdbe321ca1..7d10d8bf13 100644
--- a/ietf/templates/review/completed_review.txt
+++ b/ietf/templates/review/completed_review.txt
@@ -1,7 +1,9 @@
 {% load ietf_filters %}{% autoescape off %}{% filter maybewordwrap:80 %}{% if assignment.state_id == "part-completed" %}
 Review is partially done. Another assignment may be needed to complete it.
 
-{% endif %}Reviewer: {{ assignment.reviewer.person }}
+{% endif %}Document: {{ assignment.review_request.doc.name }}
+Title: {{ assignment.review_request.doc.title }}
+Reviewer: {{ assignment.reviewer.person }}
 Review result: {{ assignment.result.name }}
 
 {{ content }}

From 2d974ed09b8e9ff93805fb06cfa65ab6f814e261 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Fri, 21 Mar 2025 13:45:27 +1300
Subject: [PATCH 291/656] feat(agenda): Agenda meeting materials in overflow
 menu (#8698)

* feat: agenda meeting materials in overflow menu

* chore: fixing agenda meeting materials tests

* chore: fix agenda meeting material tests

* chore: fixing agenda meeting materials test

* fix: changing to -mat suffix rather than -meeting-materials

* fix: agenda meeting materials lnk -> btn

* fix: click handler on agenda meeting materials
---
 client/agenda/AgendaScheduleList.vue    | 60 ++++++++++++++++++++-----
 playwright/tests/meeting/agenda.spec.js |  8 ++--
 2 files changed, 52 insertions(+), 16 deletions(-)

diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index 5a971c25cf..1e50df5fb4 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -121,20 +121,12 @@
                     :options='item.links'
                     key-field='id'
                     :render-icon='renderLinkIcon'
-                    :render-label='renderLinkLabel'
+                    :render-label='renderLink'
                     )
                     n-button(size='tiny')
                       i.bi.bi-three-dots
                 .agenda-table-cell-links-buttons(v-else-if='item.links && item.links.length > 0')
-                    template(v-if='item.flags.agenda')
-                      n-popover
-                        template(#trigger)
-                          i.bi.bi-collection(
-                            :id='`btn-lnk-` + item.key + `-mat`'
-                            @click='showMaterials(item.key)'
-                            )
-                        span Show meeting materials
-                    template(v-else-if='item.type === `regular`')
+                    template(v-if='!item.flags.agenda && item.type === `regular`')
                       n-popover
                         template(#trigger)
                           i.no-meeting-materials
@@ -143,7 +135,16 @@
                         span No meeting materials yet.
                     n-popover(v-for='lnk of item.links', :key='lnk.id')
                       template(#trigger)
+                        button(
+                          v-if="lnk.click"
+                          type="button"
+                          :id='`btn-` + lnk.id'
+                          @click='lnk.click'
+                          :aria-label='lnk.label'
+                          :class='`border-0 bg-transparent text-` + lnk.color'
+                          ): i.bi(:class='`bi-` + lnk.icon')
                         a(
+                          v-else
                           :id='`btn-` + lnk.id'
                           :href='lnk.href'
                           :aria-label='lnk.label'
@@ -281,13 +282,28 @@ const meetingEvents = computed(() => {
     }
     acc.lastTypeName = typeName
 
-    // -> Populate event links
+    // 
+    /**
+     * -> Populate event menu items
+     * 
+     * links is an array of either,
+     * 1. { href: "...",        click: undefined, ...sharedProps }
+     * 2. { click: () => {...}, href: undefined,  ...sharedProps }
+     */
     const links = []
     const typesWithLinks = ['regular', 'plenary', 'other']
     const purposesWithoutLinks = ['admin', 'closed_meeting', 'officehours', 'social']
     if (item.flags.showAgenda || (typesWithLinks.includes(item.type) && !purposesWithoutLinks.includes(item.purpose))) {
       if (item.flags.agenda) {
         // -> Meeting Materials
+        links.push({
+          id: `btn-${item.id}-mat`,
+          label: 'Show meeting materials',
+          icon: 'collection',
+          href: undefined,
+          click: () => showMaterials(item.id),
+          color: 'black'
+        })
         links.push({
           id: `lnk-${item.id}-tar`,
           label: 'Download meeting materials as .tar archive',
@@ -585,7 +601,11 @@ function renderLinkIcon (opt) {
   return h('i', { class: `bi bi-${opt.icon} text-${opt.color}` })
 }
 
-function renderLinkLabel (opt) {
+function renderLink (opt) {
+  if (opt.click) {
+    return h('button', { type: 'button', class: 'overflow-button', onClick: opt.click }, opt.label)
+  }
+
   return h('a', { href: opt.href, target: '_blank' }, opt.label)
 }
 
@@ -1564,6 +1584,22 @@ onBeforeUnmount(() => {
   }
 }
 
+.overflow-button {
+  font-size: inherit;
+  padding: 0;
+  border: 0;
+  background: transparent;
+
+  &:before {
+    content: "";
+    position: absolute;
+    left: 0;
+    right: 0;
+    top: 0;
+    bottom: 0;
+  }
+}
+
 @keyframes fadeInAnim {
   0% {
     opacity: 0;
diff --git a/playwright/tests/meeting/agenda.spec.js b/playwright/tests/meeting/agenda.spec.js
index e6c6e17031..b7e3df5718 100644
--- a/playwright/tests/meeting/agenda.spec.js
+++ b/playwright/tests/meeting/agenda.spec.js
@@ -275,7 +275,7 @@ test.describe('past - desktop', () => {
             const eventButtons = row.locator('.agenda-table-cell-links > .agenda-table-cell-links-buttons')
             if (event.flags.agenda) {
               // Show meeting materials button
-              await expect(eventButtons.locator('i.bi.bi-collection')).toBeVisible()
+              await expect(eventButtons.locator(`#btn-btn-${event.id}-mat`)).toBeVisible()
               // ZIP materials button
               await expect(eventButtons.locator(`#btn-lnk-${event.id}-tar`)).toHaveAttribute('href', `/meeting/${meetingData.meeting.number}/agenda/${event.acronym}-drafts.tgz`)
               await expect(eventButtons.locator(`#btn-lnk-${event.id}-tar > i.bi`)).toBeVisible()
@@ -425,7 +425,7 @@ test.describe('past - desktop', () => {
       })
     })
     // Open dialog
-    await page.locator(`#agenda-rowid-${event.id} #btn-lnk-${event.id}-mat`).click()
+    await page.locator(`#agenda-rowid-${event.id} #btn-btn-${event.id}-mat`).click()
     await expect(page.locator('.agenda-eventdetails')).toBeVisible()
     // Header
     await expect(page.locator('.agenda-eventdetails .n-card-header__main > .detail-header > .bi')).toBeVisible()
@@ -507,7 +507,7 @@ test.describe('past - desktop', () => {
       })
     })
     // Open dialog
-    await page.locator(`#btn-lnk-${event.id}-mat`).click()
+    await page.locator(`#btn-btn-${event.id}-mat`).click()
     await expect(page.locator('.agenda-eventdetails')).toBeVisible()
     // Slides Tab
     await page.locator('.agenda-eventdetails .detail-nav > a').nth(1).click()
@@ -1158,7 +1158,7 @@ test.describe('future - desktop', () => {
         if (event.flags.showAgenda || (['regular', 'plenary', 'other'].includes(event.type) && !['admin', 'closed_meeting', 'officehours', 'social'].includes(event.purpose))) {
           if (event.flags.agenda) {
             // Show meeting materials button
-            await expect(eventButtons.locator('i.bi.bi-collection')).toBeVisible()
+            await expect(eventButtons.locator(`#btn-btn-${event.id}-mat`)).toBeVisible()
             // ZIP materials button
             await expect(eventButtons.locator(`#btn-lnk-${event.id}-tar`)).toHaveAttribute('href', `/meeting/${meetingData.meeting.number}/agenda/${event.acronym}-drafts.tgz`)
             await expect(eventButtons.locator(`#btn-lnk-${event.id}-tar > i.bi`)).toBeVisible()

From 27ea6234a8b0b74062c0a57e849e316087c849f3 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 21 Mar 2025 04:43:33 -0400
Subject: [PATCH 292/656] feat(agenda): keep video client + onsite tool buttons
 for plenary for rest of day (#8706)

---
 client/agenda/AgendaScheduleList.vue    |  17 ++++
 playwright/helpers/common.js            |  24 ++++++
 playwright/helpers/meeting.js           |   3 +
 playwright/tests/meeting/agenda.spec.js | 102 ++++++++++++++++++++----
 4 files changed, 129 insertions(+), 17 deletions(-)

diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index 1e50df5fb4..86c91bed85 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -464,6 +464,23 @@ const meetingEvents = computed(() => {
               color: 'purple'
             })
           }
+          // -> Keep showing video client / on-site tool for Plenary until end of day, in case it goes over the planned time range
+          if (item.type === 'plenary' && item.adjustedEnd.day === current.day) {
+            links.push({
+              id: `lnk-${item.id}-video`,
+              label: 'Full Client with Video',
+              icon: 'camera-video',
+              href: item.links.videoStream,
+              color: 'purple'
+            })
+            links.push({
+              id: `lnk-${item.id}-onsitetool`,
+              label: 'Onsite tool',
+              icon: 'telephone-outbound',
+              href: item.links.onsiteTool,
+              color: 'teal'
+            })
+          }
         }
       }
     }
diff --git a/playwright/helpers/common.js b/playwright/helpers/common.js
index 5ba39ba022..c4dd7e2640 100644
--- a/playwright/helpers/common.js
+++ b/playwright/helpers/common.js
@@ -13,5 +13,29 @@ module.exports = {
 
       return rect.top < bottom && rect.top > 0 - rect.height
     })
+  },
+  /**
+   * Override page DateTime with a new value
+   *
+   * @param {Object} page Page object
+   * @param {Object} dateTimeOverride New DateTime object
+   */
+  overridePageDateTime: async (page, dateTimeOverride) => {
+    await page.addInitScript(`{
+      // Extend Date constructor to default to fixed time
+      Date = class extends Date {
+        constructor(...args) {
+          if (args.length === 0) {
+            super(${dateTimeOverride.toMillis()});
+          } else {
+            super(...args);
+          }
+        }
+      }
+      // Override Date.now() to start from fixed time
+      const __DateNowOffset = ${dateTimeOverride.toMillis()} - Date.now();
+      const __DateNow = Date.now;
+      Date.now = () => __DateNow() + __DateNowOffset;
+    }`)
   }
 }
diff --git a/playwright/helpers/meeting.js b/playwright/helpers/meeting.js
index 9722ffc68b..634ca2e8c6 100644
--- a/playwright/helpers/meeting.js
+++ b/playwright/helpers/meeting.js
@@ -609,6 +609,9 @@ module.exports = {
             startDateTime: curDay.set({ hour: 17, minute: 30 }),
             duration: '2h',
             type: 'plenary',
+            showAgenda: true,
+            hasAgenda: true,
+            hasRecordings: true,
             ...findAreaGroup('ietf-plenary', categories[2])
           }, floors))
         }
diff --git a/playwright/tests/meeting/agenda.spec.js b/playwright/tests/meeting/agenda.spec.js
index b7e3df5718..412a3fe9b8 100644
--- a/playwright/tests/meeting/agenda.spec.js
+++ b/playwright/tests/meeting/agenda.spec.js
@@ -1213,7 +1213,7 @@ test.describe('future - desktop', () => {
             await expect(eventButtons.locator(`#btn-lnk-${event.id}-remotecallin`)).toHaveAttribute('href', remoteCallInUrl)
             await expect(eventButtons.locator(`#btn-lnk-${event.id}-remotecallin > i.bi`)).toBeVisible()
           }
-          // calendar
+          // Calendar
           if (event.links.calendar) {
             await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar`)).toHaveAttribute('href', event.links.calendar)
             await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar > i.bi`)).toBeVisible()
@@ -1278,22 +1278,7 @@ test.describe('live - desktop', () => {
     })
 
     // Override Date in page to fixed time
-    await page.addInitScript(`{
-      // Extend Date constructor to default to fixed time
-      Date = class extends Date {
-        constructor(...args) {
-          if (args.length === 0) {
-            super(${currentTime.toMillis()});
-          } else {
-            super(...args);
-          }
-        }
-      }
-      // Override Date.now() to start from fixed time
-      const __DateNowOffset = ${currentTime.toMillis()} - Date.now();
-      const __DateNow = Date.now;
-      Date.now = () => __DateNow() + __DateNowOffset;
-    }`)
+    await commonHelper.overridePageDateTime(page, currentTime)
 
     // Visit agenda page and await Meeting Data API call to complete
     await Promise.all([
@@ -1348,6 +1333,89 @@ test.describe('live - desktop', () => {
   })
 })
 
+// ====================================================================
+// AGENDA (live meeting) | DESKTOP viewport | Plenary Extended Time Buttons
+// ====================================================================
+
+test.describe('live - desktop - plenary extended time buttons', () => {
+  let meetingData
+  let plenarySessionId
+
+  test.beforeAll(async () => {
+    // Generate meeting data
+    meetingData = meetingHelper.generateAgendaResponse({ dateMode: 'current' })
+    plenarySessionId = meetingData.schedule.find(s => s.type === 'plenary').id
+  })
+
+  test.beforeEach(async ({ page }) => {
+    // Intercept Meeting Data API
+    await page.route(`**/api/meeting/${meetingData.meeting.number}/agenda-data`, route => {
+      route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify(meetingData)
+      })
+    })
+
+    await page.setViewportSize({
+      width: viewports.desktop[0],
+      height: viewports.desktop[1]
+    })
+  })
+
+  // -> BUTTONS PRESENT AFTER EVENT, SAME DAY
+
+  test('same day - after event', async ({ page }) => {
+    // Override Date in page to fixed time
+    const currentTime = DateTime.fromISO('2022-02-01T13:45:15', { zone: 'Asia/Tokyo' }).plus({ days: 1 }).set({ hour: 20, minute: 30 })
+    await commonHelper.overridePageDateTime(page, currentTime)
+
+    // Visit agenda page and await Meeting Data API call to complete
+    await Promise.all([
+      page.waitForResponse(`**/api/meeting/${meetingData.meeting.number}/agenda-data`),
+      page.goto(`/meeting/${meetingData.meeting.number}/agenda`)
+    ])
+
+    // Wait for page to be ready
+    await page.locator('.agenda h1').waitFor({ state: 'visible' })
+    await setTimeout(500)
+
+    // Check for plenary event
+    await expect(page.locator('.agenda .agenda-table-display-event.agenda-table-type-plenary')).toBeVisible()
+    await page.locator('.agenda .agenda-table-display-event.agenda-table-type-plenary').scrollIntoViewIfNeeded()
+
+    // Check for full video client + on-site tool
+    await expect(page.locator(`.agenda .agenda-table-display-event.agenda-table-type-plenary .agenda-table-cell-links-buttons a#btn-lnk-${plenarySessionId}-video`)).toBeVisible()
+    await expect(page.locator(`.agenda .agenda-table-display-event.agenda-table-type-plenary .agenda-table-cell-links-buttons a#btn-lnk-${plenarySessionId}-onsitetool`)).toBeVisible()
+  })
+
+  // -> BUTTONS NO LONGER PRESENT AFTER EVENT, NEXT DAY
+
+  test('next day - after event', async ({ page }) => {
+    // Override Date in page to fixed time
+    const currentTime = DateTime.fromISO('2022-02-01T13:45:15', { zone: 'Asia/Tokyo' }).plus({ days: 2 }).set({ hour: 2, minute: 30 })
+    await commonHelper.overridePageDateTime(page, currentTime)
+
+    // Visit agenda page and await Meeting Data API call to complete
+    await Promise.all([
+      page.waitForResponse(`**/api/meeting/${meetingData.meeting.number}/agenda-data`),
+      page.goto(`/meeting/${meetingData.meeting.number}/agenda`)
+    ])
+
+    // Wait for page to be ready
+    await page.locator('.agenda h1').waitFor({ state: 'visible' })
+    await setTimeout(500)
+
+    // Check for plenary event
+    await expect(page.locator('.agenda .agenda-table-display-event.agenda-table-type-plenary')).toBeVisible()
+    await page.locator('.agenda .agenda-table-display-event.agenda-table-type-plenary').scrollIntoViewIfNeeded()
+
+    // Check for full video client + on-site tool
+    await expect(page.locator(`.agenda .agenda-table-display-event.agenda-table-type-plenary .agenda-table-cell-links-buttons a#btn-lnk-${plenarySessionId}-video`)).not.toBeVisible()
+    await expect(page.locator(`.agenda .agenda-table-display-event.agenda-table-type-plenary .agenda-table-cell-links-buttons a#btn-lnk-${plenarySessionId}-onsitetool`)).not.toBeVisible()
+  })
+})
+
 // ====================================================================
 // AGENDA (past meeting) | SMALL DESKTOP/TABLET/MOBILE viewports
 // ====================================================================

From ed2059a5813130cbf31ce77c0978a35751addc6c Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 21 Mar 2025 15:45:26 +0700
Subject: [PATCH 293/656] fix: add blank=true to SlideSubmission.doc (#8688)

Allows admin to save an instance with a null doc reference, which was already permitted.
---
 .../0011_alter_slidesubmission_doc.py         | 26 +++++++++++++++++++
 ietf/meeting/models.py                        |  2 +-
 2 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 ietf/meeting/migrations/0011_alter_slidesubmission_doc.py

diff --git a/ietf/meeting/migrations/0011_alter_slidesubmission_doc.py b/ietf/meeting/migrations/0011_alter_slidesubmission_doc.py
new file mode 100644
index 0000000000..b9cbc58e99
--- /dev/null
+++ b/ietf/meeting/migrations/0011_alter_slidesubmission_doc.py
@@ -0,0 +1,26 @@
+# Generated by Django 4.2.19 on 2025-03-17 09:37
+
+from django.db import migrations
+import django.db.models.deletion
+import ietf.utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("doc", "0025_storedobject_storedobject_unique_name_per_store"),
+        ("meeting", "0010_alter_floorplan_image_alter_meetinghost_logo"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="slidesubmission",
+            name="doc",
+            field=ietf.utils.models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to="doc.document",
+            ),
+        ),
+    ]
diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index 5284420731..84d151c310 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -1385,7 +1385,7 @@ class SlideSubmission(models.Model):
     apply_to_all = models.BooleanField(default=False)
     submitter = ForeignKey(Person)
     status      = ForeignKey(SlideSubmissionStatusName, null=True, default='pending', on_delete=models.SET_NULL)
-    doc         = ForeignKey(Document, null=True, on_delete=models.SET_NULL)
+    doc         = ForeignKey(Document, blank=True, null=True, on_delete=models.SET_NULL)
 
     def staged_filepath(self):
         return os.path.join(settings.SLIDE_STAGING_PATH , self.filename)

From 37b4448b3d390b102b7e14890880bcfd1e965bcc Mon Sep 17 00:00:00 2001
From: Benson Muite <bkmgit@users.noreply.github.com>
Date: Fri, 21 Mar 2025 11:52:18 +0300
Subject: [PATCH 294/656] fix: Clarify legacy description in RFC banner (#8424)
 (#8670)

---
 ietf/templates/doc/disclaimer.html | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/ietf/templates/doc/disclaimer.html b/ietf/templates/doc/disclaimer.html
index 31ce6c397e..db4c42ed68 100644
--- a/ietf/templates/doc/disclaimer.html
+++ b/ietf/templates/doc/disclaimer.html
@@ -3,12 +3,18 @@
 {% load ietf_filters %}
 {% origin %}
 {% if doc.type_id == "rfc" %}
-    {% if doc.stream.slug != "ietf" and doc.std_level.slug|default:"unk" not in "bcp,ds,ps,std"|split:"," %}
+    {% if doc.stream.slug != "ietf" and doc.stream.desc != "Legacy" and doc.std_level.slug|default:"unk" not in "bcp,ds,ps,std"|split:"," %}
         <div class="alert alert-warning {% if document_html %}small p-2 mt-2{% endif %}" role="alert">
             This RFC was published on the {{ doc.stream.desc }} stream.
             This RFC is <strong>not endorsed by the IETF</strong> and has <strong>no formal standing</strong> in the
             <a href="{% url 'ietf.doc.views_doc.document_main' name='rfc2026' %}">IETF standards process</a>.
         </div>
+    {% elif doc.stream.slug != "ietf" and doc.stream.desc == "Legacy" and doc.std_level.slug|default:"unk" not in "bcp,ds,ps,std"|split:"," %}	
+        <div class="alert alert-warning {% if document_html %}small p-2 mt-2{% endif %}" role="alert">
+            This RFC is labeled as "{{ doc.stream.desc }}"; it was published before a formal source was recorded.
+            This RFC is <strong>not endorsed by the IETF</strong> and has <strong>no formal standing</strong> in the
+            <a href="{% url 'ietf.doc.views_doc.document_main' name='rfc2026' %}">IETF standards process</a>.
+        </div>
     {% endif %}
 {% elif doc|is_in_stream %}
     {% if doc.stream.slug != "ietf" and doc.std_level.slug|default:"unk" not in "bcp,ds,ps,std"|split:"," %}
@@ -25,4 +31,4 @@
         This I-D is <strong>not endorsed by the IETF</strong> and has <strong>no formal standing</strong> in the
         <a href="{% url 'ietf.doc.views_doc.document_main' name='rfc2026' %}">IETF standards process</a>.
     </div>
-{% endif %}
\ No newline at end of file
+{% endif %}

From 8fec17282f5dc7ef28355990e2f74bff2a309002 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 21 Mar 2025 19:30:40 -0400
Subject: [PATCH 295/656] ci: Update build-base-app.yml

---
 .github/workflows/build-base-app.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index c8f66a22b7..5f0a0d11b8 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -51,7 +51,7 @@ jobs:
         push: true
         tags: |
           ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }}
-          ghcr.io/ietf-tools/datatracker-app-base:latest
+          ${{ github.ref == 'refs/heads/main' && 'ghcr.io/ietf-tools/datatracker-app-base:latest' || '' }}
 
     - name: Update version references
       run: |
@@ -61,6 +61,6 @@ jobs:
     - name: Commit CHANGELOG.md
       uses: stefanzweifel/git-auto-commit-action@v5
       with:
-        branch: main
+        branch: ${{ github.ref_name }}
         commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}'
         file_pattern: dev/build/Dockerfile dev/build/TARGET_BASE

From 6da36da3fde2721d3885001e4a4174c85579db8e Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 27 Mar 2025 00:29:48 +0700
Subject: [PATCH 296/656] chore: bump rabbitmq to 3.13 (#8603)

* ci: param for rabbitmq_version

* ci: drop latest tagging from datatracker-mq

* chore: bump rabbitmq to 3.13

* Revert "ci: drop latest tagging from datatracker-mq"

This reverts commit 58cea207c7300727f9cfc19a57859ab16c3b2c01.
---
 .github/workflows/build-mq-broker.yml | 15 +++++++++------
 k8s/rabbitmq.yaml                     |  2 +-
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml
index 8c6f1e6ae1..85c27c23cc 100644
--- a/.github/workflows/build-mq-broker.yml
+++ b/.github/workflows/build-mq-broker.yml
@@ -8,10 +8,13 @@ on:
       - 'dev/mq/**'
       - '.github/workflows/build-mq-broker.yml'
 
-  workflow_dispatch: 
-
-env:
-  RABBITMQ_VERSION: 3.12-alpine
+  workflow_dispatch:
+    inputs:
+      rabbitmq_version:
+        description: 'RabbitMQ Version'
+        default: '3.13-alpine'
+        required: true
+        type: string
 
 jobs:
   publish:
@@ -45,7 +48,7 @@ jobs:
         file: dev/mq/Dockerfile
         platforms: linux/amd64,linux/arm64
         push: true
-        build-args: RABBITMQ_VERSION=${{ env.RABBITMQ_VERSION }}
+        build-args: RABBITMQ_VERSION=${{ inputs.rabbitmq_version }}
         tags: |
-          ghcr.io/ietf-tools/datatracker-mq:${{ env.RABBITMQ_VERSION }}
+          ghcr.io/ietf-tools/datatracker-mq:${{ inputs.rabbitmq_version }}
           ghcr.io/ietf-tools/datatracker-mq:latest
diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml
index 3cab7ff565..0c8f0705b5 100644
--- a/k8s/rabbitmq.yaml
+++ b/k8s/rabbitmq.yaml
@@ -29,7 +29,7 @@ spec:
         # -----------------------------------------------------
         # RabbitMQ Container
         # -----------------------------------------------------
-        - image: "ghcr.io/ietf-tools/datatracker-mq:3.12-alpine"
+        - image: "ghcr.io/ietf-tools/datatracker-mq:3.13-alpine"
           imagePullPolicy: Always
           name: rabbitmq
           ports:

From 9dd25b9aee2ccb565d74d55d90c67bf8388c910e Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Fri, 28 Mar 2025 04:07:53 +1300
Subject: [PATCH 297/656] chore: vite sourcemap (#8719)

---
 vite.config.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/vite.config.js b/vite.config.js
index 41a2cb02e0..bde2b9ed57 100644
--- a/vite.config.js
+++ b/vite.config.js
@@ -16,7 +16,8 @@ export default defineConfig(({ command, mode }) => {
           main: 'client/main.js',
           embedded: 'client/embedded.js'
         }
-      }
+      },
+      sourcemap: true
     },
     cacheDir: '.vite',
     plugins: [

From 7e9a46af63da0e45f3ea0e4ccfda69875b282190 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 27 Mar 2025 13:59:24 -0500
Subject: [PATCH 298/656] feat: move base containers to bookworm (#8710)

---
 docker/base.Dockerfile             | 17 ++++++++++-------
 docker/scripts/app-setup-debian.sh | 16 +++-------------
 ietf/submit/tests.py               |  4 ++--
 3 files changed, 15 insertions(+), 22 deletions(-)

diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile
index e2465f33c2..f364456c7a 100644
--- a/docker/base.Dockerfile
+++ b/docker/base.Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9-bullseye
+FROM python:3.9-bookworm
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -14,6 +14,9 @@ RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \
     && mkdir -p /etc/apt/keyrings\
     && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
 RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+RUN echo "Package: nodejs" >> /etc/apt/preferences.d/preferences && \
+    echo "Pin: origin deb.nodesource.com" >> /etc/apt/preferences.d/preferences && \
+    echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/preferences
 
 # Add Docker Source
 RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
@@ -56,12 +59,13 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends
 	libmagic-dev \
 	libmariadb-dev \
 	libmemcached-tools \
+	libyang2-tools \
 	locales \
 	make \
 	mariadb-client \
 	memcached \
 	nano \
-	netcat \
+	netcat-traditional \
 	nodejs \
 	pgloader \
 	pigz \
@@ -77,7 +81,6 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends
 	wget \
 	xauth \
 	xvfb \
-	yang-tools \
 	zsh
 
 # Install kramdown-rfc2629 (ruby)
@@ -106,11 +109,11 @@ RUN apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /va
 ENV DBUS_SESSION_BUS_ADDRESS=/dev/null
 
 # avoid million NPM install messages
-ENV npm_config_loglevel warn
+ENV npm_config_loglevel=warn
 # allow installing when the main user is root
-ENV npm_config_unsafe_perm true
+ENV npm_config_unsafe_perm=true
 # disable NPM funding messages
-ENV npm_config_fund false
+ENV npm_config_fund=false
 
 # Set locale to en_US.UTF-8
 RUN echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \
@@ -119,7 +122,7 @@ RUN echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \
     dpkg-reconfigure locales && \
     locale-gen en_US.UTF-8 && \
     update-locale LC_ALL en_US.UTF-8
-ENV LC_ALL en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8
 
 # Install idnits
 ADD https://raw.githubusercontent.com/ietf-tools/idnits-mirror/main/idnits /usr/local/bin/
diff --git a/docker/scripts/app-setup-debian.sh b/docker/scripts/app-setup-debian.sh
index ddfc351995..ea9cc3fb87 100644
--- a/docker/scripts/app-setup-debian.sh
+++ b/docker/scripts/app-setup-debian.sh
@@ -10,7 +10,6 @@
 # Syntax: ./common-debian.sh [install zsh flag] [username] [user UID] [user GID] [upgrade packages flag] [install Oh My Zsh! flag] [Add non-free packages]
 
 set -e
-
 INSTALL_ZSH=${1:-"true"}
 USERNAME=${2:-"automatic"}
 USER_UID=${3:-"automatic"}
@@ -116,18 +115,9 @@ if [ "${PACKAGES_ALREADY_INSTALLED}" != "true" ]; then
     # Needed for adding manpages-posix and manpages-posix-dev which are non-free packages in Debian
     if [ "${ADD_NON_FREE_PACKAGES}" = "true" ]; then
         # Bring in variables from /etc/os-release like VERSION_CODENAME
-        . /etc/os-release
-        sed -i -E "s/deb http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME} main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME} main contrib non-free/" /etc/apt/sources.list
-        sed -i -E "s/deb-src http:\/\/(deb|httredir)\.debian\.org\/debian ${VERSION_CODENAME} main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME} main contrib non-free/" /etc/apt/sources.list
-        sed -i -E "s/deb http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME}-updates main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME}-updates main contrib non-free/" /etc/apt/sources.list
-        sed -i -E "s/deb-src http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME}-updates main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME}-updates main contrib non-free/" /etc/apt/sources.list
-        sed -i "s/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main contrib non-free/" /etc/apt/sources.list
-        sed -i "s/deb-src http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main contrib non-free/" /etc/apt/sources.list
-        sed -i "s/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list 
-        sed -i "s/deb-src http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list
-        # Handle bullseye location for security https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html
-        sed -i "s/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main contrib non-free/" /etc/apt/sources.list
-        sed -i "s/deb-src http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main contrib non-free/" /etc/apt/sources.list
+        .  /etc/os-release
+        sed -i -E "s/Components: main/Components: main contrib non-free/" /etc/apt/sources.list.d/debian.sources
+
         echo "Running apt-get update..."
         apt-get update
         package_list="${package_list} manpages-posix manpages-posix-dev"
diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index 9a993480cd..6b1c998e76 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -1888,7 +1888,7 @@ def test_submit_invalid_yang(self):
         r = self.client.get(status_url)
         q = PyQuery(r.content)
         #
-        self.assertContains(r, 'The yang validation returned 1 error')
+        self.assertContains(r, 'The yang validation returned 3 errors')
         #
         m = q('#yang-validation-message').text()
         for command in ['xym', 'pyang', 'yanglint']:
@@ -1898,7 +1898,7 @@ def test_submit_invalid_yang(self):
         self.assertIn("draft-yang-testing-invalid-00.txt", m)
         self.assertIn("error: syntax error: illegal keyword: ;", m)
         if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY):
-            self.assertIn("No validation errors", m)
+            self.assertIn('libyang err : Parsing module "ietf-yang-metadata" failed.', m)
 
     def submit_conflicting_submissiondocevent_rev(self, new_rev='01', existing_rev='01'):
         """Test submitting a rev when an equal or later SubmissionDocEvent rev exists

From 431c475060778ccaaf611e0de7a9bcc0c3b93c42 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Thu, 27 Mar 2025 19:12:31 +0000
Subject: [PATCH 299/656] ci: update base image target version to 20250327T1859

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 2b02a091c5..cc55c92881 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250315T0745
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250327T1859
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index e6f490b168..50e8bfd839 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250315T0745
+20250327T1859

From 752bc2103146b534e1bb6188c7d54d11f07ba885 Mon Sep 17 00:00:00 2001
From: Rudi Matz <rudi@staff.ietf.org>
Date: Thu, 27 Mar 2025 18:19:30 -0400
Subject: [PATCH 300/656] feat: highlight unexpected state in AD dashboard
 (#8738)

* feat: highlight unexpected state in AD dashboard

* test: add tests for is_unexpected_wg_state

* test: improve tests using WgDraftFactory
---
 ietf/doc/templatetags/ietf_filters.py         | 13 +++++++++++++
 ietf/doc/templatetags/tests_ietf_filters.py   | 16 ++++++++++++++++
 ietf/templates/doc/search/status_columns.html |  4 ++++
 3 files changed, 33 insertions(+)

diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py
index 86507eeaaa..d4adf96a27 100644
--- a/ietf/doc/templatetags/ietf_filters.py
+++ b/ietf/doc/templatetags/ietf_filters.py
@@ -480,6 +480,19 @@ def state(doc, slug):
         slug = "%s-stream-%s" % (doc.type_id, doc.stream_id)
     return doc.get_state(slug)
 
+
+@register.filter
+def is_unexpected_wg_state(doc):
+    """Returns a flag indicating whether the document has an unexpected wg state."""
+    if not doc.type_id == "draft":
+        return False
+
+    draft_iesg_state = doc.get_state("draft-iesg")
+    draft_stream_state = doc.get_state("draft-stream-ietf")
+
+    return draft_iesg_state.slug != "idexists" and draft_stream_state is not None and draft_stream_state.slug != "sub-pub"
+
+
 @register.filter
 def statehelp(state):
     "Output help icon with tooltip for state."
diff --git a/ietf/doc/templatetags/tests_ietf_filters.py b/ietf/doc/templatetags/tests_ietf_filters.py
index f018b7d9b3..b5130849ea 100644
--- a/ietf/doc/templatetags/tests_ietf_filters.py
+++ b/ietf/doc/templatetags/tests_ietf_filters.py
@@ -14,12 +14,14 @@
     ConflictReviewFactory,
     BofreqFactory,
     StatementFactory,
+    RfcFactory,
 )
 from ietf.doc.models import DocEvent
 from ietf.doc.templatetags.ietf_filters import (
     urlize_ietf_docs,
     is_valid_url,
     is_in_stream,
+    is_unexpected_wg_state,
 )
 from ietf.person.models import Person
 from ietf.utils.test_utils import TestCase
@@ -174,3 +176,17 @@ def test_urlize_ietf_docs(self):
         for input, output in cases:
             # debug.show("(input, urlize_ietf_docs(input), output)")
             self.assertEqual(urlize_ietf_docs(input), output)
+    
+    def test_is_unexpected_wg_state(self):
+        """
+        Test that the unexpected_wg_state function works correctly
+        """
+        # test documents with expected wg states
+        self.assertFalse(is_unexpected_wg_state(RfcFactory()))
+        self.assertFalse(is_unexpected_wg_state(WgDraftFactory (states=[('draft-stream-ietf', 'sub-pub')])))
+        self.assertFalse(is_unexpected_wg_state(WgDraftFactory (states=[('draft-iesg', 'idexists')])))
+        self.assertFalse(is_unexpected_wg_state(WgDraftFactory (states=[('draft-stream-ietf', 'wg-cand'), ('draft-iesg','idexists')])))
+
+        # test documents with unexpected wg states due to invalid combination of states
+        self.assertTrue(is_unexpected_wg_state(WgDraftFactory (states=[('draft-stream-ietf', 'wg-cand'), ('draft-iesg','lc-req')])))
+        self.assertTrue(is_unexpected_wg_state(WgDraftFactory (states=[('draft-stream-ietf', 'chair-w'), ('draft-iesg','pub-req')])))
diff --git a/ietf/templates/doc/search/status_columns.html b/ietf/templates/doc/search/status_columns.html
index 15f284fd12..5ba41bb9c4 100644
--- a/ietf/templates/doc/search/status_columns.html
+++ b/ietf/templates/doc/search/status_columns.html
@@ -78,6 +78,10 @@
                 {% person_link action_holder.person title=action_holder.role_for_doc %}{% if action_holder|action_holder_badge %} {{ action_holder|action_holder_badge }}{% endif %}{% if not forloop.last %},{% endif %}
             {% endfor %}
         {% endif %}
+        {% if doc|is_unexpected_wg_state %}
+            <br>
+            <span class="badge rounded-pill text-bg-warning">Unexpected WG state</span>
+        {% endif %}
     {% else %}
         {# RFC #}
         {{ doc.std_level|safe }} RFC

From aa2a3217a7006c05292650bb0d75b0243d2cc67b Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 28 Mar 2025 10:59:56 -0400
Subject: [PATCH 301/656] fix(agenda): strikethrough cancelled and rescheduled
 events (#8694)

* fix(agenda): strikethrough cancelled and rescheduled events

* fix(agenda): rescheduled + canceled badges width on mobile

* fix(agenda): use em for canceled / rescheduled badge on mobile

* fix: deleting badge width/display:block CSS

---------

Co-authored-by: Matthew Holloway <matthew@staff.ietf.org>
---
 client/agenda/AgendaDetailsModal.vue     |  1 -
 client/agenda/AgendaScheduleCalendar.vue |  1 -
 client/agenda/AgendaScheduleList.vue     | 12 ++++++++++--
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/client/agenda/AgendaDetailsModal.vue b/client/agenda/AgendaDetailsModal.vue
index 037e32f9cb..2582bf2159 100644
--- a/client/agenda/AgendaDetailsModal.vue
+++ b/client/agenda/AgendaDetailsModal.vue
@@ -327,7 +327,6 @@ async function fetchSessionMaterials () {
     border-radius: 5px;
 
     .badge {
-      width: 30px;
       font-size: .7em;
       background-color: $yellow-200;
       border-bottom: 1px solid $yellow-500;
diff --git a/client/agenda/AgendaScheduleCalendar.vue b/client/agenda/AgendaScheduleCalendar.vue
index 6701ddabd7..9863296341 100644
--- a/client/agenda/AgendaScheduleCalendar.vue
+++ b/client/agenda/AgendaScheduleCalendar.vue
@@ -330,7 +330,6 @@ function close () {
     }
 
     .badge {
-      width: 30px;
       font-size: .7em;
       border: 1px solid #CCC;
       text-transform: uppercase;
diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index 86c91bed85..d38c7842eb 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -1097,8 +1097,6 @@ onBeforeUnmount(() => {
         }
 
         @media screen and (max-width: $bs5-break-md) {
-          width: 30px;
-          display: block;
           margin: 2px 0 0 0;
         }
       }
@@ -1321,6 +1319,11 @@ onBeforeUnmount(() => {
 
     &.agenda-table-cell-room {
       border-right: 1px solid darken($red-100, 5%) !important;
+      text-decoration: line-through;
+    }
+
+    &.agenda-table-cell-name > a, &.agenda-table-cell-name > span {
+      text-decoration: line-through;
     }
 
     &:last-child {
@@ -1348,6 +1351,11 @@ onBeforeUnmount(() => {
 
     &.agenda-table-cell-room {
       border-right: 1px solid darken($orange-100, 5%) !important;
+      text-decoration: line-through;
+    }
+
+    &.agenda-table-cell-name > a, &.agenda-table-cell-name > span {
+      text-decoration: line-through;
     }
 
     &:last-child {

From 527db359e3836a587ba67432b814004b75635022 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 1 Apr 2025 09:03:59 -0400
Subject: [PATCH 302/656] chore: fix line ending for
 .github/ISSUE_TEMPLATE/config.yml

---
 .github/ISSUE_TEMPLATE/config.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
index 5e27af9fed..320614b17e 100644
--- a/.github/ISSUE_TEMPLATE/config.yml
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -1,8 +1,8 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Help and questions
-    url: https://github.com/ietf-tools/datatracker/discussions/categories/help-questions
-    about: Need help? Have a question on setting up the project or its usage?
-  - name: Discuss new ideas
-    url: https://github.com/ietf-tools/datatracker/discussions/categories/ideas
-    about: Submit ideas for new features or improvements to be discussed.
+blank_issues_enabled: false
+contact_links:
+  - name: Help and questions
+    url: https://github.com/ietf-tools/datatracker/discussions/categories/help-questions
+    about: Need help? Have a question on setting up the project or its usage?
+  - name: Discuss new ideas
+    url: https://github.com/ietf-tools/datatracker/discussions/categories/ideas
+    about: Submit ideas for new features or improvements to be discussed.

From 9957cf190a4a61369f97e0224e0bb2ae6ed9c8be Mon Sep 17 00:00:00 2001
From: Rudi Matz <rudi@staff.ietf.org>
Date: Tue, 1 Apr 2025 14:35:04 -0400
Subject: [PATCH 303/656] feat: block iesg ballot for docs from outside streams
 (#8761)

* feat: block iesg ballot for docs from outside streams

* test: correct failing irsg test

* feat: improve/simplify tests and condition

* test: filter out pyflakes tests related to globals
---
 ietf/doc/tests_ballot.py      | 29 +++++++++++++++++++++++------
 ietf/doc/tests_irsg_ballot.py |  2 +-
 ietf/doc/views_ballot.py      |  5 ++++-
 ietf/utils/test_runner.py     |  9 ++++++++-
 4 files changed, 36 insertions(+), 9 deletions(-)

diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py
index c7362b58e2..ec23f3d491 100644
--- a/ietf/doc/tests_ballot.py
+++ b/ietf/doc/tests_ballot.py
@@ -17,7 +17,7 @@
 from ietf.doc.models import (Document, State, DocEvent,
                              BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, TelechatDocEvent)
 from ietf.doc.factories import (DocumentFactory, IndividualDraftFactory, IndividualRfcFactory, WgDraftFactory,
-                                BallotPositionDocEventFactory, BallotDocEventFactory, IRSGBallotDocEventFactory)
+                                BallotPositionDocEventFactory, BallotDocEventFactory, IRSGBallotDocEventFactory, RgDraftFactory)
 from ietf.doc.templatetags.ietf_filters import can_defer
 from ietf.doc.utils import create_ballot_if_not_open
 from ietf.doc.views_ballot import parse_ballot_edit_return_point
@@ -360,7 +360,7 @@ def test_request_last_call(self):
         self.assertTrue('aread@' in outbox[-1]['Cc'])
 
     def test_edit_ballot_writeup(self):
-        draft = IndividualDraftFactory(states=[('draft','active'),('draft-iesg','iesg-eva')])
+        draft = IndividualDraftFactory(states=[('draft','active'),('draft-iesg','iesg-eva')], stream_id='ietf')
         url = urlreverse('ietf.doc.views_ballot.ballot_writeupnotes', kwargs=dict(name=draft.name))
         login_testing_unauthorized(self, "secretary", url)
 
@@ -390,8 +390,25 @@ def test_edit_ballot_writeup(self):
         self.assertTrue("This is a simple test" in d.latest_event(WriteupDocEvent, type="changed_ballot_writeup_text").text)
         self.assertTrue('iesg-eva' == d.get_state_slug('draft-iesg'))
 
+    def test_edit_ballot_writeup_unauthorized_stream(self):
+        # Test that accessing a document from unauthorized (irtf) stream returns a 404 error
+        draft = RgDraftFactory()
+        url = urlreverse('ietf.doc.views_ballot.ballot_writeupnotes', kwargs=dict(name=draft.name))
+        login_testing_unauthorized(self, "ad", url)
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
+
+    def test_edit_ballot_writeup_invalid_name(self):
+        # Test that accessing a non-existent document returns a 404 error
+        url = urlreverse('ietf.doc.views_ballot.ballot_writeupnotes', kwargs=dict(name="invalid_name"))
+        login_testing_unauthorized(self, "ad", url)
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
+
     def test_edit_ballot_writeup_already_approved(self):
-        draft = IndividualDraftFactory(states=[('draft','active'),('draft-iesg','approved')])
+        draft = IndividualDraftFactory(states=[('draft','active'),('draft-iesg','approved')], stream_id='ietf')
         url = urlreverse('ietf.doc.views_ballot.ballot_writeupnotes', kwargs=dict(name=draft.name))
         login_testing_unauthorized(self, "secretary", url)
 
@@ -465,7 +482,7 @@ def test_edit_ballot_rfceditornote(self):
     def test_issue_ballot(self):
         ad = Person.objects.get(user__username="ad")
         for case in ('none','past','future'):
-            draft = IndividualDraftFactory(ad=ad)
+            draft = IndividualDraftFactory(ad=ad, stream_id='ietf')
             if case in ('past','future'):
                 LastCallDocEvent.objects.create(
                     by=Person.objects.get(name='(System)'),
@@ -504,7 +521,7 @@ def test_issue_ballot(self):
 
     def test_issue_ballot_auto_state_change(self):
         ad = Person.objects.get(user__username="ad")
-        draft = IndividualDraftFactory(ad=ad, states=[('draft','active'),('draft-iesg','writeupw')])
+        draft = IndividualDraftFactory(ad=ad, states=[('draft','active'),('draft-iesg','writeupw')], stream_id='ietf')
         url = urlreverse('ietf.doc.views_ballot.ballot_writeupnotes', kwargs=dict(name=draft.name))
         login_testing_unauthorized(self, "secretary", url)
 
@@ -528,7 +545,7 @@ def test_issue_ballot_auto_state_change(self):
 
     def test_issue_ballot_warn_if_early(self):
         ad = Person.objects.get(user__username="ad")
-        draft = IndividualDraftFactory(ad=ad, states=[('draft','active'),('draft-iesg','lc')])
+        draft = IndividualDraftFactory(ad=ad, states=[('draft','active'),('draft-iesg','lc')], stream_id='ietf')
         url = urlreverse('ietf.doc.views_ballot.ballot_writeupnotes', kwargs=dict(name=draft.name))
         login_testing_unauthorized(self, "secretary", url)
 
diff --git a/ietf/doc/tests_irsg_ballot.py b/ietf/doc/tests_irsg_ballot.py
index 92752e48c4..aa62d8aaf9 100644
--- a/ietf/doc/tests_irsg_ballot.py
+++ b/ietf/doc/tests_irsg_ballot.py
@@ -288,7 +288,7 @@ def test_edit_ballot_position_permissions(self):
 
     def test_iesg_ballot_no_irsg_actions(self):
         ad = Person.objects.get(user__username="ad")
-        wg_draft = IndividualDraftFactory(ad=ad)
+        wg_draft = IndividualDraftFactory(ad=ad, stream_id='ietf')
         irsgmember = get_active_irsg()[0]
 
         url = urlreverse('ietf.doc.views_ballot.ballot_writeupnotes', kwargs=dict(name=wg_draft.name))
diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py
index 4ff40d2268..9e2a417933 100644
--- a/ietf/doc/views_ballot.py
+++ b/ietf/doc/views_ballot.py
@@ -611,6 +611,10 @@ def clean_ballot_writeup(self):
 def ballot_writeupnotes(request, name):
     """Editing of ballot write-up and notes"""
     doc = get_object_or_404(Document, name=name)
+
+    if doc.stream_id is None or doc.stream_id != 'ietf':
+        raise Http404("The requested operation is not allowed for this document.")
+
     prev_state = doc.get_state("draft-iesg")
 
     login = request.user.person
@@ -1335,4 +1339,3 @@ def parse_ballot_edit_return_point(path, doc_name, ballot_id):
         "ietf.iesg.views.past_documents",
     }
     return validate_return_to_path(path, get_default_path, allowed_path_handlers)
-
diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py
index 3c89a2d01c..6b6f282c49 100644
--- a/ietf/utils/test_runner.py
+++ b/ietf/utils/test_runner.py
@@ -263,7 +263,14 @@ def pyflakes_test(self):
         path = os.path.join(settings.BASE_DIR)
         warnings = []
         warnings = pyflakes.checkPaths([path], verbosity=0)
-        self.assertEqual([], [str(w) for w in warnings])
+
+        # Filter out warnings about unused global variables
+        filtered_warnings = [
+            w for w in warnings
+            if not re.search(r"`global \w+` is unused: name is never assigned in scope", str(w))
+        ]
+
+        self.assertEqual([], [str(w) for w in filtered_warnings])
 
 class MyPyTest(TestCase):
 

From 48f97c68406a158a4d4bf5da58e9168d12a5f92e Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 1 Apr 2025 16:12:33 -0300
Subject: [PATCH 304/656] chore: fix/refactor code near unneeded global
 declarations (#8765)

* refactor: unused vars in draft.py

* refactor: unwrap/simplify random_faker() prep

* chore: types/globals in test_runner.py

* chore: drop unneeded globals
---
 ietf/doc/templatetags/wg_menu.py |  2 --
 ietf/person/factories.py         | 26 ++++++++++++++------------
 ietf/utils/draft.py              |  8 +-------
 ietf/utils/patch.py              |  3 +--
 ietf/utils/test_runner.py        | 31 ++++++++++++++-----------------
 5 files changed, 30 insertions(+), 40 deletions(-)

diff --git a/ietf/doc/templatetags/wg_menu.py b/ietf/doc/templatetags/wg_menu.py
index 76bf7eb4d0..3e8d209448 100644
--- a/ietf/doc/templatetags/wg_menu.py
+++ b/ietf/doc/templatetags/wg_menu.py
@@ -62,8 +62,6 @@
 
 @register.simple_tag
 def wg_menu(flavor):
-    global parents
-
     for p in parents:
         p.short_name = parent_short_names.get(p.acronym) or p.name
         if p.short_name.endswith(" Area"):
diff --git a/ietf/person/factories.py b/ietf/person/factories.py
index 45de554766..98756f26c8 100644
--- a/ietf/person/factories.py
+++ b/ietf/person/factories.py
@@ -26,20 +26,22 @@
 
 fake = faker.Factory.create()
 
-def setup():
-    global acceptable_fakers
-    # The transliteration of some Arabic and Devanagari names introduces
-    # non-alphabetic characters that don't work with the draft author
-    # extraction code, and also don't seem to match the way people with Arabic
-    # names romanize Arabic names.  Exclude those locales from name generation
-    # in order to avoid test failures.
-    locales = set( [ l for l in faker.config.AVAILABLE_LOCALES if not (l.startswith('ar_') or l.startswith('sg_') or l=='fr_QC') ] )
-    acceptable_fakers = [faker.Faker(locale) for locale in locales]
-setup()
+# The transliteration of some Arabic and Devanagari names introduces
+# non-alphabetic characters that don't work with the draft author
+# extraction code, and also don't seem to match the way people with Arabic
+# names romanize Arabic names.  Exclude those locales from name generation
+# in order to avoid test failures.
+_acceptable_fakers = [
+    faker.Faker(locale)
+    for locale in set(faker.config.AVAILABLE_LOCALES)
+    if not (locale.startswith('ar_') or locale.startswith('sg_') or locale == 'fr_QC')
+]
+
 
 def random_faker():
-    global acceptable_fakers
-    return random.sample(acceptable_fakers, 1)[0]
+    """Helper to get a random faker acceptable for User names"""
+    return random.sample(_acceptable_fakers, 1)[0]
+
 
 class UserFactory(factory.django.DjangoModelFactory):
     class Meta:
diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py
index 50add5abba..53d3d40811 100755
--- a/ietf/utils/draft.py
+++ b/ietf/utils/draft.py
@@ -65,7 +65,6 @@
 opt_debug = False
 opt_timestamp = False
 opt_trace = False
-opt_authorinfo = False
 opt_attributes = False
 # Don't forget to add the option variable to the globals list in _main below
 
@@ -1332,8 +1331,6 @@ def getmeta(fn):
 
 # ----------------------------------------------------------------------
 def _output(docname, fields, outfile=sys.stdout):
-    global company_domain
-
     if opt_attributes:
         def outputkey(key, fields):
             field = fields[key]
@@ -1373,9 +1370,8 @@ def _printmeta(fn, outfile=sys.stdout):
 # Main
 # ----------------------------------------------------------------------
 
-company_domain = {}                     # type: Dict[str, str]
 def _main(outfile=sys.stdout):
-    global opt_debug, opt_timestamp, opt_trace, opt_authorinfo, files, company_domain, opt_attributes
+    global opt_debug, opt_timestamp, opt_trace, files, opt_attributes
     # set default values, if any
     # ----------------------------------------------------------------------
     # Option processing
@@ -1423,8 +1419,6 @@ def _main(outfile=sys.stdout):
         elif opt in ["-T", "--trace"]: # Emit trace information while working
             opt_trace = True
 
-    company_domain = {}
- 
     if not files:
         files = [ "-" ]
 
diff --git a/ietf/utils/patch.py b/ietf/utils/patch.py
index 9de2270ebb..fd3e4a165d 100644
--- a/ietf/utils/patch.py
+++ b/ietf/utils/patch.py
@@ -87,8 +87,7 @@ def createLock(self):
 debugmode = False
 
 def setdebug():
-  global debugmode, streamhandler
-
+  global debugmode
   debugmode = True
   loglevel = logging.DEBUG
   logformat = "%(levelname)8s %(message)s"
diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py
index 6b6f282c49..a77377ffb5 100644
--- a/ietf/utils/test_runner.py
+++ b/ietf/utils/test_runner.py
@@ -53,9 +53,10 @@
 import factory.random
 import urllib3
 import warnings
-from urllib.parse import urlencode
 
 from fnmatch import fnmatch
+from typing import Callable, Optional
+from urllib.parse import urlencode
 
 from coverage.report import Reporter
 from coverage.results import Numbers
@@ -90,11 +91,11 @@
 from mypy_boto3_s3.service_resource import Bucket
 
 
-loaded_templates = set()
-visited_urls = set()
-test_database_name = None
-old_destroy = None
-old_create = None
+loaded_templates: set[str] = set()
+visited_urls: set[str] = set()
+test_database_name: Optional[str] = None
+old_destroy: Optional[Callable] = None
+old_create: Optional[Callable] = None
 
 template_coverage_collection = None
 code_coverage_collection = None
@@ -230,10 +231,12 @@ def load_and_run_fixtures(verbosity):
             fn()
 
 def safe_create_test_db(self, verbosity, *args, **kwargs):
-    global test_database_name, old_create
+    if old_create is None:
+        raise RuntimeError("old_create has not been set, cannot proceed")
     keepdb = kwargs.get('keepdb', False)
     if not keepdb:
         print("     Creating test database...")
+    global test_database_name
     test_database_name = old_create(self, 0, *args, **kwargs)
 
     if settings.GLOBAL_TEST_FIXTURES:
@@ -243,8 +246,9 @@ def safe_create_test_db(self, verbosity, *args, **kwargs):
     return test_database_name
 
 def safe_destroy_test_db(*args, **kwargs):
+    if old_destroy is None:
+        raise RuntimeError("old_destroy has not been set, cannot proceed")
     sys.stdout.write('\n')
-    global test_database_name, old_destroy
     keepdb = kwargs.get('keepdb', False)
     if not keepdb:
         if settings.DATABASES["default"]["NAME"] != test_database_name:
@@ -358,15 +362,13 @@ class TemplateCoverageLoader(BaseLoader):
     is_usable = True
 
     def get_template(self, template_name, skip=None):
-        global template_coverage_collection, loaded_templates
-        if template_coverage_collection == True:
+        if template_coverage_collection:
             loaded_templates.add(str(template_name))
         raise TemplateDoesNotExist(template_name)
 
 def record_urls_middleware(get_response):
     def record_urls(request):
-        global url_coverage_collection, visited_urls
-        if url_coverage_collection == True:
+        if url_coverage_collection:
             visited_urls.add(request.path)
         return get_response(request)
     return record_urls
@@ -532,7 +534,6 @@ def report_test_result(self, test):
                             ( test, test_coverage*100, latest_coverage_version, master_coverage*100, ))
 
     def template_coverage_test(self):
-        global loaded_templates
         if self.runner.check_coverage:
             apps = [ app.split('.')[-1] for app in self.runner.test_apps ]
             all = get_template_paths(apps)
@@ -760,7 +761,6 @@ def __init__(
         self.show_logging = show_logging
         self.rerun = rerun
         self.test_labels = None
-        global validation_settings
         validation_settings["validate_html"] = self if validate_html else None
         validation_settings["validate_html_harder"] = self if validate_html and validate_html_harder else None
         validation_settings["show_logging"] = show_logging
@@ -783,9 +783,6 @@ def __init__(
         self.blobstoremanager = TestBlobstoreManager() if manage_blobstore else None
 
     def setup_test_environment(self, **kwargs):
-        global template_coverage_collection
-        global url_coverage_collection
-
         ietf.utils.mail.test_mode = True
         ietf.utils.mail.SMTP_ADDR['ip4'] = '127.0.0.1'
         ietf.utils.mail.SMTP_ADDR['port'] = 2025

From 48bee8a58381e5caa6cf5d6b56bd58eff614a918 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 1 Apr 2025 17:19:30 -0300
Subject: [PATCH 305/656] chore: update libreoffice install for bookworm
 (#8768)

---
 dev/build/Dockerfile | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index cc55c92881..c25298d652 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -10,9 +10,8 @@ RUN groupadd -g 1000 datatracker && \
 RUN apt-get purge -y imagemagick imagemagick-6-common
 
 # Install libreoffice (needed via PPT2PDF_COMMAND)
-RUN echo "deb http://deb.debian.org/debian bullseye-backports main" > /etc/apt/sources.list.d/bullseye-backports.list && \
-    apt-get update && \
-    apt-get -qyt bullseye-backports install libreoffice-nogui
+RUN apt-get update && \
+    apt-get -qy install libreoffice-nogui
 
 COPY . .
 COPY ./dev/build/start.sh ./start.sh

From 15ef59133a0e8b62158836b697d9413dc395228f Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@staff.ietf.org>
Date: Thu, 3 Apr 2025 05:10:22 +1300
Subject: [PATCH 306/656] chore: libreoffice flags for resilience (#8769)

---
 ietf/settings.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ietf/settings.py b/ietf/settings.py
index 1fe5f48229..33a2f976d9 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -1103,7 +1103,12 @@ def skip_unreadable_post(record):
 
 DATATRACKER_MAX_UPLOAD_SIZE = 40960000
 PPT2PDF_COMMAND = [
-    "/usr/bin/soffice", "--headless", "--convert-to", "pdf:writer_globaldocument_pdf_Export", "--outdir"
+    "/usr/bin/soffice",
+    "--headless", # no GUI
+    "--safe-mode", # use a new libreoffice profile every time (ensures no reliance on accumulated profile config)
+    "--norestore", # don't attempt to restore files after a previous crash (ensures that one crash won't block future conversions until UI intervention)
+    "--convert-to", "pdf:writer_globaldocument_pdf_Export",
+    "--outdir"
 ]
 
 STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/'

From afa79dc55b16463d56b531bb1b6b06fbbeb12ace Mon Sep 17 00:00:00 2001
From: Jim Fenton <fenton@bluepopcorn.net>
Date: Wed, 2 Apr 2025 09:10:38 -0700
Subject: [PATCH 307/656] feat: Warn if uploading minutes before session end
 (#8700)

* Warn if uploading minutes before sessionn end

* Remove extraneous btn-primary for session future

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>

* fix: guard against unscheduled sessions

* fix: test addition of warning

* fix: another guard against unscheduled sessions

* feat: test future warning on session details pannel

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/meeting/tests_views.py                   | 286 ++++++++++--------
 ietf/meeting/views.py                         |   6 +
 .../meeting/session_details_panel.html        |   2 +-
 .../meeting/upload_session_minutes.html       |   5 +
 4 files changed, 168 insertions(+), 131 deletions(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 0f91986f77..a93a26b981 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -6541,110 +6541,130 @@ def test_upload_bluesheets_interim_chair_access(self):
         self.assertIn('Upload', str(q("title")))
         
 
-    def test_upload_minutes_agenda(self):
-        for doctype in ('minutes','agenda'):
-            session = SessionFactory(meeting__type_id='ietf')
-            if doctype == 'minutes':
-                url = urlreverse('ietf.meeting.views.upload_session_minutes',kwargs={'num':session.meeting.number,'session_id':session.id})
-            else:
-                url = urlreverse('ietf.meeting.views.upload_session_agenda',kwargs={'num':session.meeting.number,'session_id':session.id})
-            self.client.logout()
-            login_testing_unauthorized(self,"secretary",url)
-            r = self.client.get(url)
-            self.assertEqual(r.status_code, 200)
-            q = PyQuery(r.content)
-            self.assertIn('Upload', str(q("Title")))
-            self.assertFalse(session.presentations.exists())
-            self.assertFalse(q('form input[type="checkbox"]'))
-    
-            session2 = SessionFactory(meeting=session.meeting,group=session.group)
-            r = self.client.get(url)
-            self.assertEqual(r.status_code, 200)
-            q = PyQuery(r.content)
-            self.assertTrue(q('form input[type="checkbox"]'))
-    
-            # test not submitting a file
-            r = self.client.post(url, dict(submission_method="upload"))
-            self.assertEqual(r.status_code, 200)
-            q = PyQuery(r.content)
-            self.assertTrue(q("form .is-invalid"))
-    
-            test_file = BytesIO(b'this is some text for a test')
-            test_file.name = "not_really.json"
-            r = self.client.post(url,dict(submission_method="upload",file=test_file))
-            self.assertEqual(r.status_code, 200)
-            q = PyQuery(r.content)
-            self.assertTrue(q('form .is-invalid'))
-    
-            test_file = BytesIO(b'this is some text for a test'*1510000)
-            test_file.name = "not_really.pdf"
-            r = self.client.post(url,dict(submission_method="upload",file=test_file))
-            self.assertEqual(r.status_code, 200)
-            q = PyQuery(r.content)
-            self.assertTrue(q('form .is-invalid'))
-    
-            test_file = BytesIO(b'<html><frameset><frame src="foo.html"></frame><frame src="bar.html"></frame></frameset></html>')
-            test_file.name = "not_really.html"
-            r = self.client.post(url,dict(submission_method="upload",file=test_file))
-            self.assertEqual(r.status_code, 200)
-            q = PyQuery(r.content)
-            self.assertTrue(q('form .is-invalid'))
-
-            # Test html sanitization
-            test_file = BytesIO(b'<html><head><title>Title</title></head><body><h1>Title</h1><section>Some text</section></body></html>')
-            test_file.name = "some.html"
-            r = self.client.post(url,dict(submission_method="upload",file=test_file))
-            self.assertEqual(r.status_code, 302)
-            doc = session.presentations.filter(document__type_id=doctype).first().document
-            self.assertEqual(doc.rev,'00')
-            text = doc.text()
-            self.assertIn('Some text', text)
-            self.assertNotIn('<section>', text)
-            text = retrieve_str(doctype, f"{doc.name}-{doc.rev}.html")
-            self.assertIn('Some text', text)
-            self.assertNotIn('<section>', text)
-
-            # txt upload
-            test_bytes = b'This is some text for a test, with the word\nvirtual at the beginning of a line.'
-            test_file = BytesIO(test_bytes)
-            test_file.name = "some.txt"
-            r = self.client.post(url,dict(submission_method="upload",file=test_file,apply_to_all=False))
-            self.assertEqual(r.status_code, 302)
-            doc = session.presentations.filter(document__type_id=doctype).first().document
-            self.assertEqual(doc.rev,'01')
-            self.assertFalse(session2.presentations.filter(document__type_id=doctype))
-            retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt")
-            self.assertEqual(retrieved_bytes, test_bytes)
-    
+    def test_label_future_sessions(self):
+        self.client.login(username='secretary', password='secretary+password')
+        for future in (True, False):
+            mtg_date = date_today()+datetime.timedelta(days=180 if future else -180)
+            session = SessionFactory(meeting__type_id='ietf', meeting__date=mtg_date)
+            # Verify future warning shows on the session details panel
+            url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
             r = self.client.get(url)
-            self.assertEqual(r.status_code, 200)
-            q = PyQuery(r.content)
-            self.assertIn('Revise', str(q("Title")))
-            test_bytes = b'this is some different text for a test'
-            test_file = BytesIO(test_bytes)
-            test_file.name = "also_some.txt"
-            r = self.client.post(url,dict(submission_method="upload",file=test_file,apply_to_all=True))
-            self.assertEqual(r.status_code, 302)
-            doc = Document.objects.get(pk=doc.pk)
-            self.assertEqual(doc.rev,'02')
-            self.assertTrue(session2.presentations.filter(document__type_id=doctype))
-            retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt")
-            self.assertEqual(retrieved_bytes, test_bytes)
-
-            # Test bad encoding
-            test_file = BytesIO('<html><h1>Title</h1><section>Some\x93text</section></html>'.encode('latin1'))
-            test_file.name = "some.html"
-            r = self.client.post(url,dict(submission_method="upload",file=test_file))
-            self.assertContains(r, 'Could not identify the file encoding')
-            doc = Document.objects.get(pk=doc.pk)
-            self.assertEqual(doc.rev,'02')
+            self.assertTrue(r.status_code==200)
+            if future:
+                self.assertContains(r, "Session has not ended yet")
+            else:
+                self.assertNotContains(r, "Session has not ended yet")
 
-            # Verify that we don't have dead links
-            url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
-            top = '/meeting/%s/' % session.meeting.number
-            self.requests_mock.get(f'{session.notes_url()}/download', text='markdown notes')
-            self.requests_mock.get(f'{session.notes_url()}/info', text=json.dumps({'title': 'title', 'updatetime': '2021-12-01T17:11:00z'}))
-            self.crawl_materials(url=url, top=top)
+    def test_upload_minutes_agenda(self):
+        for doctype in ('minutes','agenda'):
+            for future in (True, False):
+                mtg_date = date_today()+datetime.timedelta(days=180 if future else -180)
+                session = SessionFactory(meeting__type_id='ietf', meeting__date=mtg_date)
+                if doctype == 'minutes':
+                    url = urlreverse('ietf.meeting.views.upload_session_minutes',kwargs={'num':session.meeting.number,'session_id':session.id})
+                else:
+                    url = urlreverse('ietf.meeting.views.upload_session_agenda',kwargs={'num':session.meeting.number,'session_id':session.id})
+                self.client.logout()
+                login_testing_unauthorized(self,"secretary",url)
+                r = self.client.get(url)
+                self.assertEqual(r.status_code, 200)
+                q = PyQuery(r.content)
+                self.assertIn('Upload', str(q("Title")))
+                self.assertFalse(session.presentations.exists())
+                self.assertFalse(q('form input[type="checkbox"]'))
+                if future and doctype == "minutes":
+                    self.assertContains(r, "Session has not ended yet")
+                else:
+                    self.assertNotContains(r, "Session has not ended yet")
+        
+                session2 = SessionFactory(meeting=session.meeting,group=session.group)
+                r = self.client.get(url)
+                self.assertEqual(r.status_code, 200)
+                q = PyQuery(r.content)
+                self.assertTrue(q('form input[type="checkbox"]'))
+        
+                # test not submitting a file
+                r = self.client.post(url, dict(submission_method="upload"))
+                self.assertEqual(r.status_code, 200)
+                q = PyQuery(r.content)
+                self.assertTrue(q("form .is-invalid"))
+        
+                test_file = BytesIO(b'this is some text for a test')
+                test_file.name = "not_really.json"
+                r = self.client.post(url,dict(submission_method="upload",file=test_file))
+                self.assertEqual(r.status_code, 200)
+                q = PyQuery(r.content)
+                self.assertTrue(q('form .is-invalid'))
+        
+                test_file = BytesIO(b'this is some text for a test'*1510000)
+                test_file.name = "not_really.pdf"
+                r = self.client.post(url,dict(submission_method="upload",file=test_file))
+                self.assertEqual(r.status_code, 200)
+                q = PyQuery(r.content)
+                self.assertTrue(q('form .is-invalid'))
+        
+                test_file = BytesIO(b'<html><frameset><frame src="foo.html"></frame><frame src="bar.html"></frame></frameset></html>')
+                test_file.name = "not_really.html"
+                r = self.client.post(url,dict(submission_method="upload",file=test_file))
+                self.assertEqual(r.status_code, 200)
+                q = PyQuery(r.content)
+                self.assertTrue(q('form .is-invalid'))
+
+                # Test html sanitization
+                test_file = BytesIO(b'<html><head><title>Title</title></head><body><h1>Title</h1><section>Some text</section></body></html>')
+                test_file.name = "some.html"
+                r = self.client.post(url,dict(submission_method="upload",file=test_file))
+                self.assertEqual(r.status_code, 302)
+                doc = session.presentations.filter(document__type_id=doctype).first().document
+                self.assertEqual(doc.rev,'00')
+                text = doc.text()
+                self.assertIn('Some text', text)
+                self.assertNotIn('<section>', text)
+                text = retrieve_str(doctype, f"{doc.name}-{doc.rev}.html")
+                self.assertIn('Some text', text)
+                self.assertNotIn('<section>', text)
+
+                # txt upload
+                test_bytes = b'This is some text for a test, with the word\nvirtual at the beginning of a line.'
+                test_file = BytesIO(test_bytes)
+                test_file.name = "some.txt"
+                r = self.client.post(url,dict(submission_method="upload",file=test_file,apply_to_all=False))
+                self.assertEqual(r.status_code, 302)
+                doc = session.presentations.filter(document__type_id=doctype).first().document
+                self.assertEqual(doc.rev,'01')
+                self.assertFalse(session2.presentations.filter(document__type_id=doctype))
+                retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt")
+                self.assertEqual(retrieved_bytes, test_bytes)
+        
+                r = self.client.get(url)
+                self.assertEqual(r.status_code, 200)
+                q = PyQuery(r.content)
+                self.assertIn('Revise', str(q("Title")))
+                test_bytes = b'this is some different text for a test'
+                test_file = BytesIO(test_bytes)
+                test_file.name = "also_some.txt"
+                r = self.client.post(url,dict(submission_method="upload",file=test_file,apply_to_all=True))
+                self.assertEqual(r.status_code, 302)
+                doc = Document.objects.get(pk=doc.pk)
+                self.assertEqual(doc.rev,'02')
+                self.assertTrue(session2.presentations.filter(document__type_id=doctype))
+                retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt")
+                self.assertEqual(retrieved_bytes, test_bytes)
+
+                # Test bad encoding
+                test_file = BytesIO('<html><h1>Title</h1><section>Some\x93text</section></html>'.encode('latin1'))
+                test_file.name = "some.html"
+                r = self.client.post(url,dict(submission_method="upload",file=test_file))
+                self.assertContains(r, 'Could not identify the file encoding')
+                doc = Document.objects.get(pk=doc.pk)
+                self.assertEqual(doc.rev,'02')
+
+                # Verify that we don't have dead links
+                url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
+                top = '/meeting/%s/' % session.meeting.number
+                self.requests_mock.get(f'{session.notes_url()}/download', text='markdown notes')
+                self.requests_mock.get(f'{session.notes_url()}/info', text=json.dumps({'title': 'title', 'updatetime': '2021-12-01T17:11:00z'}))
+                self.crawl_materials(url=url, top=top)
 
     def test_upload_minutes_agenda_unscheduled(self):
         for doctype in ('minutes','agenda'):
@@ -6661,6 +6681,7 @@ def test_upload_minutes_agenda_unscheduled(self):
             self.assertIn('Upload', str(q("Title")))
             self.assertFalse(session.presentations.exists())
             self.assertFalse(q('form input[type="checkbox"]'))
+            self.assertNotContains(r, "Session has not ended yet")
 
             test_file = BytesIO(b'this is some text for a test')
             test_file.name = "not_really.txt"
@@ -6669,35 +6690,40 @@ def test_upload_minutes_agenda_unscheduled(self):
 
     @override_settings(MEETING_MATERIALS_SERVE_LOCALLY=True)
     def test_upload_minutes_agenda_interim(self):
-        session=SessionFactory(meeting__type_id='interim')
         for doctype in ('minutes','agenda'):
-            if doctype=='minutes':
-                url = urlreverse('ietf.meeting.views.upload_session_minutes',kwargs={'num':session.meeting.number,'session_id':session.id})
-            else:
-                url = urlreverse('ietf.meeting.views.upload_session_agenda',kwargs={'num':session.meeting.number,'session_id':session.id})
-            self.client.logout()
-            login_testing_unauthorized(self,"secretary",url)
-            r = self.client.get(url)
-            self.assertEqual(r.status_code, 200)
-            q = PyQuery(r.content)
-            self.assertIn('Upload', str(q("title")))
-            self.assertFalse(session.presentations.filter(document__type_id=doctype))
-            test_bytes = b'this is some text for a test'
-            test_file = BytesIO(test_bytes)
-            test_file.name = "not_really.txt"
-            r = self.client.post(url,dict(submission_method="upload",file=test_file))
-            self.assertEqual(r.status_code, 302)
-            doc = session.presentations.filter(document__type_id=doctype).first().document
-            self.assertEqual(doc.rev,'00')
-            retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt")
-            self.assertEqual(retrieved_bytes, test_bytes)
-
-            # Verify that we don't have dead links
-            url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
-            top = '/meeting/%s/' % session.meeting.number
-            self.requests_mock.get(f'{session.notes_url()}/download', text='markdown notes')
-            self.requests_mock.get(f'{session.notes_url()}/info', text=json.dumps({'title': 'title', 'updatetime': '2021-12-01T17:11:00z'}))
-            self.crawl_materials(url=url, top=top)
+            for future in (True, False):
+                session=SessionFactory(meeting__type_id='interim', meeting__date = date_today()+datetime.timedelta(days=180 if future else -180))
+                if doctype=='minutes':
+                    url = urlreverse('ietf.meeting.views.upload_session_minutes',kwargs={'num':session.meeting.number,'session_id':session.id})
+                else:
+                    url = urlreverse('ietf.meeting.views.upload_session_agenda',kwargs={'num':session.meeting.number,'session_id':session.id})
+                self.client.logout()
+                login_testing_unauthorized(self,"secretary",url)
+                r = self.client.get(url)
+                self.assertEqual(r.status_code, 200)
+                q = PyQuery(r.content)
+                self.assertIn('Upload', str(q("title")))
+                self.assertFalse(session.presentations.filter(document__type_id=doctype))
+                if future and doctype == "minutes":
+                    self.assertContains(r, "Session has not ended yet")
+                else:
+                    self.assertNotContains(r, "Session has not ended yet")
+                test_bytes = b'this is some text for a test'
+                test_file = BytesIO(test_bytes)
+                test_file.name = "not_really.txt"
+                r = self.client.post(url,dict(submission_method="upload",file=test_file))
+                self.assertEqual(r.status_code, 302)
+                doc = session.presentations.filter(document__type_id=doctype).first().document
+                self.assertEqual(doc.rev,'00')
+                retrieved_bytes = retrieve_bytes(doctype, f"{doc.name}-{doc.rev}.txt")
+                self.assertEqual(retrieved_bytes, test_bytes)
+
+                # Verify that we don't have dead links
+                url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
+                top = '/meeting/%s/' % session.meeting.number
+                self.requests_mock.get(f'{session.notes_url()}/download', text='markdown notes')
+                self.requests_mock.get(f'{session.notes_url()}/info', text=json.dumps({'title': 'title', 'updatetime': '2021-12-01T17:11:00z'}))
+                self.crawl_materials(url=url, top=top)
 
     @override_settings(MEETING_MATERIALS_SERVE_LOCALLY=True)
     def test_upload_narrativeminutes(self):
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 6a73059d92..722bf829e1 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -2522,6 +2522,8 @@ def session_details(request, num, acronym):
     else:
         pending_suggestions = SlideSubmission.objects.none()
 
+    tsa = session.official_timeslotassignment()
+    future = tsa is not None and timezone.now() < tsa.timeslot.end_time()
     return render(request, "meeting/session_details.html",
                   { 'scheduled_sessions':scheduled_sessions ,
                     'unscheduled_sessions':unscheduled_sessions , 
@@ -2532,6 +2534,7 @@ def session_details(request, num, acronym):
                     'can_manage_materials' : can_manage,
                     'can_view_request': can_view_request,
                     'thisweek': datetime_today()-datetime.timedelta(days=7),
+                    'future': future,
                   })
 
 class SessionDraftsForm(forms.Form):
@@ -2823,11 +2826,14 @@ def upload_session_minutes(request, session_id, num):
     else:
         form = UploadMinutesForm(show_apply_to_all_checkbox)
 
+    tsa = session.official_timeslotassignment()
+    future = tsa is not None and timezone.now() < tsa.timeslot.end_time()
     return render(request, "meeting/upload_session_minutes.html", 
                   {'session': session,
                    'session_number': session_number,
                    'minutes_sp' : minutes_sp,
                    'form': form,
+                   'future': future,
                   })
 
 @role_required("Secretariat")
diff --git a/ietf/templates/meeting/session_details_panel.html b/ietf/templates/meeting/session_details_panel.html
index 87d9e3d672..7c52ac0b4a 100644
--- a/ietf/templates/meeting/session_details_panel.html
+++ b/ietf/templates/meeting/session_details_panel.html
@@ -109,7 +109,7 @@ <h3 class="mt-4">Agenda, Minutes, and Bluesheets</h3>
                 </a>
             {% endif %}
             {% if not session.type_counter.minutes %}
-                <a class="btn btn-primary" href="{% url 'ietf.meeting.views.import_session_minutes' num=session.meeting.number session_id=session.pk %}">Import minutes from notes.ietf.org</a>
+                <a class="btn {% if future %}btn-warning" title="Session has not ended yet{% else %}btn-primary{% endif %}" href="{% url 'ietf.meeting.views.import_session_minutes' num=session.meeting.number session_id=session.pk %}">Import minutes from notes.ietf.org</a>
                 <a class="btn btn-primary"
                    href="{% url 'ietf.meeting.views.upload_session_minutes' session_id=session.pk num=session.meeting.number %}">
                     Upload minutes
diff --git a/ietf/templates/meeting/upload_session_minutes.html b/ietf/templates/meeting/upload_session_minutes.html
index 30eadda277..324440681f 100644
--- a/ietf/templates/meeting/upload_session_minutes.html
+++ b/ietf/templates/meeting/upload_session_minutes.html
@@ -26,6 +26,11 @@ <h1>
     {% if session_number %}
         <h2>Session {{ session_number }} : {{ session.official_timeslotassignment.timeslot.time|timezone:session.meeting.time_zone|date:"D M-d-Y Hi" }}</h2>
     {% endif %}
+    {% if future %}
+    <p class="alert alert-warning my-3">
+        Caution: Session has not ended yet
+    </p>
+    {% endif %}
     <form enctype="multipart/form-data" method="post" class="my-3">
         {% csrf_token %}
         {% bootstrap_form form %}

From c70e67dc917dea9fe0347575811d5ab02ef91ce8 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 2 Apr 2025 13:11:08 -0300
Subject: [PATCH 308/656] chore: unpin importlib-metadata (#8762)

---
 requirements.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 8bd906c220..cd93f448e2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -42,7 +42,6 @@ gunicorn>=20.1.0
 hashids>=1.3.1
 html2text>=2020.1.16    # Used only to clean comment field of secr/sreq
 html5lib>=1.1    # Only used in tests
-importlib-metadata<8.5.0  # indirect req of Markdown/inflect; https://github.com/ietf-tools/datatracker/issues/7924
 inflect>= 6.0.2
 jsonfield>=3.1.0    # for SubmissionCheck.  This is https://github.com/bradjasper/django-jsonfield/.
 jsonschema[format]>=4.2.1

From b95bbe670fb1f3506ef156034fd5107c1dca3365 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Wed, 2 Apr 2025 16:24:20 +0000
Subject: [PATCH 309/656] ci: update base image target version to 20250402T1611

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index c25298d652..cd0a70667c 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250327T1859
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250402T1611
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 50e8bfd839..1195fc9a0b 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250327T1859
+20250402T1611

From 2b7d4ad414e61b001cab30703a270929852a6b57 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 4 Apr 2025 02:19:43 -0400
Subject: [PATCH 310/656] ci: add update staging DB step to deploy workflow

---
 .github/workflows/build.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 123bd5c65a..36439aa6cf 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -444,6 +444,19 @@ jobs:
       PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
       
     steps:
+      - name: Refresh Staging DB
+        uses: the-actions-org/workflow-dispatch@v4
+        with:
+          workflow: update-staging-db.yml
+          repo: ietf-tools/infra-k8s
+          ref: main
+          token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
+          inputs: '{ "sourceDb":"datatracker" }'
+          wait-for-completion: true
+          wait-for-completion-timeout: 5m
+          wait-for-completion-interval: 20s
+          display-workflow-run-url: false
+
       - name: Deploy to staging
         uses: the-actions-org/workflow-dispatch@v4
         with:

From f9ca03086493a00ca188cb4cd949ba1831fd452a Mon Sep 17 00:00:00 2001
From: Rudi Matz <rudi@staff.ietf.org>
Date: Tue, 8 Apr 2025 20:24:00 -0400
Subject: [PATCH 311/656] fix: unexpected chatlog links (#8774)

* fix: unexpected chatlog links

* fix: separate handling of polls and chatlogs
---
 ietf/meeting/tests_views.py                   | 38 +++++++++++++++++++
 ietf/meeting/views.py                         |  8 +++-
 .../meeting/session_buttons_include.html      | 11 +++---
 3 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index a93a26b981..dfb414b61b 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -7736,6 +7736,44 @@ def test_add_session_recordings(self):
         self.assertEqual(r.status_code, 404)
         self.assertFalse(mock_delete.called)
 
+    def test_show_chatlog_links(self):
+        meeting = MeetingFactory(type_id='ietf', number='122')
+        session = SessionFactory(meeting=meeting)
+        doc_name = 'chatlog-72-mars-197001010000'
+        SessionPresentation.objects.create(session=session,document=DocumentFactory(type_id='chatlog', name=doc_name))
+
+        session_url = urlreverse('ietf.meeting.views.session_details', 
+                                 kwargs={'num':meeting.number, 'acronym':session.group.acronym})
+
+        r = self.client.get(session_url)
+
+        self.assertEqual(r.status_code, 200)
+
+        q = PyQuery(r.content)
+
+        # Find the chatlog link in the desktop view
+        link_chatlog_box = q(f'a[title="Chat logs for {session.group.acronym}"]')
+        self.assertTrue(link_chatlog_box, 'Expected <a> element with title "Chat logs for {group.acronym}" not found.')
+        self.assertEqual(link_chatlog_box.attr('href'), '/doc/'+ doc_name)
+
+        # Find the chatlog link in the mobile view
+        link_chatlog_list = q('li:contains("Chat logs")')
+        self.assertTrue(link_chatlog_list, 'Expected <li> element containing "Chat logs" not found.')
+        self.assertEqual(link_chatlog_list.find('a').attr('href'), '/doc/'+ doc_name)
+
+    def test_hide_chatlog_links(self):
+        # mock meeting and session, but no chatlog document
+        meeting = MeetingFactory(type_id='ietf', number='122')
+        session = SessionFactory(meeting=meeting)
+
+        session_url = urlreverse('ietf.meeting.views.session_details', 
+                                 kwargs={'num':meeting.number, 'acronym':session.group.acronym})
+
+        r = self.client.get(session_url)
+
+        self.assertEqual(r.status_code, 200)
+        # validate no links for chat logs exist
+        self.assertNotContains(r, 'Chat logs')
 
 
 class HasMeetingsTests(TestCase):
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 722bf829e1..591b934b58 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -23,6 +23,7 @@
 from urllib.parse import parse_qs, unquote, urlencode, urlsplit, urlunsplit, urlparse
 from tempfile import mkstemp
 from wsgiref.handlers import format_date_time
+from itertools import chain
 
 from django import forms
 from django.core.cache import caches
@@ -2496,7 +2497,12 @@ def session_details(request, num, acronym):
         session.filtered_artifacts.sort(key=lambda d:artifact_types.index(d.document.type.slug))
         session.filtered_slides    = session.presentations.filter(document__type__slug='slides').order_by('order')
         session.filtered_drafts    = session.presentations.filter(document__type__slug='draft')
-        session.filtered_chatlog_and_polls = session.presentations.filter(document__type__slug__in=('chatlog', 'polls')).order_by('document__type__slug')
+        
+        filtered_polls = session.presentations.filter(document__type__slug=('polls'))
+        filtered_chatlogs = session.presentations.filter(document__type__slug=('chatlog'))
+        session.filtered_chatlog_and_polls = chain(filtered_chatlogs, filtered_polls)
+        session.chatlog = filtered_chatlogs.first()
+
         # TODO FIXME Deleted materials shouldn't be in the presentations
         for qs in [session.filtered_artifacts,session.filtered_slides,session.filtered_drafts]:
             qs = [p for p in qs if p.document.get_state_slug(p.document.type_id)!='deleted']
diff --git a/ietf/templates/meeting/session_buttons_include.html b/ietf/templates/meeting/session_buttons_include.html
index bb5144b45d..b1fd92ba5b 100644
--- a/ietf/templates/meeting/session_buttons_include.html
+++ b/ietf/templates/meeting/session_buttons_include.html
@@ -2,7 +2,8 @@
 {% load origin %}
 {% load static %}
 {% load textfilters tz %}
-{% load ietf_filters %}
+{% load ietf_filters session_filters %}
+
 {% origin %}
 {% if item and item|should_show_agenda_session_buttons %}
     {% with slug=item.slug session=item.session timeslot=item.timeslot %}
@@ -126,10 +127,10 @@
                     </a>
                 {% else %}
                     {# chat logs #}
-                    {% if meeting.has_chat_logs %}
+                    {% if meeting.has_chat_logs and session.chatlog %}
                         <a class="btn btn-outline-primary"
                             role="button"
-                            href="{{session.chat_archive_url}}"
+                            href="/doc/{{ session.chatlog.document.name }}"
                             aria-label="Chat logs for {{ session.chat_room_name }}"
                             title="Chat logs for {{ session.chat_room_name }}">
                             <i class="bi bi-file-text"></i>
@@ -303,10 +304,10 @@
                         </li>
                     {% else %}
                         {# chat logs #}
-			{% if meeting.has_chat_logs %}
+			            {% if meeting.has_chat_logs and session.chatlog %}
                             <li>
                                 <a class="dropdown-item"
-                                    href="session.chat_room_url">
+                                    href="/doc/{{ session.chatlog.document.name }}">
                                     <i class="bi bi-file-text"></i> Chat logs
                                 </a>
                             </li>

From a728cf2440ab5f4acc88714023d3b2e2f834018b Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 8 Apr 2025 21:24:22 -0300
Subject: [PATCH 312/656] test: fix apparent typo / tautological test (#8773)

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/api/tests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index ac0b37a608..7cc56c2d0a 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -1440,7 +1440,7 @@ def test_api_top_level(self):
         resource_list = r.json()
 
         for name in self.apps:
-            if not name in self.apps:
+            if not name in resource_list:
                 sys.stderr.write("Expected a REST API resource for %s, but didn't find one\n" % name)
 
         for name in self.apps:

From 0348dbd865b99692235c2f6e5a75a8064f8113a1 Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Tue, 15 Apr 2025 07:37:40 -0700
Subject: [PATCH 313/656] feat: new meeting registration implementation (#8408)

* feat: new meeting registration implementation

* fix: use on_delete=models.PROTECT for certain FKs

* fix: convert outlier reg_types,ticket_types on initial migration

* fix: fix initial migration mistake

* fix: remove management command. add task

* fix: move migration 0010 to 0011 to resolve conflict

* fix: add missing migration for model changes

* fix: add reg type names for unknown

* fix: change migration to use 'unknown'. Add test function

* fix: merge migrations

* fix: rename test_migrate_registrations to check_migrate_registrations

* fix: update names.json

* fix: fix migration issue and add task

* fix: fix broken migration dependency
---
 ietf/api/tests.py                             | 193 +++++++++++++++++-
 ietf/api/urls.py                              |   1 +
 ietf/api/views.py                             | 143 ++++++++++++-
 ietf/meeting/admin.py                         |  32 ++-
 ietf/meeting/factories.py                     |  28 ++-
 .../0011_registration_registrationticket.py   |  90 ++++++++
 ietf/meeting/models.py                        |  41 +++-
 ietf/meeting/resources.py                     |  43 +++-
 ietf/meeting/tasks.py                         |  17 ++
 ietf/meeting/tests_utils.py                   |  91 +++++++++
 ietf/meeting/utils.py                         | 109 +++++++++-
 ietf/name/admin.py                            |   4 +
 ietf/name/fixtures/names.json                 | 140 +++++++++++++
 ...ancetypename_registrationtickettypename.py |  47 +++++
 .../migrations/0017_populate_new_reg_names.py |  39 ++++
 ietf/name/models.py                           |   5 +-
 ietf/name/resources.py                        |  32 ++-
 17 files changed, 1045 insertions(+), 10 deletions(-)
 create mode 100644 ietf/meeting/migrations/0011_registration_registrationticket.py
 create mode 100644 ietf/meeting/tests_utils.py
 create mode 100644 ietf/name/migrations/0016_attendancetypename_registrationtickettypename.py
 create mode 100644 ietf/name/migrations/0017_populate_new_reg_names.py

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 7cc56c2d0a..23e9ff0103 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -1,6 +1,7 @@
 # Copyright The IETF Trust 2015-2024, All Rights Reserved
 # -*- coding: utf-8 -*-
 import base64
+import copy
 import datetime
 import json
 import html
@@ -31,7 +32,7 @@
 from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory
 from ietf.group.factories import RoleFactory
 from ietf.meeting.factories import MeetingFactory, SessionFactory
-from ietf.meeting.models import Session
+from ietf.meeting.models import Session, Registration
 from ietf.nomcom.models import Volunteer
 from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year
 from ietf.person.factories import PersonFactory, random_faker, EmailFactory, PersonalApiKeyFactory
@@ -828,6 +829,196 @@ def test_api_new_meeting_registration_nomcom_volunteer(self):
         self.assertEqual(volunteer.nomcom, nomcom)
         self.assertEqual(volunteer.origin, 'registration')
 
+    @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]})
+    def test_api_new_meeting_registration_v2(self):
+        meeting = MeetingFactory(type_id='ietf')
+        person = PersonFactory()
+        regs = [
+            {
+                'affiliation': "Alguma Corporação",
+                'country_code': 'PT',
+                'email': person.email().address,
+                'first_name': person.first_name(),
+                'last_name': person.last_name(),
+                'meeting': str(meeting.number),
+                'reg_type': 'onsite',
+                'ticket_type': 'week_pass',
+                'checkedin': False,
+                'is_nomcom_volunteer': False,
+                'cancelled': False,
+            }
+        ]
+
+        url = urlreverse('ietf.api.views.api_new_meeting_registration_v2')
+        #
+        # Test invalid key
+        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "invalid-token"})
+        self.assertEqual(r.status_code, 403)
+        #
+        # Test invalid data
+        bad_regs = copy.deepcopy(regs)
+        del(bad_regs[0]['email'])
+        r = self.client.post(url, data=json.dumps(bad_regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 400)
+        #
+        # Test valid POST
+        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        self.assertContains(r, "Success", status_code=202)
+        #
+        # Check record
+        reg = regs[0]
+        objects = Registration.objects.filter(email=reg['email'], meeting__number=reg['meeting'])
+        self.assertEqual(objects.count(), 1)
+        obj = objects[0]
+        for key in ['affiliation', 'country_code', 'first_name', 'last_name', 'checkedin']:
+            self.assertEqual(getattr(obj, key), False if key=='checkedin' else reg.get(key) , "Bad data for field '%s'" % key)
+        self.assertEqual(obj.tickets.count(), 1)
+        ticket = obj.tickets.first()
+        self.assertEqual(ticket.ticket_type.slug, regs[0]['ticket_type'])
+        self.assertEqual(ticket.attendance_type.slug, regs[0]['reg_type'])
+        self.assertEqual(obj.person, person)
+        #
+        # Test update (switch to remote)
+        regs = [
+            {
+                'affiliation': "Alguma Corporação",
+                'country_code': 'PT',
+                'email': person.email().address,
+                'first_name': person.first_name(),
+                'last_name': person.last_name(),
+                'meeting': str(meeting.number),
+                'reg_type': 'remote',
+                'ticket_type': 'week_pass',
+                'checkedin': False,
+                'is_nomcom_volunteer': False,
+                'cancelled': False,
+            }
+        ]
+        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        self.assertContains(r, "Success", status_code=202)
+        objects = Registration.objects.filter(email=reg['email'], meeting__number=reg['meeting'])
+        self.assertEqual(objects.count(), 1)
+        obj = objects[0]
+        self.assertEqual(obj.tickets.count(), 1)
+        ticket = obj.tickets.first()
+        self.assertEqual(ticket.ticket_type.slug, regs[0]['ticket_type'])
+        self.assertEqual(ticket.attendance_type.slug, regs[0]['reg_type'])
+        #
+        # Test multiple
+        regs = [
+            {
+                'affiliation': "Alguma Corporação",
+                'country_code': 'PT',
+                'email': person.email().address,
+                'first_name': person.first_name(),
+                'last_name': person.last_name(),
+                'meeting': str(meeting.number),
+                'reg_type': 'onsite',
+                'ticket_type': 'one_day',
+                'checkedin': False,
+                'is_nomcom_volunteer': False,
+                'cancelled': False,
+            },
+
+            {
+                'affiliation': "Alguma Corporação",
+                'country_code': 'PT',
+                'email': person.email().address,
+                'first_name': person.first_name(),
+                'last_name': person.last_name(),
+                'meeting': str(meeting.number),
+                'reg_type': 'remote',
+                'ticket_type': 'week_pass',
+                'checkedin': False,
+                'is_nomcom_volunteer': False,
+                'cancelled': False,
+            }
+        ]
+
+        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        self.assertContains(r, "Success", status_code=202)
+        objects = Registration.objects.filter(email=reg['email'], meeting__number=reg['meeting'])
+        self.assertEqual(objects.count(), 1)
+        obj = objects[0]
+        self.assertEqual(obj.tickets.count(), 2)
+        self.assertEqual(obj.tickets.filter(attendance_type__slug='onsite').count(), 1)
+        self.assertEqual(obj.tickets.filter(attendance_type__slug='remote').count(), 1)
+
+    @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]})
+    def test_api_new_meeting_registration_v2_cancelled(self):
+        meeting = MeetingFactory(type_id='ietf')
+        person = PersonFactory()
+        regs = [
+            {
+                'affiliation': "Acme",
+                'country_code': 'US',
+                'email': person.email().address,
+                'first_name': person.first_name(),
+                'last_name': person.last_name(),
+                'meeting': str(meeting.number),
+                'reg_type': 'onsite',
+                'ticket_type': 'week_pass',
+                'checkedin': False,
+                'is_nomcom_volunteer': False,
+                'cancelled': False,
+            }
+        ]
+        url = urlreverse('ietf.api.views.api_new_meeting_registration_v2')
+        self.assertEqual(Registration.objects.count(), 0)
+        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        self.assertContains(r, "Success", status_code=202)
+        self.assertEqual(Registration.objects.count(), 1)
+        regs[0]['cancelled'] = True
+        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        self.assertContains(r, "Success", status_code=202)
+        self.assertEqual(Registration.objects.count(), 0)
+
+    @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]})    
+    def test_api_new_meeting_registration_v2_nomcom(self):
+        meeting = MeetingFactory(type_id='ietf')
+        person = PersonFactory()
+        regs = [
+            {
+                'affiliation': "Acme",
+                'country_code': 'US',
+                'email': person.email().address,
+                'first_name': person.first_name(),
+                'last_name': person.last_name(),
+                'meeting': str(meeting.number),
+                'reg_type': 'onsite',
+                'ticket_type': 'week_pass',
+                'checkedin': False,
+                'is_nomcom_volunteer': False,
+                'cancelled': False,
+            }
+        ]
+
+        url = urlreverse('ietf.api.views.api_new_meeting_registration_v2')
+        now = datetime.datetime.now()
+        if now.month > 10:
+            year = now.year + 1
+        else:
+            year = now.year
+        # create appropriate group and nomcom objects
+        nomcom = NomComFactory.create(is_accepting_volunteers=True, **nomcom_kwargs_for_year(year))
+
+        # first test is_nomcom_volunteer False
+        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        self.assertContains(r, "Success", status_code=202)
+        # assert no Volunteers exists
+        self.assertEqual(Volunteer.objects.count(), 0)
+
+        # test is_nomcom_volunteer True
+        regs[0]['is_nomcom_volunteer'] = True
+        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        self.assertContains(r, "Success", status_code=202)
+        # assert Volunteer exists
+        self.assertEqual(Volunteer.objects.count(), 1)
+        volunteer = Volunteer.objects.last()
+        self.assertEqual(volunteer.person, person)
+        self.assertEqual(volunteer.nomcom, nomcom)
+        self.assertEqual(volunteer.origin, 'registration')
+
     def test_api_version(self):
         DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=datetime.timezone.utc), host='testapi.example.com',tz='UTC')
         url = urlreverse('ietf.api.views.version')
diff --git a/ietf/api/urls.py b/ietf/api/urls.py
index b0dbaf91ce..2fcaba6094 100644
--- a/ietf/api/urls.py
+++ b/ietf/api/urls.py
@@ -66,6 +66,7 @@
     # Let MeetEcho upload session polls
     url(r'^notify/session/polls/?$', meeting_views.api_upload_polls),    
     # Let the registration system notify us about registrations
+    url(r'^notify/meeting/registration/v2/?', api_views.api_new_meeting_registration_v2),
     url(r'^notify/meeting/registration/?', api_views.api_new_meeting_registration),
     # OpenID authentication provider
     url(r'^openid/$', TemplateView.as_view(template_name='api/openid-issuer.html'), name='ietf.api.urls.oidc_issuer'),
diff --git a/ietf/api/views.py b/ietf/api/views.py
index 2fd9d2730f..4cc3560627 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -45,7 +45,7 @@
 from ietf.ietfauth.utils import role_required
 from ietf.ietfauth.views import send_account_creation_email
 from ietf.ipr.utils import ingest_response_email as ipr_ingest_response_email
-from ietf.meeting.models import Meeting
+from ietf.meeting.models import Meeting, Registration
 from ietf.nomcom.models import Volunteer, NomCom
 from ietf.nomcom.utils import ingest_feedback_email as nomcom_ingest_feedback_email
 from ietf.person.models import Person, Email
@@ -236,6 +236,147 @@ def err(code, text):
         return HttpResponse(status=405)
 
 
+_new_registration_json_validator = jsonschema.Draft202012Validator(
+    schema={
+        "type": "array",
+        "items": {
+            "type": "object",
+            "properties": {
+                "meeting": {"type": "string"},
+                "first_name": {"type": "string"},
+                "last_name": {"type": "string"},
+                "affiliation": {"type": "string"},
+                "country_code": {"type": "string"},
+                "email": {"type": "string"},
+                "reg_type": {"type": "string"},
+                "ticket_type": {"type": "string"},
+                "checkedin": {"type": "boolean"},
+                "is_nomcom_volunteer": {"type": "boolean"},
+                "cancelled": {"type": "boolean"},
+            },
+            "required": ["meeting", "first_name", "last_name", "affiliation", "country_code", "email", "reg_type", "ticket_type", "checkedin", "is_nomcom_volunteer", "cancelled"],
+            "additionalProperties": "false"
+        }
+    }
+)
+
+
+@requires_api_token
+@csrf_exempt
+def api_new_meeting_registration_v2(request):
+    '''REST API to notify the datatracker about a new meeting registration'''
+    def _http_err(code, text):
+        return HttpResponse(text, status=code, content_type="text/plain")
+
+    def _api_response(result):
+        return JsonResponse(data={"result": result})
+
+    if request.method != "POST":
+        return _http_err(405, "Method not allowed")
+
+    if request.content_type != "application/json":
+        return _http_err(415, "Content-Type must be application/json")
+
+    # Validate
+    try:
+        payload = json.loads(request.body)
+        _new_registration_json_validator.validate(payload)
+    except json.decoder.JSONDecodeError as err:
+        return _http_err(400, f"JSON parse error at line {err.lineno} col {err.colno}: {err.msg}")
+    except jsonschema.exceptions.ValidationError as err:
+        return _http_err(400, f"JSON schema error at {err.json_path}: {err.message}")
+    except Exception:
+        return _http_err(400, "Invalid request format")
+
+    # Validate consistency
+    # - if receive multiple records they should be for same meeting, same person (email)
+    if len(payload) > 1:
+        if len(set([r['meeting'] for r in payload])) != 1:
+            return _http_err(400, "Different meeting values")
+        if len(set([r['email'] for r in payload])) != 1:
+            return _http_err(400, "Different email values")
+
+    # Validate meeting
+    number = payload[0]['meeting']
+    try:
+        meeting = Meeting.objects.get(number=number)
+    except Meeting.DoesNotExist:
+        return _http_err(400, "Invalid meeting value: '%s'" % (number, ))
+
+    # Validate email
+    email = payload[0]['email']
+    try:
+        validate_email(email)
+    except ValidationError:
+        return _http_err(400, "Invalid email value: '%s'" % (email, ))
+
+    # get person
+    person = Person.objects.filter(email__address=email).first()
+    if not person:
+        log.log(f"api_new_meeting_registration_v2 no Person found for {email}")
+
+    registration = payload[0]
+    # handle cancelled
+    if registration['cancelled']:
+        if len(payload) > 1:
+            return _http_err(400, "Error. Received cancelled registration notification with more than one record. ({})".format(email))
+        try:
+            obj = Registration.objects.get(meeting=meeting, email=email)
+        except Registration.DoesNotExist:
+            return _http_err(400, "Error. Received cancelled registration notification for non-existing registration. ({})".format(email))
+        if obj.tickets.count() == 1:
+            obj.delete()
+        else:
+            obj.tickets.filter(
+                attendance_type__slug=registration.reg_type,
+                ticket_type__slug=registration.ticket_type).delete()
+        return HttpResponse('Success', status=202, content_type='text/plain')
+
+    # create or update MeetingRegistration
+    update_fields = ['first_name', 'last_name', 'affiliation', 'country_code', 'checkedin', 'is_nomcom_volunteer']
+    try:
+        reg = Registration.objects.get(meeting=meeting, email=email)
+        for key, value in registration.items():
+            if key in update_fields:
+                setattr(reg, key, value)
+        reg.save()
+    except Registration.DoesNotExist:
+        reg = Registration.objects.create(
+            meeting_id=meeting.pk,
+            person=person,
+            email=email,
+            first_name=registration['first_name'],
+            last_name=registration['last_name'],
+            affiliation=registration['affiliation'],
+            country_code=registration['country_code'],
+            checkedin=registration['checkedin'])
+
+    # handle registration tickets
+    reg.tickets.all().delete()
+    for registration in payload:
+        reg.tickets.create(
+            attendance_type_id=registration['reg_type'],
+            ticket_type_id=registration['ticket_type'],
+        )
+        # handle nomcom volunteer
+        if registration['is_nomcom_volunteer'] and person:
+            try:
+                nomcom = NomCom.objects.get(is_accepting_volunteers=True)
+            except (NomCom.DoesNotExist, NomCom.MultipleObjectsReturned):
+                nomcom = None
+            if nomcom:
+                Volunteer.objects.get_or_create(
+                    nomcom=nomcom,
+                    person=person,
+                    defaults={
+                        "affiliation": registration["affiliation"],
+                        "origin": "registration"
+                    }
+                )
+
+    return HttpResponse('Success', status=202, content_type='text/plain')
+
+
 def version(request):
     dumpdate = None
     dumpinfo = DumpInfo.objects.order_by('-date').first()
diff --git a/ietf/meeting/admin.py b/ietf/meeting/admin.py
index b7e56c7b77..5d6adb7294 100644
--- a/ietf/meeting/admin.py
+++ b/ietf/meeting/admin.py
@@ -7,7 +7,7 @@
 from ietf.meeting.models import (Attended, Meeting, Room, Session, TimeSlot, Constraint, Schedule,
     SchedTimeSessAssignment, ResourceAssociation, FloorPlan, UrlResource,
     SessionPresentation, ImportantDate, SlideSubmission, SchedulingEvent, BusinessConstraint,
-    ProceedingsMaterial, MeetingHost)
+    ProceedingsMaterial, MeetingHost, Registration, RegistrationTicket)
 
 
 class UrlResourceAdmin(admin.ModelAdmin):
@@ -213,3 +213,33 @@ class AttendedAdmin(admin.ModelAdmin):
     search_fields = ["person__name", "session__group__acronym", "session__meeting__number", "session__name", "session__purpose__name"]
     raw_id_fields= ["person", "session"]
 admin.site.register(Attended, AttendedAdmin)
+
+class MeetingFilter(admin.SimpleListFilter):
+    title = 'Meeting Filter'
+    parameter_name = 'meeting_id'
+
+    def lookups(self, request, model_admin):
+        # Your queryset to limit choices
+        choices = Meeting.objects.filter(type='ietf').values_list('id', 'number')
+        return choices
+
+    def queryset(self, request, queryset):
+        if self.value():
+            return queryset.filter(meeting__id=self.value())
+        return queryset
+class RegistrationAdmin(admin.ModelAdmin):
+    model = Registration
+    # list_filter = [('meeting', Meeting.objects.filter(type='ietf')), ]
+    list_filter = [MeetingFilter, ]
+    list_display = ['meeting', 'first_name', 'last_name', 'affiliation', 'country_code', 'person', 'email', ]
+    search_fields = ['meeting__number', 'first_name', 'last_name', 'affiliation', 'country_code', 'email', ]
+    raw_id_fields = ['person']
+admin.site.register(Registration, RegistrationAdmin)
+
+class RegistrationTicketAdmin(admin.ModelAdmin):
+    model = RegistrationTicket
+    list_filter = ['attendance_type', ]
+    list_display = ['registration', 'attendance_type', 'ticket_type']
+    search_fields = ['registration__first_name', 'registration__last_name', 'registration__email']
+    raw_id_fields = ['registration']
+admin.site.register(RegistrationTicket, RegistrationTicketAdmin)
diff --git a/ietf/meeting/factories.py b/ietf/meeting/factories.py
index eb36e9e756..b3d21830fe 100644
--- a/ietf/meeting/factories.py
+++ b/ietf/meeting/factories.py
@@ -12,7 +12,7 @@
 from ietf.doc.storage_utils import store_str
 from ietf.meeting.models import (Attended, Meeting, Session, SchedulingEvent, Schedule,
     TimeSlot, SessionPresentation, FloorPlan, Room, SlideSubmission, Constraint,
-    MeetingHost, ProceedingsMaterial)
+    MeetingHost, ProceedingsMaterial, Registration, RegistrationTicket)
 from ietf.name.models import (ConstraintName, SessionStatusName, ProceedingsMaterialTypeName,
                               TimerangeName, SessionPurposeName)
 from ietf.doc.factories import ProceedingsMaterialDocFactory
@@ -318,3 +318,29 @@ class Meta:
 
     session = factory.SubFactory(SessionFactory)
     person = factory.SubFactory(PersonFactory)
+
+
+class RegistrationFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = Registration
+        skip_postgeneration_save = True
+
+    meeting = factory.SubFactory(MeetingFactory)
+    person = factory.SubFactory(PersonFactory)
+    email = factory.LazyAttribute(lambda obj: obj.person.email())
+    first_name = factory.LazyAttribute(lambda obj: obj.person.first_name())
+    last_name = factory.LazyAttribute(lambda obj: obj.person.last_name())
+    affiliation = factory.Faker('company')
+    country_code = factory.Faker('country_code')
+    attended = False
+    checkedin = False
+
+
+class RegistrationTicketFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = RegistrationTicket
+        skip_postgeneration_save = True
+
+    registration = factory.SubFactory(RegistrationFactory)
+    attendance_type_id = 'onsite'
+    ticket_type_id = 'week_pass'
diff --git a/ietf/meeting/migrations/0011_registration_registrationticket.py b/ietf/meeting/migrations/0011_registration_registrationticket.py
new file mode 100644
index 0000000000..fc161f303d
--- /dev/null
+++ b/ietf/meeting/migrations/0011_registration_registrationticket.py
@@ -0,0 +1,90 @@
+# Generated by Django 4.2.20 on 2025-03-16 08:37
+
+from django.db import migrations, models
+import django.db.models.deletion
+import ietf.utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("name", "0017_populate_new_reg_names"),
+        ("person", "0004_alter_person_photo_alter_person_photo_thumb"),
+        ("meeting", "0010_alter_floorplan_image_alter_meetinghost_logo"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Registration",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("first_name", models.CharField(max_length=255)),
+                ("last_name", models.CharField(max_length=255)),
+                ("affiliation", models.CharField(blank=True, max_length=255)),
+                ("country_code", models.CharField(max_length=2)),
+                ("email", models.EmailField(blank=True, max_length=254, null=True)),
+                ("attended", models.BooleanField(default=False)),
+                ("checkedin", models.BooleanField(default=False)),
+                (
+                    "meeting",
+                    ietf.utils.models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="meeting.meeting",
+                    ),
+                ),
+                (
+                    "person",
+                    ietf.utils.models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="person.person",
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="RegistrationTicket",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "attendance_type",
+                    ietf.utils.models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="name.attendancetypename",
+                    ),
+                ),
+                (
+                    "registration",
+                    ietf.utils.models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="tickets",
+                        to="meeting.registration",
+                    ),
+                ),
+                (
+                    "ticket_type",
+                    ietf.utils.models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="name.registrationtickettypename",
+                    ),
+                ),
+            ],
+        ),
+    ]
diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index 84d151c310..003f8cd76e 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -1,5 +1,5 @@
-# Copyright The IETF Trust 2007-2024, All Rights Reserved
 # -*- coding: utf-8 -*-
+# Copyright The IETF Trust 2007-2024, All Rights Reserved
 
 
 # old meeting models can be found in ../proceedings/models.py
@@ -34,7 +34,7 @@
 from ietf.name.models import (
     MeetingTypeName, TimeSlotTypeName, SessionStatusName, ConstraintName, RoomResourceName,
     ImportantDateName, TimerangeName, SlideSubmissionStatusName, ProceedingsMaterialTypeName,
-    SessionPurposeName,
+    SessionPurposeName, AttendanceTypeName, RegistrationTicketTypeName
 )
 from ietf.person.models import Person
 from ietf.utils.decorators import memoize
@@ -1483,3 +1483,40 @@ class Meta:
 
     def __str__(self):
         return f'{self.person} at {self.session}'
+
+
+class RegistrationManager(models.Manager):
+    def onsite(self):
+        return self.get_queryset().filter(registrationticket__attendance_type__slug='onsite')
+
+    def remote(self):
+        return self.get_queryset().filter(registrationticket__attendance_type__slug='remote').exclude(registrationticket__attendance_type__slug='onsite')
+
+class Registration(models.Model):
+    """Registration attendee records from the IETF registration system"""
+    meeting = ForeignKey(Meeting)
+    first_name = models.CharField(max_length=255)
+    last_name = models.CharField(max_length=255)
+    affiliation = models.CharField(blank=True, max_length=255)
+    country_code = models.CharField(max_length=2)        # ISO 3166
+    person = ForeignKey(Person, blank=True, null=True, on_delete=models.PROTECT)
+    email = models.EmailField(blank=True, null=True)
+    # attended was used prior to the introduction of the ietf.meeting.Attended model and is still used by
+    # Meeting.get_attendance() for older meetings. It should not be used except for dealing with legacy data.
+    attended = models.BooleanField(default=False)
+    # checkedin indicates that the badge was picked up
+    checkedin = models.BooleanField(default=False)
+
+    # custom manager
+    objects = RegistrationManager()
+
+    def __str__(self):
+        return "{} {}".format(self.first_name, self.last_name)
+
+class RegistrationTicket(models.Model):
+    registration = ForeignKey(Registration, related_name='tickets')
+    attendance_type = ForeignKey(AttendanceTypeName, on_delete=models.PROTECT)
+    ticket_type = ForeignKey(RegistrationTicketTypeName, on_delete=models.PROTECT)
+
+    def __str__(self):
+        return "{}:{}".format(self.attendance_type, self.ticket_type)
diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py
index de9ca01476..ede2b5b993 100644
--- a/ietf/meeting/resources.py
+++ b/ietf/meeting/resources.py
@@ -14,7 +14,8 @@
 from ietf.meeting.models import ( Meeting, ResourceAssociation, Constraint, Room, Schedule, Session,
                                 TimeSlot, SchedTimeSessAssignment, SessionPresentation, FloorPlan,
                                 UrlResource, ImportantDate, SlideSubmission, SchedulingEvent,
-                                BusinessConstraint, ProceedingsMaterial, MeetingHost, Attended)
+                                BusinessConstraint, ProceedingsMaterial, MeetingHost, Attended,
+                                Registration, RegistrationTicket)
 
 from ietf.name.resources import MeetingTypeNameResource
 class MeetingResource(ModelResource):
@@ -431,3 +432,43 @@ class Meta:
             "session": ALL_WITH_RELATIONS,
         }
 api.meeting.register(AttendedResource())
+
+from ietf.meeting.resources import MeetingResource
+from ietf.person.resources import PersonResource
+class RegistrationResource(ModelResource):
+    meeting          = ToOneField(MeetingResource, 'meeting')
+    person           = ToOneField(PersonResource, 'person', null=True)
+    class Meta:
+        queryset = Registration.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'registration'
+        ordering = ['id', ]
+        filtering = { 
+            "id": ALL,
+            "first_name": ALL,
+            "last_name": ALL,
+            "affiliation": ALL,
+            "country_code": ALL,
+            "email": ALL,
+            "attended": ALL,
+            "meeting": ALL_WITH_RELATIONS,
+            "person": ALL_WITH_RELATIONS,
+        }
+api.meeting.register(RegistrationResource())
+
+class RegistrationTicketResource(ModelResource):
+    registration          = ToOneField(RegistrationResource, 'registration')
+    class Meta:
+        queryset = RegistrationTicket.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'registrationticket'
+        ordering = ['id', ]
+        filtering = { 
+            "id": ALL,
+            "ticket_type": ALL,
+            "attendance_type": ALL,
+            "registration": ALL_WITH_RELATIONS,
+        }
+api.meeting.register(RegistrationTicketResource())
diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py
index 2b7c2fca91..f1b7e8e079 100644
--- a/ietf/meeting/tasks.py
+++ b/ietf/meeting/tasks.py
@@ -9,6 +9,7 @@
 from .models import Meeting
 from .utils import generate_proceedings_content
 from .views import generate_agenda_data
+from .utils import migrate_registrations, check_migrate_registrations
 
 
 @shared_task
@@ -17,6 +18,22 @@ def agenda_data_refresh():
 
 
 @shared_task
+def migrate_registrations_task(initial=False):
+    """ Migrate ietf.stats.MeetingRegistration to ietf.meeting.Registration
+        If initial is True, migrate all meetings otherwise only future meetings.
+        This function is idempotent. It can be run regularly from cron.
+    """
+    migrate_registrations(initial=initial)
+
+
+@shared_task
+def check_migrate_registrations_task():
+    """ Compare MeetingRegistration with Registration to ensure
+        all records migrated
+    """
+    check_migrate_registrations()
+
+
 def proceedings_content_refresh_task(*, all=False):
     """Refresh meeting proceedings cache
 
diff --git a/ietf/meeting/tests_utils.py b/ietf/meeting/tests_utils.py
new file mode 100644
index 0000000000..4bda3a65fa
--- /dev/null
+++ b/ietf/meeting/tests_utils.py
@@ -0,0 +1,91 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+import datetime
+import debug  # pyflakes: ignore
+from ietf.meeting.factories import MeetingFactory # RegistrationFactory, RegistrationTicketFactory
+from ietf.meeting.models import Registration
+from ietf.meeting.utils import migrate_registrations, get_preferred
+from ietf.stats.factories import MeetingRegistrationFactory
+from ietf.utils.test_utils import TestCase
+
+
+class MigrateRegistrationsTests(TestCase):
+    def test_new_meeting_registration(self):
+        meeting = MeetingFactory(type_id='ietf', number='109')
+        reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
+        self.assertEqual(Registration.objects.count(), 0)
+        migrate_registrations(initial=True)
+        self.assertEqual(Registration.objects.count(), 1)
+        new = Registration.objects.first()
+        self.assertEqual(new.first_name, reg.first_name)
+        self.assertEqual(new.last_name, reg.last_name)
+        self.assertEqual(new.email, reg.email)
+        self.assertEqual(new.person, reg.person)
+        self.assertEqual(new.meeting, meeting)
+        self.assertEqual(new.affiliation, reg.affiliation)
+        self.assertEqual(new.country_code, reg.country_code)
+        self.assertEqual(new.checkedin, reg.checkedin)
+        self.assertEqual(new.attended, reg.attended)
+
+    def test_migrate_non_initial(self):
+        # with only old meeting
+        meeting = MeetingFactory(type_id='ietf', number='109')
+        MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
+        self.assertEqual(Registration.objects.count(), 0)
+        migrate_registrations()
+        self.assertEqual(Registration.objects.count(), 0)
+        # with new meeting
+        new_meeting = MeetingFactory(type_id='ietf', number='150')
+        new_meeting.date = datetime.date.today() + datetime.timedelta(days=30)
+        new_meeting.save()
+        MeetingRegistrationFactory(meeting=new_meeting, reg_type='onsite', ticket_type='week_pass')
+        migrate_registrations()
+        self.assertEqual(Registration.objects.count(), 1)
+
+    def test_updated_meeting_registration(self):
+        # setup test initial conditions
+        meeting = MeetingFactory(type_id='ietf', number='109')
+        reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
+        migrate_registrations(initial=True)
+        # change first_name and save
+        original = reg.first_name
+        reg.first_name = 'NewBob'
+        reg.save()
+        new = Registration.objects.first()
+        self.assertEqual(new.first_name, original)
+        migrate_registrations(initial=True)
+        new.refresh_from_db()
+        self.assertEqual(new.first_name, reg.first_name)
+
+    def test_additional_ticket(self):
+        # setup test initial conditions
+        meeting = MeetingFactory(type_id='ietf', number='109')
+        reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
+        migrate_registrations(initial=True)
+        new = Registration.objects.first()
+        self.assertEqual(new.tickets.count(), 1)
+        # add a second ticket
+        reg.reg_type = 'remote'
+        reg.pk = None
+        reg.save()
+        migrate_registrations(initial=True)
+        # new.refresh_from_db()
+        self.assertEqual(new.tickets.count(), 2)
+        
+    def test_cancelled_registration(self):
+        # setup test initial conditions
+        meeting = MeetingFactory(type_id='ietf', number='109')
+        reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
+        migrate_registrations(initial=True)
+        reg.delete()
+        # do test
+        migrate_registrations(initial=True)
+        self.assertEqual(Registration.objects.count(), 0)
+    
+    def test_get_preferred(self):
+        meeting = MeetingFactory(type_id='ietf', number='109')
+        onsite = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
+        hackathon = MeetingRegistrationFactory(meeting=meeting, reg_type='hackathon_onsite', ticket_type='week_pass')
+        result = get_preferred([onsite, hackathon])
+        self.assertEqual(result, onsite)
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index 9603278399..d35914fd60 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -26,7 +26,8 @@
 from ietf.dbtemplate.models import DBTemplate
 from ietf.doc.storage_utils import store_bytes, store_str
 from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot,
-    Constraint, SchedTimeSessAssignment, SessionPresentation, Attended)
+    Constraint, SchedTimeSessAssignment, SessionPresentation, Attended,
+    Registration, Meeting)
 from ietf.doc.models import Document, State, NewRevisionDocEvent, StateDocEvent
 from ietf.doc.models import DocEvent
 from ietf.group.models import Group
@@ -1013,6 +1014,112 @@ def participants_for_meeting(meeting):
     return (checked_in, attended)
 
 
+def get_preferred(regs):
+    """ Return a preferred regular registration (non hackathon) from 
+        a list of registrations if there is one, otherwise any.
+    """
+    for reg in regs:
+        if reg.reg_type in ['onsite', 'remote']:
+            return reg
+    return reg
+
+
+def migrate_registrations(initial=False):
+    """ Migrate ietf.stats.MeetingRegistration to ietf.meeting.Registration
+        If initial is True, migrate all meetings otherwise only future meetings.
+        This function is idempotent. It can be run regularly from cron.
+    """
+    if initial:
+        meetings = Meeting.objects.filter(type='ietf')
+        MeetingRegistration.objects.filter(reg_type='hackathon').update(reg_type='hackathon_remote')
+        MeetingRegistration.objects.filter(ticket_type='full_week_pass').update(ticket_type='week_pass')
+        MeetingRegistration.objects.filter(pk=49645).update(ticket_type='one_day')
+        MeetingRegistration.objects.filter(pk=50804).update(ticket_type='week_pass')
+        MeetingRegistration.objects.filter(pk=42386).update(ticket_type='week_pass')
+        MeetingRegistration.objects.filter(pk=42782).update(ticket_type='one_day')
+        MeetingRegistration.objects.filter(pk=43464).update(ticket_type='week_pass')
+    else:
+        # still process records during week of meeting
+        one_week_ago = datetime.date.today() - datetime.timedelta(days=7)
+        meetings = Meeting.objects.filter(type='ietf', date__gt=one_week_ago)
+
+    for meeting in meetings:
+        # gather all MeetingRegistrations by person (email)
+        emails = {}
+        for meeting_reg in MeetingRegistration.objects.filter(meeting=meeting):
+            if meeting_reg.email in emails:
+                emails[meeting_reg.email].append(meeting_reg)
+            else:
+                emails[meeting_reg.email] = [meeting_reg]
+        # process each person's registrations
+        for email, meeting_regs in emails.items():
+            preferred_reg = get_preferred(meeting_regs)
+            reg, created = Registration.objects.get_or_create(
+                meeting=meeting,
+                email=email,
+                defaults={
+                    'first_name': preferred_reg.first_name,
+                    'last_name': preferred_reg.last_name,
+                    'affiliation': preferred_reg.affiliation,
+                    'country_code': preferred_reg.country_code,
+                    'person': preferred_reg.person,
+                    'attended': preferred_reg.attended,
+                    'checkedin': preferred_reg.checkedin,
+                }
+            )
+            if created:
+                for meeting_reg in meeting_regs:
+                    reg.tickets.create(
+                        attendance_type_id=meeting_reg.reg_type or 'unknown',
+                        ticket_type_id=meeting_reg.ticket_type or 'unknown',
+                    )
+            else:
+                # check if tickets differ
+                reg_tuple_list = [(t.attendance_type_id, t.ticket_type_id) for t in reg.tickets.all()]
+                meeting_reg_tuple_list = [(mr.reg_type or 'unknown', mr.ticket_type or 'unknown') for mr in meeting_regs]
+                if not set(reg_tuple_list) == set(meeting_reg_tuple_list):
+                    # update tickets
+                    reg.tickets.all().delete()
+                    for meeting_reg in meeting_regs:
+                        reg.tickets.create(
+                            attendance_type_id=meeting_reg.reg_type or 'unknown',
+                            ticket_type_id=meeting_reg.ticket_type or 'unknown',
+                        )
+                # check fields for updates
+                fields_to_check = [
+                    'first_name', 'last_name', 'affiliation', 'country_code',
+                    'attended', 'checkedin'
+                ]
+
+                changed = False
+                for field in fields_to_check:
+                    new_value = getattr(preferred_reg, field)
+                    if getattr(reg, field) != new_value:
+                        setattr(reg, field, new_value)
+                        changed = True
+
+                if changed:
+                    reg.save()
+        # delete cancelled Registrations
+        meeting_reg_email_set = set(emails.keys())
+        reg_email_set = set(Registration.objects.filter(meeting=meeting).values_list('email', flat=True))
+        for email in reg_email_set - meeting_reg_email_set:
+            Registration.objects.filter(meeting=meeting, email=email).delete()
+
+    return
+
+
+def check_migrate_registrations():
+    """A simple utility function to test that all MeetingRegistration
+    records got migrated
+    """
+    for mr in MeetingRegistration.objects.all():
+        reg = Registration.objects.get(meeting=mr.meeting, email=mr.email)
+        assert reg.tickets.filter(
+            attendance_type__slug=mr.reg_type or 'unknown',
+            ticket_type__slug=mr.ticket_type or 'unknown').exists()
+
+
 def generate_proceedings_content(meeting, force_refresh=False):
     """Render proceedings content for a meeting and update cache
     
diff --git a/ietf/name/admin.py b/ietf/name/admin.py
index 2458da37d9..4336e0569c 100644
--- a/ietf/name/admin.py
+++ b/ietf/name/admin.py
@@ -3,6 +3,7 @@
 
 from ietf.name.models import (
     AgendaTypeName,
+    AttendanceTypeName,
     BallotPositionName,
     ConstraintName,
     ContinentName,
@@ -29,6 +30,7 @@
     LiaisonStatementTagName,
     MeetingTypeName,
     NomineePositionStateName,
+    RegistrationTicketTypeName,
     ReviewRequestStateName,
     ReviewResultName,
     ReviewTypeName,
@@ -137,6 +139,7 @@ class ProceedingsMaterialTypeNameAdmin(NameAdmin):
 admin.site.register(AgendaFilterTypeName, NameAdmin)
 admin.site.register(AgendaTypeName, NameAdmin)
 admin.site.register(AppealArtifactTypeName, NameAdmin)
+admin.site.register(AttendanceTypeName, NameAdmin)
 admin.site.register(BallotPositionName, NameAdmin)
 admin.site.register(ConstraintName, NameAdmin)
 admin.site.register(ContinentName, NameAdmin)
@@ -158,6 +161,7 @@ class ProceedingsMaterialTypeNameAdmin(NameAdmin):
 admin.site.register(LiaisonStatementTagName, NameAdmin)
 admin.site.register(MeetingTypeName, NameAdmin)
 admin.site.register(NomineePositionStateName, NameAdmin)
+admin.site.register(RegistrationTicketTypeName, NameAdmin)
 admin.site.register(ReviewRequestStateName, NameAdmin)
 admin.site.register(ReviewAssignmentStateName, NameAdmin)
 admin.site.register(ReviewResultName, NameAdmin)
diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json
index 96273fbc35..15ae71d849 100644
--- a/ietf/name/fixtures/names.json
+++ b/ietf/name/fixtures/names.json
@@ -6827,6 +6827,66 @@
     "model": "name.appealartifacttypename",
     "pk": "response"
   },
+  {
+    "fields": {
+      "desc": "",
+      "name": "ANRW Onsite",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.attendancetypename",
+    "pk": "anrw_onsite"
+  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "Hackathon Onsite",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.attendancetypename",
+    "pk": "hackathon_onsite"
+  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "Hackathon Remote",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.attendancetypename",
+    "pk": "hackathon_remote"
+  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "Onsite",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.attendancetypename",
+    "pk": "onsite"
+  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "Remote",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.attendancetypename",
+    "pk": "remote"
+  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "Unknown",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.attendancetypename",
+    "pk": "unknown"
+  },
   {
     "fields": {
       "blocking": false,
@@ -12877,6 +12937,86 @@
     "model": "name.proceedingsmaterialtypename",
     "pk": "wiki"
   },
+  {
+    "fields": {
+      "desc": "",
+      "name": "ANRW Combo",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.registrationtickettypename",
+    "pk": "anrw_combo"
+  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "ANRW Only",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.registrationtickettypename",
+    "pk": "anrw_only"
+  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "Hackathon Combo",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.registrationtickettypename",
+    "pk": "hackathon_combo"
+  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "Hackathon Only",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.registrationtickettypename",
+    "pk": "hackathon_only"
+  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "One Day",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.registrationtickettypename",
+    "pk": "one_day"
+  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "Student",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.registrationtickettypename",
+    "pk": "student"
+  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "Unknown",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.registrationtickettypename",
+    "pk": "unknown"
+  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "Week Pass",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.registrationtickettypename",
+    "pk": "week_pass"
+  },
   {
     "fields": {
       "desc": "The reviewer has accepted the assignment",
diff --git a/ietf/name/migrations/0016_attendancetypename_registrationtickettypename.py b/ietf/name/migrations/0016_attendancetypename_registrationtickettypename.py
new file mode 100644
index 0000000000..9376d3a4c6
--- /dev/null
+++ b/ietf/name/migrations/0016_attendancetypename_registrationtickettypename.py
@@ -0,0 +1,47 @@
+# Generated by Django 4.2.17 on 2025-01-02 18:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("name", "0015_last_call_name"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="AttendanceTypeName",
+            fields=[
+                (
+                    "slug",
+                    models.CharField(max_length=32, primary_key=True, serialize=False),
+                ),
+                ("name", models.CharField(max_length=255)),
+                ("desc", models.TextField(blank=True)),
+                ("used", models.BooleanField(default=True)),
+                ("order", models.IntegerField(default=0)),
+            ],
+            options={
+                "ordering": ["order", "name"],
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="RegistrationTicketTypeName",
+            fields=[
+                (
+                    "slug",
+                    models.CharField(max_length=32, primary_key=True, serialize=False),
+                ),
+                ("name", models.CharField(max_length=255)),
+                ("desc", models.TextField(blank=True)),
+                ("used", models.BooleanField(default=True)),
+                ("order", models.IntegerField(default=0)),
+            ],
+            options={
+                "ordering": ["order", "name"],
+                "abstract": False,
+            },
+        ),
+    ]
diff --git a/ietf/name/migrations/0017_populate_new_reg_names.py b/ietf/name/migrations/0017_populate_new_reg_names.py
new file mode 100644
index 0000000000..51954885c0
--- /dev/null
+++ b/ietf/name/migrations/0017_populate_new_reg_names.py
@@ -0,0 +1,39 @@
+# Generated by Django 4.2.17 on 2025-01-02 18:26
+
+from django.db import migrations
+
+def forward(apps, schema_editor):
+    AttendanceTypeName = apps.get_model('name', 'AttendanceTypeName')
+    RegistrationTicketTypeName = apps.get_model('name', 'RegistrationTicketTypeName')
+    AttendanceTypeName.objects.create(slug='onsite', name='Onsite')
+    AttendanceTypeName.objects.create(slug='remote', name='Remote')
+    AttendanceTypeName.objects.create(slug='hackathon_onsite', name='Hackathon Onsite')
+    AttendanceTypeName.objects.create(slug='hackathon_remote', name='Hackathon Remote')
+    AttendanceTypeName.objects.create(slug='anrw_onsite', name='ANRW Onsite')
+    AttendanceTypeName.objects.create(slug='unknown', name='Unknown')
+    RegistrationTicketTypeName.objects.create(slug='week_pass', name='Week Pass')
+    RegistrationTicketTypeName.objects.create(slug='one_day', name='One Day')
+    RegistrationTicketTypeName.objects.create(slug='student', name='Student')
+    RegistrationTicketTypeName.objects.create(slug='hackathon_only', name='Hackathon Only')
+    RegistrationTicketTypeName.objects.create(slug='hackathon_combo', name='Hackathon Combo')
+    RegistrationTicketTypeName.objects.create(slug='anrw_only', name='ANRW Only')
+    RegistrationTicketTypeName.objects.create(slug='anrw_combo', name='ANRW Combo')
+    RegistrationTicketTypeName.objects.create(slug='unknown', name='Unknown')
+
+
+def reverse(apps, schema_editor):
+    AttendanceTypeName = apps.get_model('name', 'AttendanceTypeName')
+    RegistrationTicketTypeName = apps.get_model('name', 'RegistrationTicketTypeName')
+    AttendanceTypeName.objects.delete()
+    RegistrationTicketTypeName.objects.delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("name", "0016_attendancetypename_registrationtickettypename"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]
diff --git a/ietf/name/models.py b/ietf/name/models.py
index 8c2520a489..0e87d43548 100644
--- a/ietf/name/models.py
+++ b/ietf/name/models.py
@@ -151,6 +151,9 @@ class SlideSubmissionStatusName(NameModel):
     "Pending, Accepted, Rejected"
 class TelechatAgendaSectionName(NameModel):
     """roll_call, minutes, action_items"""
-
 class AppealArtifactTypeName(NameModel):
     pass
+class AttendanceTypeName(NameModel):
+    """onsite, remote, hackathon_onsite, hackathon_remote"""
+class RegistrationTicketTypeName(NameModel):
+    """week, one_day, student"""
diff --git a/ietf/name/resources.py b/ietf/name/resources.py
index dffa7669db..0cb0e41e0b 100644
--- a/ietf/name/resources.py
+++ b/ietf/name/resources.py
@@ -19,7 +19,7 @@
     RoleName, RoomResourceName, SessionStatusName, StdLevelName, StreamName, TimeSlotTypeName,
     TopicAudienceName, ReviewerQueuePolicyName, TimerangeName, ExtResourceTypeName, ExtResourceName,
     SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName, TelechatAgendaSectionName,
-    AppealArtifactTypeName )
+    AppealArtifactTypeName, AttendanceTypeName, RegistrationTicketTypeName )
 
 class TimeSlotTypeNameResource(ModelResource):
     class Meta:
@@ -752,3 +752,33 @@ class Meta:
             "order": ALL,
         }
 api.name.register(AppealArtifactTypeNameResource())
+
+
+class AttendanceTypeNameResource(ModelResource):
+    class Meta:
+        cache = SimpleCache()
+        queryset = AttendanceTypeName.objects.all()
+        serializer = api.Serializer()
+        filtering = { 
+            "slug": ALL,
+            "name": ALL,
+            "desc": ALL,
+            "used": ALL,
+            "order": ALL,
+        }
+api.name.register(AttendanceTypeNameResource())
+
+
+class RegistrationTicketTypeNameResource(ModelResource):
+    class Meta:
+        cache = SimpleCache()
+        queryset = RegistrationTicketTypeName.objects.all()
+        serializer = api.Serializer()
+        filtering = { 
+            "slug": ALL,
+            "name": ALL,
+            "desc": ALL,
+            "used": ALL,
+            "order": ALL,
+        }
+api.name.register(RegistrationTicketTypeNameResource())

From d5e3840d4dbeec37415fae89ab578b1b71046b6a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 15 Apr 2025 11:17:33 -0400
Subject: [PATCH 314/656] fix: 500->400 for bad /api/v1/ requests (#8802)

* fix: 400->500 for bad /api/v1/ requests

* chore: remove unused import
---
 ietf/api/views.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/ietf/api/views.py b/ietf/api/views.py
index 4cc3560627..e5b1f96aed 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -18,7 +18,7 @@
 from django.contrib.auth.models import User
 from django.core.exceptions import ValidationError
 from django.core.validators import validate_email
-from django.http import HttpResponse, Http404, JsonResponse
+from django.http import HttpResponse, Http404, JsonResponse, HttpResponseBadRequest
 from django.shortcuts import render, get_object_or_404
 from django.urls import reverse
 from django.utils.decorators import method_decorator
@@ -68,7 +68,10 @@ def top_level(request):
         }
 
     serializer = Serializer()
-    desired_format = determine_format(request, serializer)
+    try:
+        desired_format = determine_format(request, serializer)
+    except BadRequest as err:
+        return HttpResponseBadRequest(str(err))
 
     options = {}
 
@@ -76,10 +79,12 @@ def top_level(request):
         callback = request.GET.get('callback', 'callback')
 
         if not is_valid_jsonp_callback_value(callback):
-            raise BadRequest('JSONP callback name is invalid.')
+            return HttpResponseBadRequest("JSONP callback name is invalid")
 
         options['callback'] = callback
 
+    # This might raise UnsupportedFormat, but that indicates a real server misconfiguration
+    # so let it bubble up unhandled and trigger a 500 / email to admins.
     serialized = serializer.serialize(available_resources, desired_format, options)
     return HttpResponse(content=serialized, content_type=build_content_type(desired_format))
 

From 86988eb6072b1331c393f5f7999350f45d653f04 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Tue, 15 Apr 2025 11:45:48 -0400
Subject: [PATCH 315/656] fix: remove credentials omit from agenda fetch call
 (#8779)

---
 client/agenda/store.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/agenda/store.js b/client/agenda/store.js
index 359c5fbf05..b5498303a6 100644
--- a/client/agenda/store.js
+++ b/client/agenda/store.js
@@ -141,7 +141,7 @@ export const useAgendaStore = defineStore('agenda', {
           meetingNumber = meetingData.meetingNumber
         }
 
-        const resp = await fetch(`/api/meeting/${meetingNumber}/agenda-data`, { credentials: 'omit' })
+        const resp = await fetch(`/api/meeting/${meetingNumber}/agenda-data`)
         if (!resp.ok) {
           throw new Error(resp.statusText)
         }

From be06d7c4d34a6fc18bc30d4a064dc3ffcf18c2cf Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 16 Apr 2025 12:00:04 -0500
Subject: [PATCH 316/656] chore: reorder migrations (#8804)

* chore: reorder migrations

* chore: copyright
---
 ...ationticket.py => 0012_registration_registrationticket.py} | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename ietf/meeting/migrations/{0011_registration_registrationticket.py => 0012_registration_registrationticket.py} (96%)

diff --git a/ietf/meeting/migrations/0011_registration_registrationticket.py b/ietf/meeting/migrations/0012_registration_registrationticket.py
similarity index 96%
rename from ietf/meeting/migrations/0011_registration_registrationticket.py
rename to ietf/meeting/migrations/0012_registration_registrationticket.py
index fc161f303d..c555f52e8b 100644
--- a/ietf/meeting/migrations/0011_registration_registrationticket.py
+++ b/ietf/meeting/migrations/0012_registration_registrationticket.py
@@ -1,4 +1,4 @@
-# Generated by Django 4.2.20 on 2025-03-16 08:37
+# Copyright The IETF Trust 2025, All Rights Reserved
 
 from django.db import migrations, models
 import django.db.models.deletion
@@ -10,7 +10,7 @@ class Migration(migrations.Migration):
     dependencies = [
         ("name", "0017_populate_new_reg_names"),
         ("person", "0004_alter_person_photo_alter_person_photo_thumb"),
-        ("meeting", "0010_alter_floorplan_image_alter_meetinghost_logo"),
+        ("meeting", "0011_alter_slidesubmission_doc"),
     ]
 
     operations = [

From 3d69b2c03e24d78f28a824e04f2254ff33f651d7 Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Wed, 16 Apr 2025 12:46:27 -0700
Subject: [PATCH 317/656] feat: add API for related emails (#8671)

* feat: add api for related emails. Fixes #8275

* fix: switch from querystring parameter to URL parameter

* fix: exclude null character in URL regex

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/api/tests.py | 33 +++++++++++++++++++++++++++++++++
 ietf/api/urls.py  |  2 ++
 ietf/api/views.py | 25 +++++++++++++++++++++++++
 3 files changed, 60 insertions(+)

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 23e9ff0103..809b45cc2b 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -1164,6 +1164,39 @@ def test_active_email_list(self):
         self.assertCountEqual(result.keys(), ["addresses"])
         self.assertCountEqual(result["addresses"], Email.objects.filter(active=True).values_list("address", flat=True))
 
+    @override_settings(APP_API_TOKENS={"ietf.api.views.related_email_list": ["valid-token"]})
+    def test_related_email_list(self):
+        joe = EmailFactory(address='joe@work.com')
+        EmailFactory(address='joe@home.com', person=joe.person)
+        EmailFactory(address='jòe@spain.com', person=joe.person)
+        url = urlreverse("ietf.api.views.related_email_list", kwargs={'email': 'joe@home.com'})
+        # no api key
+        r = self.client.get(url, headers={})
+        self.assertEqual(r.status_code, 403)
+        # invalid api key
+        r = self.client.get(url, headers={"X-Api-Key": "not-the-valid-token"})
+        self.assertEqual(r.status_code, 403)
+        # wrong method
+        r = self.client.post(url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 405)
+        # valid
+        r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.headers["Content-Type"], "application/json")
+        result = json.loads(r.content)
+        self.assertCountEqual(result.keys(), ["addresses"])
+        self.assertCountEqual(result["addresses"], joe.person.email_set.exclude(address='joe@home.com').values_list("address", flat=True))
+        # non-ascii
+        non_ascii_url = urlreverse("ietf.api.views.related_email_list", kwargs={'email': 'jòe@spain.com'})
+        r = self.client.get(non_ascii_url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 200)
+        result = json.loads(r.content)
+        self.assertTrue('joe@home.com' in result["addresses"])
+        # email not found
+        not_found_url = urlreverse("ietf.api.views.related_email_list", kwargs={'email': 'nobody@nowhere.com'})
+        r = self.client.get(not_found_url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 404)
+
     @override_settings(APP_API_TOKENS={"ietf.api.views.role_holder_addresses": ["valid-token"]})
     def test_role_holder_addresses(self):
         url = urlreverse("ietf.api.views.role_holder_addresses")
diff --git a/ietf/api/urls.py b/ietf/api/urls.py
index 2fcaba6094..bafd5c5b76 100644
--- a/ietf/api/urls.py
+++ b/ietf/api/urls.py
@@ -73,6 +73,8 @@
     url(r'^openid/', include('oidc_provider.urls', namespace='oidc_provider')),
     # Email alias listing
     url(r'^person/email/$', api_views.active_email_list),
+    # Related Email listing
+    url(r'^person/email/(?P<email>[^/\x00]+)/related/$', api_views.related_email_list),
     # Draft submission API
     url(r'^submit/?$', submit_views.api_submit_tombstone),
     # Draft upload API
diff --git a/ietf/api/views.py b/ietf/api/views.py
index e5b1f96aed..97b9793048 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -691,6 +691,31 @@ def active_email_list(request):
     return HttpResponse(status=405)
 
 
+@requires_api_token
+@csrf_exempt
+def related_email_list(request, email):
+    """Given an email address, returns all other email addresses known
+    to Datatracker, via Person object
+    """
+    def _http_err(code, text):
+        return HttpResponse(text, status=code, content_type="text/plain")
+
+    if request.method == "GET":
+        try:
+            email_obj = Email.objects.get(address=email)
+        except Email.DoesNotExist:
+            return _http_err(404, "Email not found")
+        person = email_obj.person
+        if not person:
+            return JsonResponse({"addresses": []})
+        return JsonResponse(
+            {
+                "addresses": list(person.email_set.exclude(address=email).values_list("address", flat=True)),
+            }
+        )
+    return HttpResponse(status=405)
+
+
 @requires_api_token
 def role_holder_addresses(request):
     if request.method == "GET":

From 4a5716e2e89044981905e9af08775ba2f5386fa7 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 18 Apr 2025 19:39:32 -0400
Subject: [PATCH 318/656] ci: Update build workflow to handle AWS breaking
 things for no reason

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 36439aa6cf..e5c627c950 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -235,7 +235,7 @@ jobs:
         docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh
         echo "Pushing statics..."
         cd static
-        aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
+        aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors --checksum-algorithm CRC32
 
     - name: Augment dockerignore for docker image build
       env:

From 24042631348135dd8b345715526815f245f9ad51 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 21 Apr 2025 13:00:32 -0300
Subject: [PATCH 319/656] refactor: smtpd->aiosmtpd (#8805)

* refactor: smtpd -> aiosmtpd

* test: set mock return value for EmailOnFailureCommandTests

The test has been working, but in a broken way, for as long as it has existed. The smtpd-based test_smtpserver was masking an exception that did not interfere with the test's effectiveness.

* test: increase SMTP.line_length_limit
---
 ietf/utils/management/tests.py |   2 +-
 ietf/utils/test_runner.py      |   2 +-
 ietf/utils/test_smtpserver.py  | 116 ++++++++++++---------------------
 requirements.txt               |   1 +
 4 files changed, 43 insertions(+), 78 deletions(-)

diff --git a/ietf/utils/management/tests.py b/ietf/utils/management/tests.py
index e94c39354f..d704999cd1 100644
--- a/ietf/utils/management/tests.py
+++ b/ietf/utils/management/tests.py
@@ -12,7 +12,7 @@
 from ietf.utils.test_utils import TestCase
 
 
-@mock.patch.object(EmailOnFailureCommand, 'handle')
+@mock.patch.object(EmailOnFailureCommand, 'handle', return_value=None)
 class EmailOnFailureCommandTests(TestCase):
     def test_calls_handle(self, handle_method):
         call_command(EmailOnFailureCommand())
diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py
index a77377ffb5..c06e7876db 100644
--- a/ietf/utils/test_runner.py
+++ b/ietf/utils/test_runner.py
@@ -863,7 +863,7 @@ def setup_test_environment(self, **kwargs):
             try:
                 # remember the value so ietf.utils.mail.send_smtp() will use the same
                 ietf.utils.mail.SMTP_ADDR['port'] = base + offset
-                self.smtpd_driver = SMTPTestServerDriver((ietf.utils.mail.SMTP_ADDR['ip4'],ietf.utils.mail.SMTP_ADDR['port']),None)
+                self.smtpd_driver = SMTPTestServerDriver(ietf.utils.mail.SMTP_ADDR['ip4'],ietf.utils.mail.SMTP_ADDR['port'], None)
                 self.smtpd_driver.start()
                 print(("     Running an SMTP test server on %(ip4)s:%(port)s to catch outgoing email." % ietf.utils.mail.SMTP_ADDR))
                 break
diff --git a/ietf/utils/test_smtpserver.py b/ietf/utils/test_smtpserver.py
index 66675aa0b1..40da758d66 100644
--- a/ietf/utils/test_smtpserver.py
+++ b/ietf/utils/test_smtpserver.py
@@ -1,92 +1,56 @@
-# Copyright The IETF Trust 2014-2020, All Rights Reserved
+# Copyright The IETF Trust 2014-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
+from aiosmtpd.controller import Controller
+from aiosmtpd.smtp import SMTP
+from email.utils import parseaddr
+from typing import Optional
 
-import smtpd
-import threading
-import asyncore
 
-import debug                            # pyflakes:ignore
+class SMTPTestHandler:
 
-class AsyncCoreLoopThread(object):
+    def __init__(self, inbox: list):
+        self.inbox = inbox
 
-    def wrap_loop(self, exit_condition, timeout=1.0, use_poll=False, map=None):
-        if map is None:
-            map = asyncore.socket_map
-            while map and not exit_condition:
-                asyncore.loop(timeout=1.0, use_poll=False, map=map, count=1)
+    async def handle_DATA(self, server, session, envelope):
+        """Handle the DATA command and 'deliver' the message"""
 
-    def start(self):
-        """Start the listening service"""
-        self.exit_condition = []
-        kwargs={'exit_condition':self.exit_condition,'timeout':1.0} 
-        self.thread = threading.Thread(target=self.wrap_loop, kwargs=kwargs)
-        self.thread.daemon = True
-        self.thread.daemon = True
-        self.thread.start()     
-
-    def stop(self):
-        """Stop the listening service"""
-        self.exit_condition.append(True)
-        self.thread.join()
-
-
-class SMTPTestChannel(smtpd.SMTPChannel):
+        self.inbox.append(envelope.content)
+        # Per RFC2033: https://datatracker.ietf.org/doc/html/rfc2033.html#section-4.2
+        #     ...after the final ".", the server returns one reply
+        #     for each previously successful RCPT command in the mail transaction,
+        #     in the order that the RCPT commands were issued.  Even if there were
+        #     multiple successful RCPT commands giving the same forward-path, there
+        #     must be one reply for each successful RCPT command.
+        return "\n".join("250 OK" for _ in envelope.rcpt_tos)
 
-#    mail_options = ['BODY=8BITMIME', 'SMTPUTF8']
-
-    def smtp_RCPT(self, arg):
-        if not self.mailfrom:
-            self.push(str('503 Error: need MAIL command'))
-            return
-        arg = self._strip_command_keyword('TO:', arg)
-        address, __ = self._getaddr(arg)
-        if not address:
-            self.push(str('501 Syntax: RCPT TO: <address>'))
-            return
+    async def handle_RCPT(self, server, session, envelope, address, rcpt_options):
+        """Handle an RCPT command and add the address to the envelope if it is acceptable"""
+        _, address = parseaddr(address)
+        if address == "":
+            return "501 Syntax: RCPT TO: <address>"
         if "poison" in address:
-            self.push(str('550 Error: Not touching that'))
-            return
-        self.rcpt_options = []
-        self.rcpttos.append(address)
-        self.push(str('250 Ok'))
-
-class SMTPTestServer(smtpd.SMTPServer):
-
-    def __init__(self,localaddr,remoteaddr,inbox):
-        if inbox is not None:
-            self.inbox=inbox
-        else:
-            self.inbox = []
-        smtpd.SMTPServer.__init__(self,localaddr,remoteaddr)
+            return "550 Error: Not touching that"
+        # At this point the address is acceptable
+        envelope.rcpt_tos.append(address)
+        return "250 OK"
 
-    def handle_accept(self):
-        pair = self.accept()
-        if pair is not None:
-            conn, addr = pair
-            #channel = SMTPTestChannel(self, conn, addr)
-            SMTPTestChannel(self, conn, addr)
 
-    def process_message(self, peer, mailfrom, rcpttos, data, mail_options=None, rcpt_options=None):
-        self.inbox.append(data)
+class SMTPTestServerDriver:
 
-
-class SMTPTestServerDriver(object):
-    def __init__(self, localaddr, remoteaddr, inbox=None):
-        self.localaddr=localaddr
-        self.remoteaddr=remoteaddr
-        if inbox is not None:
-            self.inbox = inbox
-        else:
-            self.inbox = []
-        self.thread_driver = None
+    def __init__(self, address: str, port: int, inbox: Optional[list] = None):
+        # Allow longer lines than the 1001 that RFC 5321 requires. As of 2025-04-16 the
+        # datatracker emits some non-compliant messages.
+        # See https://aiosmtpd.aio-libs.org/en/latest/smtp.html
+        SMTP.line_length_limit = 4000  # tests start failing between 3000 and 4000
+        self.controller = Controller(
+            hostname=address,
+            port=port,
+            handler=SMTPTestHandler(inbox=[] if inbox is None else inbox),
+        )
 
     def start(self):
-        self.smtpserver = SMTPTestServer(self.localaddr,self.remoteaddr,self.inbox)
-        self.thread_driver = AsyncCoreLoopThread()
-        self.thread_driver.start()
+        self.controller.start()
 
     def stop(self):
-        if self.thread_driver:
-            self.thread_driver.stop()
-
+        self.controller.stop()
diff --git a/requirements.txt b/requirements.txt
index cd93f448e2..b00a21ab49 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
 # -*- conf-mode -*-
 setuptools>=51.1.0    # Require this first, to prevent later errors
 #
+aiosmtpd>=1.4.6
 argon2-cffi>=21.3.0    # For the Argon2 password hasher option
 beautifulsoup4>=4.11.1    # Only used in tests
 bibtexparser>=1.2.0    # Only used in tests

From 2652c96c8db968a021f6f95953d8ab2337aa167e Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Mon, 21 Apr 2025 16:13:32 +0000
Subject: [PATCH 320/656] ci: update base image target version to 20250421T1600

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index cd0a70667c..bdf3cff0e4 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250402T1611
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250421T1600
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 1195fc9a0b..bdfdb2eed1 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250402T1611
+20250421T1600

From 35db33961d91ada14c5744af3cf8c160e02c5614 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 22 Apr 2025 12:31:07 -0500
Subject: [PATCH 321/656] fix: lengthen liaison response_contact field (#8823)

---
 ...lter_liaisonstatement_response_contacts.py | 20 +++++++++++++++++++
 ietf/liaisons/models.py                       |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)
 create mode 100644 ietf/liaisons/migrations/0002_alter_liaisonstatement_response_contacts.py

diff --git a/ietf/liaisons/migrations/0002_alter_liaisonstatement_response_contacts.py b/ietf/liaisons/migrations/0002_alter_liaisonstatement_response_contacts.py
new file mode 100644
index 0000000000..ac0a11101b
--- /dev/null
+++ b/ietf/liaisons/migrations/0002_alter_liaisonstatement_response_contacts.py
@@ -0,0 +1,20 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("liaisons", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="liaisonstatement",
+            name="response_contacts",
+            field=models.TextField(
+                blank=True, help_text="Where to send a response", max_length=1024
+            ),
+        ),
+    ]
diff --git a/ietf/liaisons/models.py b/ietf/liaisons/models.py
index f357d6cf8c..2ad502102c 100644
--- a/ietf/liaisons/models.py
+++ b/ietf/liaisons/models.py
@@ -33,7 +33,7 @@ class LiaisonStatement(models.Model):
     to_groups = models.ManyToManyField(Group, blank=True, related_name='liaisonstatement_to_set')
     to_contacts = models.CharField(max_length=2000, help_text="Contacts at recipient group")
 
-    response_contacts = models.CharField(blank=True, max_length=255, help_text="Where to send a response") # RFC4053
+    response_contacts = models.TextField(blank=True, max_length=1024, help_text="Where to send a response") # RFC4053
     technical_contacts = models.CharField(blank=True, max_length=255, help_text="Who to contact for clarification") # RFC4053
     action_holder_contacts = models.CharField(blank=True, max_length=255, help_text="Who makes sure action is completed")  # incoming only?
     cc_contacts = models.TextField(blank=True)

From 7be503cf1c6c02ed7715f6b72417cbc760369175 Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Wed, 23 Apr 2025 08:33:13 -0700
Subject: [PATCH 322/656] feat: use new mail archive search API for review app
 lookups (#8788)

* feat: use new mail archive search API for review app lookups

* fix: fix patch call

* fix: remove url from query_data

* fix: force utc timezone on datetime from archive API

* fix: remove query_data['url'] from test
---
 ietf/doc/tests_review.py                      | 220 ++++++++++--------
 ietf/doc/views_review.py                      |  22 +-
 ietf/review/mailarch.py                       | 126 ++++------
 ietf/settings.py                              |   2 +
 .../templates/doc/review/complete_review.html |   4 +-
 5 files changed, 176 insertions(+), 198 deletions(-)

diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py
index 13ddbc22ba..9850beca75 100644
--- a/ietf/doc/tests_review.py
+++ b/ietf/doc/tests_review.py
@@ -3,12 +3,12 @@
 
 
 from pathlib import Path
-import datetime, os, shutil
+import datetime
 import io
-import tarfile, tempfile, mailbox
-import email.mime.multipart, email.mime.text, email.utils
+import os
+import shutil
 
-from mock import patch
+from mock import patch, Mock
 from requests import Response
 
 from django.apps import apps
@@ -650,112 +650,132 @@ def test_accept_reviewer_assignment_after_reject(self):
         assignment = reload_db_objects(assignment)
         self.assertEqual(assignment.state_id, "accepted")
 
-    def make_test_mbox_tarball(self, review_req):
-        mbox_path = os.path.join(self.review_dir, "testmbox.tar.gz")
-        with tarfile.open(mbox_path, "w:gz") as tar:
-            with tempfile.NamedTemporaryFile(dir=self.review_dir, suffix=".mbox") as tmp:
-                mbox = mailbox.mbox(tmp.name)
+    @patch('ietf.review.mailarch.requests.post')
+    def test_retrieve_messages(self, mock_post):
+        mock_data = {
+            "results": [
+                {
+                    "from": "Alice <alice@example.com>",
+                    "subject": "Hello",
+                    "content": "Hi\n This is a really good document.\n",
+                    "message_id": "abc123",
+                    "url": "https://example.com/message",
+                    "date": "2025-04-07T12:00:00",
+                }
+            ]
+        }
+        mock_post.return_value.json.return_value = mock_data
 
-                # plain text
-                msg = email.mime.text.MIMEText("Hello,\n\nI have reviewed the document and did not find any problems.\n\nJohn Doe")
-                msg["From"] = "John Doe <johndoe@example.com>"
-                msg["To"] = review_req.team.list_email
-                msg["Subject"] = "Review of {}-01".format(review_req.doc.name)
-                msg["Message-ID"] = email.utils.make_msgid()
-                msg["Archived-At"] = "<https://www.example.com/testmessage>"
-                msg["Date"] = email.utils.formatdate()
-
-                mbox.add(msg)
-
-                # plain text + HTML
-                msg = email.mime.multipart.MIMEMultipart('alternative')
-                msg["From"] = "John Doe II <johndoe2@example.com>"
-                msg["To"] = review_req.team.list_email
-                msg["Subject"] = "Review of {}".format(review_req.doc.name)
-                msg["Message-ID"] = email.utils.make_msgid()
-                msg["Archived-At"] = "<https://www.example.com/testmessage2>"
-
-                msg.attach(email.mime.text.MIMEText("Hi!,\r\nLooks OK!\r\n-John", "plain"))
-                msg.attach(email.mime.text.MIMEText("<html><body><p>Hi!,</p><p>Looks OK!</p><p>-John</p></body></html>", "html"))
-                mbox.add(msg)
-
-                tmp.flush()
-
-                tar.add(os.path.relpath(tmp.name))
+        doc = WgDraftFactory(group__acronym='mars', rev='01')
+        review_team = ReviewTeamFactory(acronym="reviewteam", name="Review Team", type_id="review", list_email="reviewteam@ietf.org", parent=Group.objects.get(acronym="farfut"))
+        rev_role = RoleFactory(group=review_team, person__user__username='reviewer', person__user__email='reviewer@example.com', name_id='reviewer')
+        RoleFactory(group=review_team, person__user__username='reviewsecretary', person__user__email='reviewsecretary@example.com', name_id='secr')
+        review_req = ReviewRequestFactory(doc=doc, team=review_team, type_id='early', state_id='assigned', requested_by=rev_role.person, deadline=timezone.now() + datetime.timedelta(days=20))
+        ReviewAssignmentFactory(review_request=review_req, reviewer=rev_role.person.email_set.first(), state_id='accepted')
 
-                mbox.close()
+        query_data = ietf.review.mailarch.construct_query_data(doc, review_team, query=None)
+        response = ietf.review.mailarch.retrieve_messages(query_data)
 
-        return mbox_path
+        self.assertEqual(len(response), 1)
+        self.assertEqual(response[0]['from'], 'Alice <alice@example.com>')
+        self.assertEqual(response[0]['splitfrom'], ('Alice', 'alice@example.com'))
+        self.assertEqual(response[0]['subject'], 'Hello')
+        self.assertEqual(response[0]['content'], 'Hi\n This is a really good document.')
+        self.assertEqual(response[0]['message_id'], 'abc123')
+        self.assertEqual(response[0]['url'], 'https://example.com/message')
+        self.assertEqual(response[0]['utcdate'], ('2025-04-07', '12:00:00'))
 
-    def test_search_mail_archive(self):
-        doc = WgDraftFactory(group__acronym='mars',rev='01')
+    def test_construct_query_data(self):
+        doc = WgDraftFactory(group__acronym='mars', rev='01')
         review_team = ReviewTeamFactory(acronym="reviewteam", name="Review Team", type_id="review", list_email="reviewteam@ietf.org", parent=Group.objects.get(acronym="farfut"))
-        rev_role = RoleFactory(group=review_team,person__user__username='reviewer',person__user__email='reviewer@example.com',name_id='reviewer')
-        RoleFactory(group=review_team,person__user__username='reviewsecretary',person__user__email='reviewsecretary@example.com',name_id='secr')
-        review_req = ReviewRequestFactory(doc=doc,team=review_team,type_id='early',state_id='assigned',requested_by=rev_role.person,deadline=timezone.now()+datetime.timedelta(days=20))
+        data = ietf.review.mailarch.construct_query_data(doc, review_team, query=None)
+        self.assertEqual(data['start_date'], (date_today() - datetime.timedelta(days=180)).isoformat())
+        self.assertEqual(data['email_list'], 'reviewteam')
+        self.assertEqual(data['query_value'], doc.name)
+        self.assertEqual(data['query'], f'subject:({doc.name})')
+        self.assertEqual(data['limit'], '30')
+
+    @patch('ietf.doc.views_review.requests.post')
+    def test_search_mail_archive(self, mock_post):
+        doc = WgDraftFactory(group__acronym='mars', rev='01')
+        review_team = ReviewTeamFactory(acronym="reviewteam", name="Review Team", type_id="review", list_email="reviewteam@ietf.org", parent=Group.objects.get(acronym="farfut"))
+        rev_role = RoleFactory(group=review_team, person__user__username='reviewer', person__user__email='reviewer@example.com', name_id='reviewer')
+        RoleFactory(group=review_team, person__user__username='reviewsecretary', person__user__email='reviewsecretary@example.com', name_id='secr')
+        review_req = ReviewRequestFactory(doc=doc, team=review_team, type_id='early', state_id='assigned', requested_by=rev_role.person, deadline=timezone.now() + datetime.timedelta(days=20))
         assignment = ReviewAssignmentFactory(review_request=review_req, reviewer=rev_role.person.email_set.first(), state_id='accepted')
 
+        mock_data = {
+            "results": [
+                {
+                    "from": "Alice <alice@example.com>",
+                    "subject": "Review of {}-01".format(review_req.doc.name),
+                    "content": "Hi\n This is a really good document.\n",
+                    "message_id": "abc123",
+                    "url": "https://example.com/message",
+                    "date": "2025-04-07T12:00:00",
+                },
+                {
+                    "from": "Joe <joe@example.com>",
+                    "subject": "Review of {}".format(review_req.doc.name),
+                    "content": "Hi\n I believe this is the best document.\n",
+                    "message_id": "abc456",
+                    "url": "https://example.com/message",
+                    "date": "2025-04-07T12:00:00",
+                }
+            ]
+        }
+        response1 = Mock()
+        response1.json.return_value = mock_data
+
+        response2 = Mock()
+        response2.json.return_value = mock_data
+
+        response3 = Mock()
+        response3.json.return_value = {"results": []}
+
+        mock_post.side_effect = [response1, response2, response3]
+
         # test URL construction
-        query_urls = ietf.review.mailarch.construct_query_urls(doc, review_team)
-        self.assertTrue(review_req.doc.name in query_urls["query_data_url"])
+        query_data = ietf.review.mailarch.construct_query_data(doc, review_team)
+        self.assertTrue(review_req.doc.name in query_data["query_value"])
 
-        # test parsing
-        mbox_path = self.make_test_mbox_tarball(review_req)
+        url = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={"name": doc.name, "assignment_id": assignment.pk})
+        url2 = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={"name": doc.name, "acronym": review_team.acronym})
+        login_testing_unauthorized(self, "reviewsecretary", url)
 
-        try:
-            # mock URL generator and point it to local file - for this
-            # to work, the module (and not the function) must be
-            # imported in the view
-            real_fn = ietf.review.mailarch.construct_query_urls
-            ietf.review.mailarch.construct_query_urls = lambda doc, team, query=None: { "query_data_url": "file://" + os.path.abspath(mbox_path) }
-            url = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={ "name": doc.name, "assignment_id": assignment.pk })
-            url2 = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={ "name": doc.name, "acronym": review_team.acronym })
-            login_testing_unauthorized(self, "reviewsecretary", url)
-
-            r = self.client.get(url)
-            self.assertEqual(r.status_code, 200)
-            messages = r.json()["messages"]
-            self.assertEqual(len(messages), 2)
-
-            r = self.client.get(url2)
-            self.assertEqual(r.status_code, 200)
-            messages = r.json()["messages"]
-            self.assertEqual(len(messages), 2)
-
-            today = date_today(datetime.timezone.utc)
-
-            self.assertEqual(messages[0]["url"], "https://www.example.com/testmessage")
-            self.assertTrue("John Doe" in messages[0]["content"])
-            self.assertEqual(messages[0]["subject"], "Review of {}-01".format(review_req.doc.name))
-            self.assertEqual(messages[0]["revision_guess"], "01")
-            self.assertEqual(messages[0]["splitfrom"], ["John Doe", "johndoe@example.com"])
-            self.assertEqual(messages[0]["utcdate"][0], today.isoformat())
-
-            self.assertEqual(messages[1]["url"], "https://www.example.com/testmessage2")
-            self.assertTrue("Looks OK" in messages[1]["content"])
-            self.assertTrue("<html>" not in messages[1]["content"])
-            self.assertEqual(messages[1]["subject"], "Review of {}".format(review_req.doc.name))
-            self.assertFalse('revision_guess' in messages[1])
-            self.assertEqual(messages[1]["splitfrom"], ["John Doe II", "johndoe2@example.com"])
-            self.assertEqual(messages[1]["utcdate"][0], "")
-
-
-            # Test failure to return mailarch results
-            no_result_path = os.path.join(self.review_dir, "mailarch_no_result.html")
-            with io.open(no_result_path, "w") as f:
-                f.write('Content-Type: text/html\n\n<html><body><div class="xtr"><div class="xtd no-results">No results found</div></div>')
-            ietf.review.mailarch.construct_query_urls = lambda doc, team, query=None: { "query_data_url": "file://" + os.path.abspath(no_result_path) }
-
-            url = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={ "name": doc.name, "assignment_id": assignment.pk })
-
-            r = self.client.get(url)
-            self.assertEqual(r.status_code, 200)
-            result = r.json()
-            self.assertNotIn('messages', result)
-            self.assertIn('No results found', result['error'])
-
-        finally:
-            ietf.review.mailarch.construct_query_urls = real_fn
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        messages = r.json()["messages"]
+        self.assertEqual(len(messages), 2)
+
+        r = self.client.get(url2)
+        self.assertEqual(r.status_code, 200)
+        messages = r.json()["messages"]
+        self.assertEqual(len(messages), 2)
+
+        self.assertEqual(messages[0]["url"], "https://example.com/message")
+        self.assertTrue("Hi" in messages[0]["content"])
+        self.assertEqual(messages[0]["subject"], "Review of {}-01".format(review_req.doc.name))
+        self.assertEqual(messages[0]["revision_guess"], "01")
+        self.assertEqual(messages[0]["splitfrom"], ["Alice", "alice@example.com"])
+        self.assertEqual(messages[0]["utcdate"], ['2025-04-07', '12:00:00'])
+
+        self.assertEqual(messages[1]["url"], "https://example.com/message")
+        self.assertTrue("Hi" in messages[1]["content"])
+        self.assertTrue("<html>" not in messages[1]["content"])
+        self.assertEqual(messages[1]["subject"], "Review of {}".format(review_req.doc.name))
+        self.assertFalse('revision_guess' in messages[1])
+        self.assertEqual(messages[1]["splitfrom"], ["Joe", "joe@example.com"])
+        self.assertEqual(messages[1]["utcdate"], ['2025-04-07', '12:00:00'])
+
+        # Test failure to return mailarch results
+        url = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={"name": doc.name, "assignment_id": assignment.pk})
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        result = r.json()
+        self.assertNotIn('messages', result)
+        self.assertIn('No results found', result['error'])
 
     def test_submit_unsolicited_review_choose_team(self):
         doc = WgDraftFactory(group__acronym='mars', rev='01')
diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py
index 1f23c435fa..6ac0e17791 100644
--- a/ietf/doc/views_review.py
+++ b/ietf/doc/views_review.py
@@ -958,14 +958,14 @@ def complete_review(request, name, assignment_id=None, acronym=None):
 
         form = CompleteReviewForm(assignment, doc, team, is_reviewer, initial=initial)
 
-    mail_archive_query_urls = mailarch.construct_query_urls(doc, team)
+    mail_archive_query_data = mailarch.construct_query_data(doc, team)
 
     return render(request, 'doc/review/complete_review.html', {
         'doc': doc,
         'team': team,
         'assignment': assignment,
         'form': form,
-        'mail_archive_query_urls': mail_archive_query_urls,
+        'mail_archive_query_data': mail_archive_query_data,
         'revising_review': revising_review,
         'review_to': to,
         'review_cc': cc,
@@ -987,27 +987,25 @@ def search_mail_archive(request, name, acronym=None, assignment_id=None):
     if not (is_reviewer or can_manage_request):
         permission_denied(request, "You do not have permission to perform this action")
 
-    res = mailarch.construct_query_urls(doc, team, query=request.GET.get("query"))
-    if not res:
-        return JsonResponse({ "error": "Couldn't do lookup in mail archive - don't know where to look"})
-
-    MAX_RESULTS = 30
+    query_data = mailarch.construct_query_data(doc, team, query=request.GET.get("query"))
+    if not query_data:
+        return JsonResponse({"error": "Couldn't do lookup in mail archive - don't know where to look"})
 
     try:
-        res["messages"] = mailarch.retrieve_messages(res["query_data_url"])[:MAX_RESULTS]
-        for message in res["messages"]:
+        query_data["messages"] = mailarch.retrieve_messages(query_data)
+        for message in query_data["messages"]:
             try:
                 revision_guess = message["subject"].split(name)[1].split('-')[1]
                 message["revision_guess"] = revision_guess if revision_guess.isnumeric() else None
             except IndexError:
                 pass
     except KeyError as e:
-        res["error"] = "No results found (%s)" % str(e)
+        query_data["error"] = "No results found (%s)" % str(e)
     except Exception as e:
-        res["error"] = "Retrieval from mail archive failed: %s" % str(e)
+        query_data["error"] = "Retrieval from mail archive failed: %s" % str(e)
         # raise # useful when debugging
 
-    return JsonResponse(res)
+    return JsonResponse(query_data)
 
 class EditReviewRequestCommentForm(forms.ModelForm):
     comment = forms.CharField(widget=forms.Textarea, strip=False)
diff --git a/ietf/review/mailarch.py b/ietf/review/mailarch.py
index c34a6079ce..61abc83aa5 100644
--- a/ietf/review/mailarch.py
+++ b/ietf/review/mailarch.py
@@ -6,25 +6,18 @@
 # mailarchive.ietf.org
 
 import base64
-import contextlib
 import datetime
 import email.utils
 import hashlib
-import mailbox
-import tarfile
-import tempfile
-
-from urllib.parse import urlencode
-from urllib.request import urlopen
+import requests
 
 import debug                            # pyflakes:ignore
 
-from pyquery import PyQuery
 
 from django.conf import settings
 from django.utils.encoding import force_bytes, force_str
 
-from ietf.utils.mail import get_payload_text
+from ietf.utils.log import log
 from ietf.utils.timezone import date_today
 
 
@@ -43,7 +36,7 @@ def hash_list_message_id(list_name, msgid):
     sha.update(force_bytes(list_name))
     return force_str(base64.urlsafe_b64encode(sha.digest()).rstrip(b"="))
 
-def construct_query_urls(doc, team, query=None):
+def construct_query_data(doc, team, query=None):
     list_name = list_name_from_email(team.list_email)
     if not list_name:
         return None
@@ -51,83 +44,48 @@ def construct_query_urls(doc, team, query=None):
     if not query:
         query = doc.name
 
-    encoded_query = "?" + urlencode({
-        "qdr": "c", # custom time frame
-        "start_date": (date_today() - datetime.timedelta(days=180)).isoformat(),
-        "email_list": list_name,
-        "q": "subject:({})".format(query),
-        "as": "1", # this is an advanced search
-    })
-
-    return {
-        "query": query,
-        "query_url": settings.MAILING_LIST_ARCHIVE_URL + "/arch/search/" + encoded_query,
-        "query_data_url": settings.MAILING_LIST_ARCHIVE_URL + "/arch/export/mbox/" + encoded_query,
+    query_data = {
+        'start_date': (date_today() - datetime.timedelta(days=180)).isoformat(),
+        'email_list': list_name,
+        'query_value': query,
+        'query': f'subject:({query})',
+        'limit': '30',
     }
+    return query_data
 
 def construct_message_url(list_name, msgid):
     return "{}/arch/msg/{}/{}".format(settings.MAILING_LIST_ARCHIVE_URL, list_name, hash_list_message_id(list_name, msgid))
 
-def retrieve_messages_from_mbox(mbox_fileobj):
-    """Return selected content in message from mbox from mailarch."""
-    res = []
-    with tempfile.NamedTemporaryFile(suffix=".mbox") as mbox_file:
-        # mailbox.mbox needs a path, so we need to put the contents
-        # into a file
-        mbox_data = mbox_fileobj.read()
-        mbox_file.write(mbox_data)
-        mbox_file.flush()
-
-        mbox = mailbox.mbox(mbox_file.name, create=False)
-        for msg in mbox:
-            content = ""
-
-            for part in msg.walk():
-                if part.get_content_type() == "text/plain":
-                    charset = part.get_content_charset() or "utf-8"
-                    content += get_payload_text(part, default_charset=charset)
-
-            # parse a couple of things for the front end
-            utcdate = None
-            d = email.utils.parsedate_tz(msg["Date"])
-            if d:
-                utcdate = datetime.datetime.fromtimestamp(email.utils.mktime_tz(d), datetime.timezone.utc)
-
-            res.append({
-                "from": msg["From"],
-                "splitfrom": email.utils.parseaddr(msg["From"]),
-                "subject": msg["Subject"],
-                "content": content.replace("\r\n", "\n").replace("\r", "\n").strip("\n"),
-                "message_id": email.utils.unquote(msg["Message-ID"].strip()),
-                "url": email.utils.unquote(msg["Archived-At"].strip()),
-                "date": msg["Date"],
-                "utcdate": (utcdate.date().isoformat(), utcdate.time().isoformat()) if utcdate else ("", ""),
-            })
-        mbox.close()
-
-    return res
-
-def retrieve_messages(query_data_url):
+def retrieve_messages(query_data):
     """Retrieve and return selected content from mailarch."""
-    res = []
-
-    # This has not been rewritten to use requests.get() because get() does
-    # not handle file URLs out of the box, which we need for tesing
-    with contextlib.closing(urlopen(query_data_url, timeout=15)) as fileobj:
-        content_type = fileobj.info()["Content-type"]
-        if not content_type.startswith("application/x-tar"):
-            if content_type.startswith("text/html"):
-                r = fileobj.read(20000)
-                q = PyQuery(r)
-                div = q('div[class~="no-results"]')
-                if div:
-                    raise KeyError("No results: %s -> %s" % (query_data_url, div.text(), ))
-            raise Exception("Export failed - this usually means no matches were found")
-
-        with tarfile.open(fileobj=fileobj, mode='r|*') as tar:
-            for entry in tar:
-                if entry.isfile():
-                    mbox_fileobj = tar.extractfile(entry)
-                    res.extend(retrieve_messages_from_mbox(mbox_fileobj))
-
-    return res
+
+    headers = {'X-Api-Key': settings.MAILING_LIST_ARCHIVE_API_KEY}
+    try:
+        response = requests.post(
+            settings.MAILING_LIST_ARCHIVE_SEARCH_URL,
+            headers=headers,
+            json=query_data,
+            timeout=settings.DEFAULT_REQUESTS_TIMEOUT)
+    except requests.Timeout as exc:
+        log(f'POST request failed for [{query_data["url"]}]: {exc}')
+        raise RuntimeError(f'Timeout retrieving [{query_data["url"]}]') from exc
+
+    results = []
+    jresponse = response.json()
+    if 'results' not in jresponse or len(jresponse['results']) == 0:
+        raise KeyError(f'No results: {query_data["query"]}')
+    for msg in jresponse['results']:
+        # datetime is already UTC
+        dt = datetime.datetime.fromisoformat(msg['date'])
+        dt_utc = dt.replace(tzinfo=datetime.timezone.utc)
+        results.append({
+            "from": msg["from"],
+            "splitfrom": email.utils.parseaddr(msg["from"]),
+            "subject": msg["subject"],
+            "content": msg["content"].replace("\r\n", "\n").replace("\r", "\n").strip("\n"),
+            "message_id": msg["message_id"],
+            "url": msg["url"],
+            "utcdate": (dt_utc.date().isoformat(), dt_utc.time().isoformat()),
+        })
+
+    return results
diff --git a/ietf/settings.py b/ietf/settings.py
index 33a2f976d9..8005aca8cc 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -808,6 +808,8 @@ def skip_unreadable_post(record):
 # Mailing list info URL for lists hosted on the IETF servers
 MAILING_LIST_INFO_URL = "https://mailman3.%(domain)s/mailman3/lists/%(list_addr)s.%(domain)s"
 MAILING_LIST_ARCHIVE_URL = "https://mailarchive.ietf.org"
+MAILING_LIST_ARCHIVE_SEARCH_URL = "https://mailarchive.ietf.org/api/v1/message/search/"
+MAILING_LIST_ARCHIVE_API_KEY = "changeme"
 
 # Liaison Statement Tool settings (one is used in DOC_HREFS below)
 LIAISON_UNIVERSAL_FROM = 'Liaison Statement Management Tool <statements@' + IETF_DOMAIN + '>'
diff --git a/ietf/templates/doc/review/complete_review.html b/ietf/templates/doc/review/complete_review.html
index 091f30532e..8aeb1486a4 100644
--- a/ietf/templates/doc/review/complete_review.html
+++ b/ietf/templates/doc/review/complete_review.html
@@ -93,7 +93,7 @@ <h1>
             <a class="btn btn-secondary float-end" href="{{ doc.get_absolute_url }}">Back</a>
         {% endif %}
         <div class="template d-none">
-            {% if mail_archive_query_urls %}
+            {% if mail_archive_query_data %}
                 <div class="mail-archive-search"
                     {% if assignment %}
                      data-search-mail-archive-url="{% url "ietf.doc.views_review.search_mail_archive" name=doc.name assignment_id=assignment.pk %}"
@@ -106,7 +106,7 @@ <h1>
                         <div class="input-group mb-3">
                             <input type="text" id="mail-archive-subjects"
                                    class="query-input form-control"
-                                   value="{{ mail_archive_query_urls.query }}">
+                                   value="{{ mail_archive_query_data.query_value }}">
                             <button type="button" class="search btn btn-primary">Search</button>
                         </div>
                         <div class="retrieving d-none my-3">

From f4209ea5a3061639f2adb9245271bb52f44a2140 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 23 Apr 2025 13:21:23 -0300
Subject: [PATCH 323/656] chore: k8s plumbing for mailarchive secret (#8827)

* chore: k8s plumbing for mailarchive secret

plus a typo fix

* style: remove empty line

* chore: hints for working with APP_API_TOKENS value
---
 k8s/settings_local.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index a0e197b7f5..0e128b1b4c 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -135,6 +135,12 @@ def _multiline_to_list(s):
     queue=os.environ.get("RABBITMQ_QUEUE", "dt"),
 )
 
+# mailarchive API key
+_mailing_list_archive_api_key = os.environ.get("DATATRACKER_MAILING_LIST_ARCHIVE_API_KEY", None)
+if _mailing_list_archive_api_key is None:
+    raise RuntimeError("DATATRACKER_MAILING_LIST_ARCHIVE_API_KEY must be set")
+MAILING_LIST_ARCHIVE_API_KEY = _mailing_list_archive_api_key
+
 IANA_SYNC_USERNAME = "ietfsync"
 IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes"
 IANA_SYNC_PROTOCOLS_URL = "http://www.iana.org/protocols/"
@@ -173,7 +179,13 @@ def _multiline_to_list(s):
         "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set"
     )
 
-# For APP_API_TOKENS, ccept either base64-encoded JSON or raw JSON, but not both
+# For APP_API_TOKENS, accept either base64-encoded JSON or raw JSON, but not both.
+# To decode / pretty-print the encoded form, run:
+#    base64 -d | jq .
+# paste the encoded secret into stdin. Copy/paste that into an editor you trust not
+# to leave a copy lying around. When done editing, copy/paste the final JSON through
+#    jq -c | base64
+# and copy/paste the output into the secret store. 
 if "DATATRACKER_APP_API_TOKENS_JSON_B64" in os.environ:
     if "DATATRACKER_APP_API_TOKENS_JSON" in os.environ:
         raise RuntimeError(

From 1bede712301206b808d2650e726b82d864e74d5d Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Wed, 23 Apr 2025 09:26:34 -0700
Subject: [PATCH 324/656] refactor: update secretariat dashboard style (#8140)

* refactor: update secretariat dashboard style

* refactor: change announcement styling

* feat: hide to_custom field when not needed

* fix: apply Black to python files

* fix: add announcement.js to package.json

* fix: move announcement playwright test to tests-legacy

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/secr/announcement/forms.py               | 137 ++++++++++++------
 ietf/secr/announcement/tests.py               | 109 +++++++-------
 ietf/secr/announcement/urls.py                |   5 +-
 ietf/secr/announcement/views.py               |  95 ++++++------
 ietf/secr/templates/announcement/confirm.html |  33 ++---
 ietf/secr/templates/announcement/index.html   |  31 ++++
 ietf/secr/templates/announcement/main.html    |  36 -----
 ietf/secr/templates/index.html                |  33 +++++
 ietf/secr/templates/main.html                 |  69 ---------
 ietf/secr/urls.py                             |   2 +-
 ietf/static/js/announcement.js                |  57 ++++++++
 package.json                                  |   1 +
 playwright/.gitignore                         |   1 +
 .../tests-legacy/secr/announcement.spec.js    |  77 ++++++++++
 14 files changed, 413 insertions(+), 273 deletions(-)
 create mode 100644 ietf/secr/templates/announcement/index.html
 delete mode 100644 ietf/secr/templates/announcement/main.html
 create mode 100644 ietf/secr/templates/index.html
 delete mode 100644 ietf/secr/templates/main.html
 create mode 100644 ietf/static/js/announcement.js
 create mode 100644 playwright/tests-legacy/secr/announcement.spec.js

diff --git a/ietf/secr/announcement/forms.py b/ietf/secr/announcement/forms.py
index 3fe58bdaaa..820ef79a09 100644
--- a/ietf/secr/announcement/forms.py
+++ b/ietf/secr/announcement/forms.py
@@ -14,30 +14,41 @@
 # Globals
 # ---------------------------------------------
 
-TO_LIST = ('IETF Announcement List <ietf-announce@ietf.org>',
-           'I-D Announcement List <i-d-announce@ietf.org>',
-           'RFP Announcement List <rfp-announce@ietf.org>',
-           'The IESG <iesg@ietf.org>',
-           'Working Group Chairs <wgchairs@ietf.org>',
-           'BOF Chairs <bofchairs@ietf.org>',
-           'Other...')
+TO_LIST = (
+    "IETF Announcement List <ietf-announce@ietf.org>",
+    "I-D Announcement List <i-d-announce@ietf.org>",
+    "RFP Announcement List <rfp-announce@ietf.org>",
+    "The IESG <iesg@ietf.org>",
+    "Working Group Chairs <wgchairs@ietf.org>",
+    "BOF Chairs <bofchairs@ietf.org>",
+    "Other...",
+)
 
 # ---------------------------------------------
 # Helper Functions
 # ---------------------------------------------
 
+
 def get_from_choices(user):
-    '''
+    """
     This function returns a choices tuple containing
     all the Announced From choices.  Including
     leadership chairs and other entities.
-    '''
+    """
     addresses = []
-    if has_role(user,'Secretariat'):
-        addresses = AnnouncementFrom.objects.values_list('address', flat=True).order_by('address').distinct()
+    if has_role(user, "Secretariat"):
+        addresses = (
+            AnnouncementFrom.objects.values_list("address", flat=True)
+            .order_by("address")
+            .distinct()
+        )
     else:
         for role in user.person.role_set.all():
-            addresses.extend(AnnouncementFrom.objects.filter(name=role.name, group=role.group).values_list('address', flat=True).order_by('address'))
+            addresses.extend(
+                AnnouncementFrom.objects.filter(name=role.name, group=role.group)
+                .values_list("address", flat=True)
+                .order_by("address")
+            )
 
     nomcom_choices = get_nomcom_choices(user)
     if nomcom_choices:
@@ -45,65 +56,91 @@ def get_from_choices(user):
 
     choices = list(zip(addresses, addresses))
     if len(choices) > 1:
-        choices.insert(0, ('', '(Choose an option)'))
+        choices.insert(0, ("", "(Choose an option)"))
     return choices
 
 
 def get_nomcom_choices(user):
-    '''
+    """
     Returns the list of nomcom email addresses for given user
-    '''
-    nomcoms = Role.objects.filter(name="chair",
-                                  group__acronym__startswith="nomcom",
-                                  group__state="active",
-                                  group__type="nomcom",
-                                  person=user.person)
+    """
+    nomcoms = Role.objects.filter(
+        name="chair",
+        group__acronym__startswith="nomcom",
+        group__state="active",
+        group__type="nomcom",
+        person=user.person,
+    )
     addresses = []
     for nomcom in nomcoms:
         year = nomcom.group.acronym[-4:]
-        addresses.append('NomCom Chair %s <nomcom-chair-%s@ietf.org>' % (year,year))
+        addresses.append("NomCom Chair %s <nomcom-chair-%s@ietf.org>" % (year, year))
 
     return addresses
-        
+
 
 def get_to_choices():
-    return list(zip(TO_LIST,TO_LIST))
+    return list(zip(TO_LIST, TO_LIST))
 
 
 # ---------------------------------------------
 # Forms
 # ---------------------------------------------
 
+
 class AnnounceForm(forms.ModelForm):
-    nomcom = forms.ModelChoiceField(queryset=Group.objects.filter(acronym__startswith='nomcom',type='nomcom',state='active'),required=False)
+    nomcom = forms.ModelChoiceField(
+        queryset=Group.objects.filter(
+            acronym__startswith="nomcom", type="nomcom", state="active"
+        ),
+        required=False,
+    )
     to_custom = MultiEmailField(required=False)
 
     class Meta:
         model = Message
-        fields = ('nomcom', 'to','to_custom','frm','cc','bcc','reply_to','subject','body')
+        fields = (
+            "nomcom",
+            "to",
+            "to_custom",
+            "frm",
+            "cc",
+            "bcc",
+            "reply_to",
+            "subject",
+            "body",
+        )
+        labels = {"frm": "From"}
+        help_texts = {
+            "to": "Select name OR select Other... and enter email below",
+            "cc": "Use comma separated lists for emails (Cc, Bcc, Reply To)",
+        }
 
     def __init__(self, *args, **kwargs):
-        if 'hidden' in kwargs:
-            self.hidden = kwargs.pop('hidden')
+        if "hidden" in kwargs:
+            self.hidden = kwargs.pop("hidden")
         else:
             self.hidden = False
-        user = kwargs.pop('user')
+        user = kwargs.pop("user")
         person = user.person
         super(AnnounceForm, self).__init__(*args, **kwargs)
-        self.fields['to'].widget = forms.Select(choices=get_to_choices())
-        self.fields['to'].help_text = 'Select name OR select Other... and enter email below'
-        self.fields['cc'].help_text = 'Use comma separated lists for emails (Cc, Bcc, Reply To)'
-        self.fields['frm'].widget = forms.Select(choices=get_from_choices(user))
-        self.fields['frm'].label = 'From'
-        self.fields['reply_to'].required = True
-        self.fields['nomcom'].label = 'NomCom message:'
-        nomcom_roles = person.role_set.filter(group__in=self.fields['nomcom'].queryset,name='chair')
-        secr_roles = person.role_set.filter(group__acronym='secretariat',name='secr')
+        self.fields["to"].widget = forms.Select(choices=get_to_choices())
+        self.fields["frm"].widget = forms.Select(choices=get_from_choices(user))
+        self.fields["reply_to"].required = True
+        # nomcom field is defined declaratively so label and help_text must be set here
+        self.fields["nomcom"].label = "NomCom message:"
+        self.fields["nomcom"].help_text = (
+            "If this is a NomCom announcement specifiy which NomCom group here"
+        )
+        nomcom_roles = person.role_set.filter(
+            group__in=self.fields["nomcom"].queryset, name="chair"
+        )
+        secr_roles = person.role_set.filter(group__acronym="secretariat", name="secr")
         if nomcom_roles:
-            self.initial['nomcom'] = nomcom_roles[0].group.pk
+            self.initial["nomcom"] = nomcom_roles[0].group.pk
         if not nomcom_roles and not secr_roles:
-            self.fields['nomcom'].widget = forms.HiddenInput()
-        
+            self.fields["nomcom"].widget = forms.HiddenInput()
+
         if self.hidden:
             for key in list(self.fields.keys()):
                 self.fields[key].widget = forms.HiddenInput()
@@ -113,25 +150,29 @@ def clean(self):
         data = self.cleaned_data
         if self.errors:
             return self.cleaned_data
-        if data['to'] == 'Other...' and not data['to_custom']:
+        if data["to"] == "Other..." and not data["to_custom"]:
             raise forms.ValidationError('You must enter a "To" email address')
-        for k in ['to', 'frm', 'cc',]:
+        for k in [
+            "to",
+            "frm",
+            "cc",
+        ]:
             data[k] = unescape(data[k])
 
         return data
 
     def save(self, *args, **kwargs):
-        user = kwargs.pop('user')
+        user = kwargs.pop("user")
         message = super(AnnounceForm, self).save(commit=False)
         message.by = user.person
-        if self.cleaned_data['to'] == 'Other...':
-            message.to = self.cleaned_data['to_custom']
-        if kwargs['commit']:
+        if self.cleaned_data["to"] == "Other...":
+            message.to = self.cleaned_data["to_custom"]
+        if kwargs["commit"]:
             message.save()
 
         # handle nomcom message
-        nomcom = self.cleaned_data.get('nomcom',False)
+        nomcom = self.cleaned_data.get("nomcom", False)
         if nomcom:
             message.related_groups.add(nomcom)
 
-        return message
\ No newline at end of file
+        return message
diff --git a/ietf/secr/announcement/tests.py b/ietf/secr/announcement/tests.py
index c147c301b6..f08e824397 100644
--- a/ietf/secr/announcement/tests.py
+++ b/ietf/secr/announcement/tests.py
@@ -6,7 +6,7 @@
 
 from django.urls import reverse
 
-import debug                            # pyflakes:ignore
+import debug  # pyflakes:ignore
 
 from ietf.utils.test_utils import TestCase
 from ietf.group.factories import RoleFactory
@@ -17,97 +17,102 @@
 from ietf.message.models import AnnouncementFrom
 from ietf.utils.mail import outbox, empty_outbox
 
-SECR_USER='secretary'
-WG_USER=''
-AD_USER=''
+SECR_USER = "secretary"
+WG_USER = ""
+AD_USER = ""
+
 
 class SecrAnnouncementTestCase(TestCase):
     def setUp(self):
         super().setUp()
-        chair = RoleName.objects.get(slug='chair')
-        secr = RoleName.objects.get(slug='secr')
-        ietf = Group.objects.get(acronym='ietf')
-        iab = Group.objects.get(acronym='iab')
-        secretariat = Group.objects.get(acronym='secretariat')
-        AnnouncementFrom.objects.create(name=secr,group=secretariat,address='IETF Secretariat <ietf-secretariat@ietf.org>')
-        AnnouncementFrom.objects.create(name=chair,group=ietf,address='IETF Chair <chair@ietf.org>')
-        AnnouncementFrom.objects.create(name=chair,group=iab,address='IAB Chair <iab-chair@iab.org>')
+        chair = RoleName.objects.get(slug="chair")
+        secr = RoleName.objects.get(slug="secr")
+        ietf = Group.objects.get(acronym="ietf")
+        iab = Group.objects.get(acronym="iab")
+        secretariat = Group.objects.get(acronym="secretariat")
+        AnnouncementFrom.objects.create(
+            name=secr,
+            group=secretariat,
+            address="IETF Secretariat <ietf-secretariat@ietf.org>",
+        )
+        AnnouncementFrom.objects.create(
+            name=chair, group=ietf, address="IETF Chair <chair@ietf.org>"
+        )
+        AnnouncementFrom.objects.create(
+            name=chair, group=iab, address="IAB Chair <iab-chair@iab.org>"
+        )
 
     def test_main(self):
         "Main Test"
-        url = reverse('ietf.secr.announcement.views.main')
+        url = reverse("ietf.secr.announcement.views.main")
         self.client.login(username="secretary", password="secretary+password")
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-    
+
     def test_main_announce_from(self):
-        url = reverse('ietf.secr.announcement.views.main')
+        url = reverse("ietf.secr.announcement.views.main")
 
         # Secretariat
         self.client.login(username="secretary", password="secretary+password")
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('#id_frm option')),4)
+        self.assertEqual(len(q("#id_frm option")), 4)
 
         # IAB Chair
         self.client.login(username="iab-chair", password="iab-chair+password")
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('#id_frm option')),1)
-        self.assertTrue('<iab-chair@iab.org>' in q('#id_frm option').val())
+        self.assertEqual(len(q("#id_frm option")), 1)
+        self.assertTrue("<iab-chair@iab.org>" in q("#id_frm option").val())
 
         # IETF Chair
         self.client.login(username="ietf-chair", password="ietf-chair+password")
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('#id_frm option')),1)
-        self.assertTrue('<chair@ietf.org>' in q('#id_frm option').val())
+        self.assertEqual(len(q("#id_frm option")), 1)
+        self.assertTrue("<chair@ietf.org>" in q("#id_frm option").val())
+
 
 class UnauthorizedAnnouncementCase(TestCase):
     def test_unauthorized(self):
         "Unauthorized Test"
-        url = reverse('ietf.secr.announcement.views.main')
-        person = RoleFactory(name_id='chair',group__acronym='mars').person
-        self.client.login(username=person.user.username, password=person.user.username+"+password")
+        url = reverse("ietf.secr.announcement.views.main")
+        person = RoleFactory(name_id="chair", group__acronym="mars").person
+        self.client.login(
+            username=person.user.username, password=person.user.username + "+password"
+        )
         r = self.client.get(url)
         self.assertEqual(r.status_code, 403)
-    
+
+
 class SubmitAnnouncementCase(TestCase):
-    def test_invalid_submit(self):
-        "Invalid Submit"
-        url = reverse('ietf.secr.announcement.views.main')
-        post_data = {'id_subject':''}
-        self.client.login(username="secretary", password="secretary+password")
-        r = self.client.post(url,post_data)
-        self.assertEqual(r.status_code, 200)
-        q = PyQuery(r.content)
-        self.assertTrue(len(q('form ul.errorlist')) > 0)
-        
     def test_valid_submit(self):
         "Valid Submit"
         nomcom_test_data()
         empty_outbox()
-        url = reverse('ietf.secr.announcement.views.main')
-        confirm_url = reverse('ietf.secr.announcement.views.confirm')
-        nomcom = Group.objects.get(type='nomcom')
-        post_data = {'nomcom': nomcom.pk,
-                     'to':'Other...',
-                     'to_custom':'rcross@amsl.com',
-                     'frm':'IETF Secretariat &lt;ietf-secretariat@ietf.org&gt;',
-                     'reply_to':'secretariat@ietf.org',
-                     'subject':'Test Subject',
-                     'body':'This is a test.'}
+        url = reverse("ietf.secr.announcement.views.main")
+        confirm_url = reverse("ietf.secr.announcement.views.confirm")
+        nomcom = Group.objects.get(type="nomcom")
+        post_data = {
+            "nomcom": nomcom.pk,
+            "to": "Other...",
+            "to_custom": "phil@example.com",
+            "frm": "IETF Secretariat &lt;ietf-secretariat@ietf.org&gt;",
+            "reply_to": "secretariat@ietf.org",
+            "subject": "Test Subject",
+            "body": "This is a test.",
+        }
         self.client.login(username="secretary", password="secretary+password")
-        response = self.client.post(url,post_data)
-        self.assertContains(response, 'Confirm Announcement')
-        response = self.client.post(confirm_url,post_data,follow=True)
+        response = self.client.post(url, post_data)
+        self.assertContains(response, "Confirm Announcement")
+        response = self.client.post(confirm_url, post_data, follow=True)
         self.assertRedirects(response, url)
-        self.assertEqual(len(outbox),1)
-        self.assertEqual(outbox[0]['subject'],'Test Subject')
-        self.assertEqual(outbox[0]['to'],'<rcross@amsl.com>')
-        message = Message.objects.filter(by__user__username='secretary').last()
-        self.assertEqual(message.subject,'Test Subject')
+        self.assertEqual(len(outbox), 1)
+        self.assertEqual(outbox[0]["subject"], "Test Subject")
+        self.assertEqual(outbox[0]["to"], "<phil@example.com>")
+        message = Message.objects.filter(by__user__username="secretary").last()
+        self.assertEqual(message.subject, "Test Subject")
         self.assertTrue(nomcom in message.related_groups.all())
diff --git a/ietf/secr/announcement/urls.py b/ietf/secr/announcement/urls.py
index 3c3c05a09c..dc534f64ae 100644
--- a/ietf/secr/announcement/urls.py
+++ b/ietf/secr/announcement/urls.py
@@ -1,8 +1,7 @@
-
 from ietf.secr.announcement import views
 from ietf.utils.urls import url
 
 urlpatterns = [
-    url(r'^$', views.main),
-    url(r'^confirm/$', views.confirm),
+    url(r"^$", views.main),
+    url(r"^confirm/$", views.confirm),
 ]
diff --git a/ietf/secr/announcement/views.py b/ietf/secr/announcement/views.py
index 42de089c59..5617ae9e6f 100644
--- a/ietf/secr/announcement/views.py
+++ b/ietf/secr/announcement/views.py
@@ -18,86 +18,93 @@
 # Helper Functions
 # -------------------------------------------------
 def check_access(user):
-    '''
+    """
     This function takes a Django User object and returns true if the user has access to
     the Announcement app.
-    '''
+    """
     if hasattr(user, "person"):
         person = user.person
         if has_role(user, "Secretariat"):
             return True
-        
+
         for role in person.role_set.all():
-            if AnnouncementFrom.objects.filter(name=role.name,group=role.group):
+            if AnnouncementFrom.objects.filter(name=role.name, group=role.group):
                 return True
 
-        if Role.objects.filter(name="chair",
-                               group__acronym__startswith="nomcom",
-                               group__state="active",
-                               group__type="nomcom",
-                               person=person):
+        if Role.objects.filter(
+            name="chair",
+            group__acronym__startswith="nomcom",
+            group__state="active",
+            group__type="nomcom",
+            person=person,
+        ):
             return True
 
     return False
 
+
 # --------------------------------------------------
 # STANDARD VIEW FUNCTIONS
 # --------------------------------------------------
 # this seems to cause some kind of circular problem
 # @check_for_cancel(reverse('home'))
 @login_required
-@check_for_cancel('../')
+@check_for_cancel("../")
 def main(request):
-    '''
+    """
     Main view for Announcement tool.  Authrozied users can fill out email details: header, body, etc
     and send.
-    '''
+    """
     if not check_access(request.user):
-        permission_denied(request, 'Restricted to: Secretariat, IAD, or chair of IETF, IAB, RSOC, RSE, IAOC, ISOC, NomCom.')
+        permission_denied(
+            request,
+            "Restricted to: Secretariat, IAD, or chair of IETF, IAB, RSOC, RSE, IAOC, ISOC, NomCom.",
+        )
 
-    form = AnnounceForm(request.POST or None,user=request.user)
+    form = AnnounceForm(request.POST or None, user=request.user)
 
     if form.is_valid():
         # recast as hidden form for next page of process
         form = AnnounceForm(request.POST, user=request.user, hidden=True)
-        if form.data['to'] == 'Other...':
-            to = form.data['to_custom']
+        if form.data["to"] == "Other...":
+            to = form.data["to_custom"]
         else:
-            to = form.data['to']
+            to = form.data["to"]
 
-        return render(request, 'announcement/confirm.html', {
-            'message': form.data,
-            'to': to,
-            'form': form},
+        return render(
+            request,
+            "announcement/confirm.html",
+            {"message": form.data, "to": to, "form": form},
         )
 
-    return render(request, 'announcement/main.html', { 'form': form} )
+    return render(request, "announcement/index.html", {"form": form})
+
 
 @login_required
-@check_for_cancel('../')
+@check_for_cancel("../")
 def confirm(request):
 
     if not check_access(request.user):
-        permission_denied(request, 'Restricted to: Secretariat, IAD, or chair of IETF, IAB, RSOC, RSE, IAOC, ISOC, NomCom.')
+        permission_denied(
+            request,
+            "Restricted to: Secretariat, IAD, or chair of IETF, IAB, RSOC, RSE, IAOC, ISOC, NomCom.",
+        )
 
-    if request.method == 'POST':
+    if request.method == "POST":
         form = AnnounceForm(request.POST, user=request.user)
-        if request.method == 'POST':
-            message = form.save(user=request.user,commit=True)
-            extra = {'Reply-To': message.get('reply_to') }
-            send_mail_text(None,
-                           message.to,
-                           message.frm,
-                           message.subject,
-                           message.body,
-                           cc=message.cc,
-                           bcc=message.bcc,
-                           extra=extra,
-                       )
-
-            messages.success(request, 'The announcement was sent.')
-            return redirect('ietf.secr.announcement.views.main')
-
-
-
-
+        if request.method == "POST":
+            message = form.save(user=request.user, commit=True)
+            extra = {"Reply-To": message.get("reply_to")}
+            send_mail_text(
+                None,
+                message.to,
+                message.frm,
+                message.subject,
+                message.body,
+                cc=message.cc,
+                bcc=message.bcc,
+                extra=extra,
+            )
+
+            messages.success(request, "The announcement was sent.")
+            return redirect("ietf.secr.announcement.views.main")
diff --git a/ietf/secr/templates/announcement/confirm.html b/ietf/secr/templates/announcement/confirm.html
index ddf2a6de6e..0e1f72c54b 100644
--- a/ietf/secr/templates/announcement/confirm.html
+++ b/ietf/secr/templates/announcement/confirm.html
@@ -1,22 +1,17 @@
-{% extends "base_site.html" %}
+{# Copyright The IETF Trust 2024, All Rights Reserved #}
+{% extends "base.html" %}
 {% load static %}
-
+{% load ietf_filters %}
+{% load django_bootstrap5 %}
 {% block title %}Announcement{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-    <script src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }}
-    &raquo; Announcement
-{% endblock %}
-
 {% block content %}
+    <h1>Announcement</h1>
+
 
     <div class="module">
         <h2>Confirm Announcement</h2>
 
-        <form action="{% url "ietf.secr.announcement.views.confirm" %}" method="post">{% csrf_token %}
+        <form action="{% url 'ietf.secr.announcement.views.confirm' %}" method="post">{% csrf_token %}
 
             <pre id="announce-confirm">
 To: {{ to }}
@@ -29,15 +24,13 @@ <h2>Confirm Announcement</h2>
 {{ message.body }}
             </pre>
 
-            {{ form }}
-            <div class="button-group">
-                <ul id="announcement-button-list">
-                    <li><button type="submit" name="submit" value="Send">Send</button></li>
-                    <li><button type="button" onclick="history.go(-1);return false">Back</button></li>
-                    <li><button type="submit" name="submit" value="Cancel">Cancel</button></li>
-                </ul>
-            </div> <!-- button-group -->
+            {% bootstrap_form form %}
 
+            <div class="d-flex justify-content-around">
+                <button class="btn btn-primary" type="submit" name="submit" value="Send">Send</button>
+                <button class="btn btn-secondary" type="button" onclick="history.go(-1);return false">Back</button>
+                <button class="btn btn-danger" type="submit" name="submit" value="Cancel">Cancel</button>
+            </div>
         </form>
     </div> <!-- module -->
 
diff --git a/ietf/secr/templates/announcement/index.html b/ietf/secr/templates/announcement/index.html
new file mode 100644
index 0000000000..ad7226e3bc
--- /dev/null
+++ b/ietf/secr/templates/announcement/index.html
@@ -0,0 +1,31 @@
+{# Copyright The IETF Trust 2024, All Rights Reserved #}
+{% extends "base.html" %}
+{% load static %}
+{% load ietf_filters %}
+{% load django_bootstrap5 %}
+{% block title %}Announcement{% endblock %}
+{% block content %}
+    <h1 class="announcement">Announcement</h1>
+    {% if form.non_field_errors %}<div class="my-3 alert alert-danger">{{ form.non_field_errors }}</div>{% endif %}
+
+    <form id="announcement-form" method="post">
+        {% csrf_token %}
+        {% bootstrap_field form.nomcom layout='horizontal' %}
+        {% bootstrap_field form.to layout='horizontal' %}
+        {% bootstrap_field form.to_custom layout='horizontal' %}
+        {% bootstrap_field form.frm layout='horizontal' %}
+        {% bootstrap_field form.cc layout='horizontal' %}
+        {% bootstrap_field form.bcc layout='horizontal' %}
+        {% bootstrap_field form.reply_to layout='horizontal' %}
+        {% bootstrap_field form.subject layout='horizontal' %}
+        {% bootstrap_field form.body layout='horizontal' %}
+
+        <button type="submit" class="btn btn-primary">Continue</button>
+        <a class="btn btn-secondary float-end"
+               href="{% url 'ietf.secr' %}">Cancel</a>
+    </form>
+
+{% endblock %}
+{% block js %}
+    <script src="{% static 'ietf/js/announcement.js' %}"></script>
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/secr/templates/announcement/main.html b/ietf/secr/templates/announcement/main.html
deleted file mode 100644
index c88b4a2406..0000000000
--- a/ietf/secr/templates/announcement/main.html
+++ /dev/null
@@ -1,36 +0,0 @@
-{% extends "base_site.html" %}
-
-{% block title %}Announcement{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }}
-    &raquo; Announcement
-{% endblock %}
-
-{% block content %}
-
-    <div class="module">
-        <h2>Announcement</h2>
-
-        <form method="post">{% csrf_token %}
-            <table class="new-style full-width amstable" id="announce-table">
-                <tbody>
-                {% if form.non_field_errors %}{{ form.non_field_errors }}{% endif %}
-                {% for field in form.visible_fields %}
-                    <tr>
-                        <th scope="row">{{ field.label_tag }}{% if field.field.required %}<span class="required"> *</span>{% endif %}</th>
-                        <td>{{ field.errors }}{{ field }}{% if field.help_text %}<br>{{ field.help_text }}{% endif %}</td>
-                    </tr>
-                {% endfor %}
-                </tbody>
-            </table>
-            <div class="button-group">
-                <ul id="announcement-button-list">
-                    <li><button type="submit" name="submit" value="Continue">Continue</button></li>
-                    <li><button type="button" onclick="window.location='../'" value="Cancel">Cancel</button></li>
-                </ul>
-            </div> <!-- button-group -->
-
-        </form>
-    </div> <!-- module -->
-
-{% endblock %}
diff --git a/ietf/secr/templates/index.html b/ietf/secr/templates/index.html
new file mode 100644
index 0000000000..05fa3db41f
--- /dev/null
+++ b/ietf/secr/templates/index.html
@@ -0,0 +1,33 @@
+{# Copyright The IETF Trust 2007, All Rights Reserved #}
+{% extends "base.html" %}
+{% load static %}
+{% load ietf_filters %}
+{% block title %}Secretariat Dashboard{% endblock %}
+{% block content %}
+    <h1>Secretariat Dashboard</h1>
+    <div class="">
+        {% if user|has_role:"Secretariat" %}
+            <h2>IESG</h2>
+            <ul>
+                <li><a href="{% url 'ietf.secr.announcement.views.main' %}"><b>Announcement</b></a></li>
+                <li><a href="{% url 'ietf.secr.telechat.views.main' %}"><b>Telechat</b></a></li>
+            </ul>
+
+            <h2>IDs and WGs Process</h2>
+            <ul>
+                <li> <a href="{% url 'ietf.secr.rolodex.views.search' %}"><b>Rolodex</b></a></li>
+            </ul>
+
+            <h2>Meetings and Proceedings</h2>
+            <ul>
+                <li> <a href="{% url 'ietf.secr.sreq.views.main' %} "><b>Session Requests</b></a></li>
+                <li> <a href="{% url 'ietf.secr.meetings.views.main' %}"><b>Meeting Manager</b></a></li>
+            </ul>
+        {% else %}
+            <ul>
+                <li> <a href="{% url 'ietf.secr.sreq.views.main' %}"><b>Session Requests</b></a></li>
+                <li> <a href="{% url 'ietf.secr.announcement.views.main' %}"><b>Announcements</b></a></li>
+            </ul>
+        {% endif %}
+    </div>
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/secr/templates/main.html b/ietf/secr/templates/main.html
deleted file mode 100644
index 42d6e8f6a1..0000000000
--- a/ietf/secr/templates/main.html
+++ /dev/null
@@ -1,69 +0,0 @@
-{% extends "base_site.html" %}
-{% load ietf_filters %}
-
-{% block content %}
-    <div id="content-main">
-
-        {% if user|has_role:"Secretariat" %}
-
-            <table class="menu">
-                <tbody>
-                <tr>
-                    <td>
-                        <h2>IESG</h2>
-                        <ul>
-                        <li> <a href="{% url 'ietf.secr.announcement.views.main' %}"><b>Announcement</b></a></li>
-                        <li> <a href="{% url 'ietf.secr.telechat.views.main' %}"><b>Telechat</b></a></li>
-                        </ul>
-                    </td>
-                    <td>
-                        <h2>IDs and WGs Process</h2>
-                        <ul>
-                        <li> <a href="{% url 'ietf.secr.rolodex.views.search' %}"><b>Rolodex</b></a></li>
-                        </ul>
-                    </td>
-                </tr>
-                <tr>
-                    <td>
-                        <h2>Meetings and Proceedings</h2>
-                        <ul>
-                        <li> <a href="{% url "ietf.secr.sreq.views.main" %}"><b>Session Requests</b></a></li>
-                        <li> <a href="{% url "ietf.secr.meetings.views.main" %}"><b>Meeting Manager</b></a></li>
-                        </ul>
-                    </td>
-                    <td></td>
-                </tr>
-                </tbody>
-            </table>
-
-        {% else %}
-
-            <table class="menu">
-                <tbody>
-                <tr>
-                    <td>
-                        <h2>Section 1</h2>
-                        <ul>
-                        <li> <a href="{% url "ietf.secr.sreq.views.main" %}"><b>Session Requests</b></a></li>
-                        <li> <a href="{% url 'ietf.secr.announcement.views.main' %}"><b>Announcements</b></a></li>
-                        </ul>
-                    </td>
-                    <td>
-                        <h2>Section 2</h2>
-                    </td>
-                </tr>
-                <tr>
-                    <td>
-                        <h2>Section 3</h2>
-                    </td>
-                    <td>
-                        <h2>Section 4</h2>
-                    </td>
-                </tr>
-                </tbody>
-            </table>
-
-        {% endif %}
-
-    </div>
-{% endblock %}
\ No newline at end of file
diff --git a/ietf/secr/urls.py b/ietf/secr/urls.py
index 0ce14a449a..4a3e5b0363 100644
--- a/ietf/secr/urls.py
+++ b/ietf/secr/urls.py
@@ -2,7 +2,7 @@
 from django.views.generic import TemplateView
 
 urlpatterns = [
-    re_path(r'^$', TemplateView.as_view(template_name='main.html')),
+    re_path(r'^$', TemplateView.as_view(template_name='index.html'), name='ietf.secr'),
     re_path(r'^announcement/', include('ietf.secr.announcement.urls')),
     re_path(r'^meetings/', include('ietf.secr.meetings.urls')),
     re_path(r'^rolodex/', include('ietf.secr.rolodex.urls')),
diff --git a/ietf/static/js/announcement.js b/ietf/static/js/announcement.js
new file mode 100644
index 0000000000..95465120fa
--- /dev/null
+++ b/ietf/static/js/announcement.js
@@ -0,0 +1,57 @@
+const announcementApp = (function() {
+    'use strict';
+    return {
+        // functions for Announcement
+        checkToField: function() {
+            document.documentElement.scrollTop = 0; // For most browsers
+            const toField = document.getElementById('id_to');
+            const toCustomInput = document.getElementById('id_to_custom');
+            const toCustomDiv = toCustomInput.closest('div.row');
+            
+            if (toField.value === 'Other...') {
+                toCustomDiv.style.display = 'flex'; // Show the custom field
+            } else {
+                toCustomDiv.style.display = 'none'; // Hide the custom field
+                toCustomInput.value = ''; // Optionally clear the input value if hidden
+            }
+        }
+    };
+})();
+
+// Extra care is required to ensure the back button 
+// works properly for the optional to_custom field.
+// Take the case when a user selects "Other..." for
+// "To" field. The "To custom" field appears and they
+// enter a new address there.
+// In Chrome, when the form is submitted and then the user
+// uses the back button (or browser back), the page loads
+// from bfcache then the javascript DOMContentLoaded event
+// handler is run, hiding the empty to_custom field, THEN the
+// browser autofills the form fields. Because to_submit
+// is now hidden it does not get a value. This is a very
+// bad experience for the user because the to_custom field
+// was unexpectedly cleared and hidden. If they notice this
+// they would need to know to first select another "To"
+// option, then select "Other..." again just to get the
+// to_custom field visible so they can re-enter the custom
+// address.
+// The solution is to use setTimeout to run checkToField
+// after a short delay, giving the browser time to autofill
+// the form fields before it checks to see if the to_custom
+// field is empty and hides it.
+
+document.addEventListener('DOMContentLoaded', function() {
+    // Run the visibility check after allowing cache to populate values
+    setTimeout(announcementApp.checkToField, 300);
+
+    const toField = document.getElementById('id_to');
+    toField.addEventListener('change', announcementApp.checkToField);
+});
+
+// Handle back/forward navigation with pageshow
+window.addEventListener('pageshow', function(event) {
+    if (event.persisted) {
+        // Then apply visibility logic after cache restoration
+        setTimeout(announcementApp.checkToField, 300);
+    }
+});
\ No newline at end of file
diff --git a/package.json b/package.json
index 6d1591ca15..e3e89288e7 100644
--- a/package.json
+++ b/package.json
@@ -114,6 +114,7 @@
         "ietf/static/js/add_session_recordings.js",
         "ietf/static/js/agenda_filter.js",
         "ietf/static/js/agenda_materials.js",
+        "ietf/static/js/announcement.js",
         "ietf/static/js/complete-review.js",
         "ietf/static/js/create_timeslot.js",
         "ietf/static/js/create_timeslot.js",
diff --git a/playwright/.gitignore b/playwright/.gitignore
index 75e854d8dc..f38d036a79 100644
--- a/playwright/.gitignore
+++ b/playwright/.gitignore
@@ -2,3 +2,4 @@ node_modules/
 /test-results/
 /playwright-report/
 /playwright/.cache/
+auth.json
\ No newline at end of file
diff --git a/playwright/tests-legacy/secr/announcement.spec.js b/playwright/tests-legacy/secr/announcement.spec.js
new file mode 100644
index 0000000000..4dbbc25a81
--- /dev/null
+++ b/playwright/tests-legacy/secr/announcement.spec.js
@@ -0,0 +1,77 @@
+const { test, expect } = require('@playwright/test')
+const viewports = require('../../helpers/viewports')
+const { setTimeout } = require('timers/promises')
+
+// ====================================================================
+// ANNOUNCEMENT | DESKTOP viewport
+// ====================================================================
+
+test.describe('desktop', () => {
+  
+  test.beforeAll(async ({ browser }) => {
+    const context = await browser.newContext();
+    const page = await context.newPage();
+
+    await page.goto('/accounts/login/');
+
+    await page.fill('input#id_username', 'glen');
+    await page.fill('input#id_password', 'password');
+
+    await page.click('button[type="submit"]');
+    await page.waitForURL('/accounts/profile/');
+
+    await context.storageState({ path: 'auth.json' });
+
+    await context.close();
+  });
+
+  test.beforeEach(async ({ browser }) => {
+    // Reuse the authentication state in each test
+    const context = await browser.newContext({ storageState: 'auth.json' });
+    const page = await context.newPage();
+    await page.setViewportSize({
+      width: viewports.desktop[0],
+      height: viewports.desktop[1]
+    })
+    await page.goto(`/secr/announcement/`); 
+    await page.locator('h1:text("Announcement")').waitFor({ state: 'visible' })
+    await setTimeout(500)
+    // Attach the page to the test context
+    test.info().page = page;
+  })
+
+  test('show to custom', async () => {
+    const page = test.info().page;
+
+    // to_custom should initially be hidden
+    const element = page.locator('#id_to_custom');
+    await expect(element).toBeHidden();
+    await page.selectOption('select#id_to', 'Other...');
+    await expect(element).toBeVisible();
+  })
+
+  test('back button', async () => {
+    const page = test.info().page;
+
+    const element = page.locator('#id_to_custom');
+    await page.selectOption('select#id_to', 'Other...');
+    await expect(element).toBeVisible();
+    await page.fill('input#id_to_custom', 'custom@example.com');
+    await page.selectOption('select#id_frm', 'IETF Chair <chair@ietf.org>');
+    await page.fill('input#id_reply_to', 'greg@example.com');
+    await page.fill('input#id_subject', 'About Stuff');
+    await page.fill('textarea#id_body', 'This is the stuff');
+
+    await page.click('text="Continue"');
+    const h2Locator = page.locator('h2:text("Confirm Announcement")');
+    await h2Locator.waitFor({ state: 'visible' });
+
+    // click back button and check to_custom
+    await page.click('text="Back"');
+    const subjectLocator = page.locator('input#id_subject');
+    await subjectLocator.waitFor({ state: 'visible' });
+    await expect(element).toBeVisible();
+    await expect(element).toHaveValue('custom@example.com');
+  })
+
+})
\ No newline at end of file

From 1a309b7464161e5e7e590eecacef96d752af02a0 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 23 Apr 2025 14:45:45 -0400
Subject: [PATCH 325/656] ci: Revert build.yml s3 sync command change

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e5c627c950..36439aa6cf 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -235,7 +235,7 @@ jobs:
         docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh
         echo "Pushing statics..."
         cd static
-        aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors --checksum-algorithm CRC32
+        aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
 
     - name: Augment dockerignore for docker image build
       env:

From 3c8f9727acc4825ec33816d77275b2793e9a9713 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 24 Apr 2025 11:43:23 -0300
Subject: [PATCH 326/656] chore: get django out of the forwarded header
 business (#8634)

---
 k8s/settings_local.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 0e128b1b4c..0f4ecaa631 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -19,13 +19,6 @@ def _multiline_to_list(s):
 # Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env!
 SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development")
 
-# Use X-Forwarded-Proto to determine request.is_secure(). This relies on CloudFlare overwriting the
-# value of the header if an incoming request sets it, which it does:
-# https://developers.cloudflare.com/fundamentals/reference/http-request-headers/#x-forwarded-proto
-# See also, especially the warnings:
-# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header
-SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
-
 # Secrets
 _SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None)
 if _SECRET_KEY is not None:

From 1ccf83d300a7fa0436c09c39a7d7fb955af00d0c Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 24 Apr 2025 12:59:18 -0300
Subject: [PATCH 327/656] ci: increase timeout for staging db refresh

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 36439aa6cf..3d0b7d80fe 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -453,7 +453,7 @@ jobs:
           token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
           inputs: '{ "sourceDb":"datatracker" }'
           wait-for-completion: true
-          wait-for-completion-timeout: 5m
+          wait-for-completion-timeout: 10m
           wait-for-completion-interval: 20s
           display-workflow-run-url: false
 

From f64470c7759ebcfd0e2589947dd7d372f510e436 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 25 Apr 2025 12:36:06 -0300
Subject: [PATCH 328/656] chore: fix typo in help_text (#8841)

---
 ietf/secr/announcement/forms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/secr/announcement/forms.py b/ietf/secr/announcement/forms.py
index 820ef79a09..91004ea270 100644
--- a/ietf/secr/announcement/forms.py
+++ b/ietf/secr/announcement/forms.py
@@ -130,7 +130,7 @@ def __init__(self, *args, **kwargs):
         # nomcom field is defined declaratively so label and help_text must be set here
         self.fields["nomcom"].label = "NomCom message:"
         self.fields["nomcom"].help_text = (
-            "If this is a NomCom announcement specifiy which NomCom group here"
+            "If this is a NomCom announcement specify which NomCom group here"
         )
         nomcom_roles = person.role_set.filter(
             group__in=self.fields["nomcom"].queryset, name="chair"

From 6f76df2d4902b458e65a74b0efa8cc1e46202495 Mon Sep 17 00:00:00 2001
From: Mark Nottingham <mnot@mnot.net>
Date: Sat, 26 Apr 2025 01:50:37 +1000
Subject: [PATCH 329/656] Put draft name first in subject of review email
 (#8830)

Fixes #5946.

This is the minimal fix; other optimisations might be possible,
but this gets the job done for the common case -- people scanning
for drafts they're interested in.
---
 ietf/doc/views_review.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py
index 6ac0e17791..1968b133ce 100644
--- a/ietf/doc/views_review.py
+++ b/ietf/doc/views_review.py
@@ -897,7 +897,7 @@ def complete_review(request, name, assignment_id=None, acronym=None):
 
             if need_to_email_review:
                 # email the review
-                subject = "{} {} {} of {}-{}".format(assignment.review_request.team.acronym.capitalize(),assignment.review_request.type.name.lower(),"partial review" if assignment.state_id == "part-completed" else "review", assignment.review_request.doc.name, assignment.reviewed_rev)
+                subject = "{}-{} {} {} {}".format(assignment.review_request.doc.name, assignment.reviewed_rev, assignment.review_request.type.name.lower(), assignment.review_request.team.acronym.capitalize(), "partial review" if assignment.state_id == "part-completed" else "review")
                 related_groups = [ assignment.review_request.team, ]
                 if assignment.review_request.doc.group:
                     related_groups.append(assignment.review_request.doc.group)

From 3ece7bd5fb526414451cdc5502d781c524ac47ae Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 25 Apr 2025 13:14:53 -0300
Subject: [PATCH 330/656] chore: tag db docker img with dump info via env
 (#8831)

---
 docker/db.Dockerfile | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docker/db.Dockerfile b/docker/db.Dockerfile
index 31c41011e6..0b57a80b70 100644
--- a/docker/db.Dockerfile
+++ b/docker/db.Dockerfile
@@ -28,3 +28,10 @@ ENV POSTGRES_PASSWORD=hk2j22sfiv
 ENV POSTGRES_USER=django
 ENV POSTGRES_DB=datatracker
 ENV POSTGRES_HOST_AUTH_METHOD=trust
+
+# build-args for db dump tagging - exposed in the environment and
+# in image metadata
+ARG datatracker_dumpinfo_date=""
+ENV DATATRACKER_DUMPINFO_DATE=$datatracker_dumpinfo_date
+ARG datatracker_snapshot=""
+ENV DATATRACKER_SNAPSHOT=$datatracker_snapshot

From afa57830a34db56f08105115d9dcc1d99a3eff84 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 25 Apr 2025 18:50:56 -0300
Subject: [PATCH 331/656] fix: restore proceedings refresh task (#8842)

* fix: merge error dropped @shared_task

* fix: cache proceedings longer than recompute interval

* test: update tests
---
 ietf/meeting/tasks.py       | 1 +
 ietf/meeting/tests_views.py | 6 ++++--
 ietf/meeting/utils.py       | 5 ++++-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py
index f1b7e8e079..20bb602682 100644
--- a/ietf/meeting/tasks.py
+++ b/ietf/meeting/tasks.py
@@ -34,6 +34,7 @@ def check_migrate_registrations_task():
     check_migrate_registrations()
 
 
+@shared_task
 def proceedings_content_refresh_task(*, all=False):
     """Refresh meeting proceedings cache
 
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index dfb414b61b..889a1bc882 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -8679,7 +8679,8 @@ def test_generate_proceedings_content(self, mock_caches):
         self.assertTrue(mock_default_cache.get.called)
         self.assertEqual(mock_default_cache.get.call_args.args[0], cache_key, "same cache key each time")
         self.assertTrue(mock_default_cache.set.called)
-        self.assertEqual(mock_default_cache.set.call_args, call(cache_key, proceedings_content, timeout=86400))
+        self.assertEqual(mock_default_cache.set.call_args.args, (cache_key, proceedings_content))
+        self.assertGreater(mock_default_cache.set.call_args.kwargs["timeout"], 86400)
         mock_default_cache.get.reset_mock()
         mock_default_cache.set.reset_mock()
 
@@ -8725,7 +8726,8 @@ def test_generate_proceedings_content(self, mock_caches):
         self.assertEqual(result, proceedings_content)  # should have recomputed the same thing
         self.assertFalse(mock_default_cache.get.called, "don't bother reading cache when force_refresh is True")
         self.assertTrue(mock_default_cache.set.called)
-        self.assertEqual(mock_default_cache.set.call_args, call(cache_key, proceedings_content, timeout=86400))
+        self.assertEqual(mock_default_cache.set.call_args.args, (cache_key, proceedings_content))
+        self.assertGreater(mock_default_cache.set.call_args.kwargs["timeout"], 86400)
 
     def test_named_session(self):
         """Session with a name should appear separately in the proceedings"""
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index d35914fd60..af12b6a274 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -1123,6 +1123,9 @@ def check_migrate_registrations():
 def generate_proceedings_content(meeting, force_refresh=False):
     """Render proceedings content for a meeting and update cache
     
+    Caches its value for 25 hours to ensure that the cache never expires if
+    we recompute the value daily.
+
     :meeting: meeting whose proceedings should be rendered
     :force_refresh: true to force regeneration and cache refresh
     """
@@ -1205,7 +1208,7 @@ def area_and_group_acronyms_from_session(s):
     cache.set(
         cache_key,
         rendered_content,
-        timeout=86400,  # one day, in seconds
+        timeout=3600 + 86400,  # one day + one hour, in seconds
     )
     return rendered_content
 

From c36ece86c70ec9b13e87c4e5852b4b2af4cc12b7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 28 Apr 2025 12:17:46 -0400
Subject: [PATCH 332/656] chore(deps): bump the npm group across 1 directory
 with 3 updates (#8850)

Bumps the npm group with 3 updates in the /dev/deploy-to-container directory: [dockerode](https://github.com/apocas/dockerode), [nanoid](https://github.com/ai/nanoid) and [nanoid-dictionary](https://github.com/CyberAP/nanoid-dictionary).


Updates `dockerode` from 4.0.4 to 4.0.6
- [Release notes](https://github.com/apocas/dockerode/releases)
- [Commits](https://github.com/apocas/dockerode/compare/v4.0.4...v4.0.6)

Updates `nanoid` from 5.1.4 to 5.1.5
- [Release notes](https://github.com/ai/nanoid/releases)
- [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md)
- [Commits](https://github.com/ai/nanoid/compare/5.1.4...5.1.5)

Updates `nanoid-dictionary` from 5.0.0-beta.1 to 5.0.0
- [Release notes](https://github.com/CyberAP/nanoid-dictionary/releases)
- [Changelog](https://github.com/CyberAP/nanoid-dictionary/blob/master/CHANGELOG.md)
- [Commits](https://github.com/CyberAP/nanoid-dictionary/commits/5.0.0)

---
updated-dependencies:
- dependency-name: dockerode
  dependency-version: 4.0.6
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
- dependency-name: nanoid
  dependency-version: 5.1.5
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
- dependency-name: nanoid-dictionary
  dependency-version: 5.0.0
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/deploy-to-container/package-lock.json | 206 ++++++++++++----------
 dev/deploy-to-container/package.json      |   6 +-
 2 files changed, 112 insertions(+), 100 deletions(-)

diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json
index f4ce2c367c..0954ec9af4 100644
--- a/dev/deploy-to-container/package-lock.json
+++ b/dev/deploy-to-container/package-lock.json
@@ -6,10 +6,10 @@
     "": {
       "name": "deploy-to-container",
       "dependencies": {
-        "dockerode": "^4.0.4",
+        "dockerode": "^4.0.6",
         "fs-extra": "^11.3.0",
-        "nanoid": "5.1.4",
-        "nanoid-dictionary": "5.0.0-beta.1",
+        "nanoid": "5.1.5",
+        "nanoid-dictionary": "5.0.0",
         "slugify": "1.6.6",
         "tar": "^7.4.3",
         "yargs": "^17.7.2"
@@ -285,7 +285,8 @@
           "type": "consulting",
           "url": "https://feross.org/support"
         }
-      ]
+      ],
+      "license": "MIT"
     },
     "node_modules/bcrypt-pbkdf": {
       "version": "1.0.2",
@@ -296,6 +297,17 @@
         "tweetnacl": "^0.14.3"
       }
     },
+    "node_modules/bl": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
+      "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+      "license": "MIT",
+      "dependencies": {
+        "buffer": "^5.5.0",
+        "inherits": "^2.0.4",
+        "readable-stream": "^3.4.0"
+      }
+    },
     "node_modules/brace-expansion": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
@@ -304,6 +316,30 @@
         "balanced-match": "^1.0.0"
       }
     },
+    "node_modules/buffer": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.1.13"
+      }
+    },
     "node_modules/buildcheck": {
       "version": "0.0.6",
       "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz",
@@ -316,7 +352,8 @@
     "node_modules/chownr": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
-      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
+      "license": "ISC"
     },
     "node_modules/cliui": {
       "version": "8.0.1",
@@ -407,9 +444,9 @@
       }
     },
     "node_modules/dockerode": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.4.tgz",
-      "integrity": "sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==",
+      "version": "4.0.6",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz",
+      "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==",
       "license": "Apache-2.0",
       "dependencies": {
         "@balena/dockerignore": "^1.0.2",
@@ -417,7 +454,7 @@
         "@grpc/proto-loader": "^0.7.13",
         "docker-modem": "^5.0.6",
         "protobufjs": "^7.3.2",
-        "tar-fs": "~2.0.1",
+        "tar-fs": "~2.1.2",
         "uuid": "^10.0.0"
       },
       "engines": {
@@ -438,6 +475,7 @@
       "version": "1.4.4",
       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
       "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
+      "license": "MIT",
       "dependencies": {
         "once": "^1.4.0"
       }
@@ -468,7 +506,8 @@
     "node_modules/fs-constants": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
-      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
+      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
+      "license": "MIT"
     },
     "node_modules/fs-extra": {
       "version": "11.3.0",
@@ -535,7 +574,8 @@
           "type": "consulting",
           "url": "https://feross.org/support"
         }
-      ]
+      ],
+      "license": "BSD-3-Clause"
     },
     "node_modules/inherits": {
       "version": "2.0.4",
@@ -652,7 +692,8 @@
     "node_modules/mkdirp-classic": {
       "version": "0.5.3",
       "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
-      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
+      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
+      "license": "MIT"
     },
     "node_modules/ms": {
       "version": "2.1.3",
@@ -668,9 +709,9 @@
       "optional": true
     },
     "node_modules/nanoid": {
-      "version": "5.1.4",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.4.tgz",
-      "integrity": "sha512-GTFcMIDgR7tqji/LpSY8rtg464VnJl/j6ypoehYnuGb+Y8qZUdtKB8WVCXon0UEZgFDbuUxpIl//6FHLHgXSNA==",
+      "version": "5.1.5",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz",
+      "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==",
       "funding": [
         {
           "type": "github",
@@ -686,14 +727,16 @@
       }
     },
     "node_modules/nanoid-dictionary": {
-      "version": "5.0.0-beta.1",
-      "resolved": "https://registry.npmjs.org/nanoid-dictionary/-/nanoid-dictionary-5.0.0-beta.1.tgz",
-      "integrity": "sha512-xBkL9zzkNjzJ/UnmWyiOUDVX/COoi05eS0oU28RYKFFQhdnzO5dTOPbVZ/fCFgIOGr1zNinDHJ68mm/KQfcgcw=="
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/nanoid-dictionary/-/nanoid-dictionary-5.0.0.tgz",
+      "integrity": "sha512-/iCyQHwt36XkaIvSE9fcC8p6DiMPCZMTSMj9UT56Cv6T7f5CuxvOMhpNncaNieQ4z4d32p7ruEtAfRsb7Ya8Gw==",
+      "license": "MIT"
     },
     "node_modules/once": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
       "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+      "license": "ISC",
       "dependencies": {
         "wrappy": "1"
       }
@@ -745,9 +788,10 @@
       }
     },
     "node_modules/pump": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-      "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz",
+      "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==",
+      "license": "MIT",
       "dependencies": {
         "end-of-stream": "^1.1.0",
         "once": "^1.3.1"
@@ -952,20 +996,22 @@
       }
     },
     "node_modules/tar-fs": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz",
-      "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz",
+      "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==",
+      "license": "MIT",
       "dependencies": {
         "chownr": "^1.1.1",
         "mkdirp-classic": "^0.5.2",
         "pump": "^3.0.0",
-        "tar-stream": "^2.0.0"
+        "tar-stream": "^2.1.4"
       }
     },
     "node_modules/tar-stream": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
       "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
+      "license": "MIT",
       "dependencies": {
         "bl": "^4.0.3",
         "end-of-stream": "^1.4.1",
@@ -977,39 +1023,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/tar-stream/node_modules/bl": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
-      "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
-      "dependencies": {
-        "buffer": "^5.5.0",
-        "inherits": "^2.0.4",
-        "readable-stream": "^3.4.0"
-      }
-    },
-    "node_modules/tar-stream/node_modules/buffer": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
-      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "dependencies": {
-        "base64-js": "^1.3.1",
-        "ieee754": "^1.1.13"
-      }
-    },
     "node_modules/tar/node_modules/chownr": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
@@ -1104,7 +1117,8 @@
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
+      "license": "ISC"
     },
     "node_modules/y18n": {
       "version": "5.0.8",
@@ -1352,6 +1366,16 @@
         "tweetnacl": "^0.14.3"
       }
     },
+    "bl": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
+      "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+      "requires": {
+        "buffer": "^5.5.0",
+        "inherits": "^2.0.4",
+        "readable-stream": "^3.4.0"
+      }
+    },
     "brace-expansion": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
@@ -1360,6 +1384,15 @@
         "balanced-match": "^1.0.0"
       }
     },
+    "buffer": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+      "requires": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.1.13"
+      }
+    },
     "buildcheck": {
       "version": "0.0.6",
       "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz",
@@ -1434,16 +1467,16 @@
       }
     },
     "dockerode": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.4.tgz",
-      "integrity": "sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==",
+      "version": "4.0.6",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz",
+      "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==",
       "requires": {
         "@balena/dockerignore": "^1.0.2",
         "@grpc/grpc-js": "^1.11.1",
         "@grpc/proto-loader": "^0.7.13",
         "docker-modem": "^5.0.6",
         "protobufjs": "^7.3.2",
-        "tar-fs": "~2.0.1",
+        "tar-fs": "~2.1.2",
         "uuid": "^10.0.0"
       }
     },
@@ -1613,14 +1646,14 @@
       "optional": true
     },
     "nanoid": {
-      "version": "5.1.4",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.4.tgz",
-      "integrity": "sha512-GTFcMIDgR7tqji/LpSY8rtg464VnJl/j6ypoehYnuGb+Y8qZUdtKB8WVCXon0UEZgFDbuUxpIl//6FHLHgXSNA=="
+      "version": "5.1.5",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz",
+      "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw=="
     },
     "nanoid-dictionary": {
-      "version": "5.0.0-beta.1",
-      "resolved": "https://registry.npmjs.org/nanoid-dictionary/-/nanoid-dictionary-5.0.0-beta.1.tgz",
-      "integrity": "sha512-xBkL9zzkNjzJ/UnmWyiOUDVX/COoi05eS0oU28RYKFFQhdnzO5dTOPbVZ/fCFgIOGr1zNinDHJ68mm/KQfcgcw=="
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/nanoid-dictionary/-/nanoid-dictionary-5.0.0.tgz",
+      "integrity": "sha512-/iCyQHwt36XkaIvSE9fcC8p6DiMPCZMTSMj9UT56Cv6T7f5CuxvOMhpNncaNieQ4z4d32p7ruEtAfRsb7Ya8Gw=="
     },
     "once": {
       "version": "1.4.0",
@@ -1664,9 +1697,9 @@
       }
     },
     "pump": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-      "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz",
+      "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==",
       "requires": {
         "end-of-stream": "^1.1.0",
         "once": "^1.3.1"
@@ -1809,14 +1842,14 @@
       }
     },
     "tar-fs": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz",
-      "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz",
+      "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==",
       "requires": {
         "chownr": "^1.1.1",
         "mkdirp-classic": "^0.5.2",
         "pump": "^3.0.0",
-        "tar-stream": "^2.0.0"
+        "tar-stream": "^2.1.4"
       }
     },
     "tar-stream": {
@@ -1829,27 +1862,6 @@
         "fs-constants": "^1.0.0",
         "inherits": "^2.0.3",
         "readable-stream": "^3.1.1"
-      },
-      "dependencies": {
-        "bl": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
-          "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
-          "requires": {
-            "buffer": "^5.5.0",
-            "inherits": "^2.0.4",
-            "readable-stream": "^3.4.0"
-          }
-        },
-        "buffer": {
-          "version": "5.7.1",
-          "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
-          "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
-          "requires": {
-            "base64-js": "^1.3.1",
-            "ieee754": "^1.1.13"
-          }
-        }
       }
     },
     "tweetnacl": {
diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json
index eb8cb9012a..09716c3094 100644
--- a/dev/deploy-to-container/package.json
+++ b/dev/deploy-to-container/package.json
@@ -2,10 +2,10 @@
   "name": "deploy-to-container",
   "type": "module",
   "dependencies": {
-    "dockerode": "^4.0.4",
+    "dockerode": "^4.0.6",
     "fs-extra": "^11.3.0",
-    "nanoid": "5.1.4",
-    "nanoid-dictionary": "5.0.0-beta.1",
+    "nanoid": "5.1.5",
+    "nanoid-dictionary": "5.0.0",
     "slugify": "1.6.6",
     "tar": "^7.4.3",
     "yargs": "^17.7.2"

From f67aa6b4a341df9100ad73638eacec5f2afafd15 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 28 Apr 2025 12:18:33 -0400
Subject: [PATCH 333/656] chore(deps): bump actions/download-artifact from
 4.1.9 to 4.3.0 (#8849)

Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.1.9 to 4.3.0.
- [Release notes](https://github.com/actions/download-artifact/releases)
- [Commits](https://github.com/actions/download-artifact/compare/v4.1.9...v4.3.0)

---
updated-dependencies:
- dependency-name: actions/download-artifact
  dependency-version: 4.3.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3d0b7d80fe..0273fa4302 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -185,7 +185,7 @@ jobs:
 
     - name: Download a Coverage Results
       if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
-      uses: actions/download-artifact@v4.1.9
+      uses: actions/download-artifact@v4.3.0
       with:
         name: coverage
 
@@ -292,7 +292,7 @@ jobs:
 
     - name: Download Coverage Results
       if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
-      uses: actions/download-artifact@v4.1.9
+      uses: actions/download-artifact@v4.3.0
       with:
         name: coverage
 
@@ -407,7 +407,7 @@ jobs:
     - uses: actions/checkout@v4
         
     - name: Download a Release Artifact
-      uses: actions/download-artifact@v4.1.9
+      uses: actions/download-artifact@v4.3.0
       with:
         name: release-${{ env.PKG_VERSION }}
         

From 31c6c32ffb651bb99c1157788c3db5e9e3a25300 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 28 Apr 2025 12:19:03 -0400
Subject: [PATCH 334/656] chore(deps): bump the npm group in /dev/diff with 2
 updates (#8759)

Bumps the npm group in /dev/diff with 2 updates: [dockerode](https://github.com/apocas/dockerode) and [luxon](https://github.com/moment/luxon).


Updates `dockerode` from 4.0.4 to 4.0.5
- [Release notes](https://github.com/apocas/dockerode/releases)
- [Commits](https://github.com/apocas/dockerode/compare/v4.0.4...v4.0.5)

Updates `luxon` from 3.5.0 to 3.6.0
- [Changelog](https://github.com/moment/luxon/blob/master/CHANGELOG.md)
- [Commits](https://github.com/moment/luxon/compare/3.5.0...3.6.0)

---
updated-dependencies:
- dependency-name: dockerode
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
- dependency-name: luxon
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/diff/package-lock.json | 174 +++++++++++++++++++------------------
 dev/diff/package.json      |   4 +-
 2 files changed, 93 insertions(+), 85 deletions(-)

diff --git a/dev/diff/package-lock.json b/dev/diff/package-lock.json
index 4e92d2a6e0..d1c2fbd763 100644
--- a/dev/diff/package-lock.json
+++ b/dev/diff/package-lock.json
@@ -7,7 +7,7 @@
       "name": "diff",
       "dependencies": {
         "chalk": "^5.4.1",
-        "dockerode": "^4.0.4",
+        "dockerode": "^4.0.6",
         "enquirer": "^2.4.1",
         "extract-zip": "^2.0.1",
         "fs-extra": "^11.3.0",
@@ -15,7 +15,7 @@
         "keypress": "^0.2.1",
         "listr2": "^6.6.1",
         "lodash-es": "^4.17.21",
-        "luxon": "^3.5.0",
+        "luxon": "^3.6.1",
         "pretty-bytes": "^6.1.1",
         "tar": "^7.4.3",
         "yargs": "^17.7.2"
@@ -359,7 +359,8 @@
           "type": "consulting",
           "url": "https://feross.org/support"
         }
-      ]
+      ],
+      "license": "MIT"
     },
     "node_modules/bcrypt-pbkdf": {
       "version": "1.0.2",
@@ -370,6 +371,17 @@
         "tweetnacl": "^0.14.3"
       }
     },
+    "node_modules/bl": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
+      "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+      "license": "MIT",
+      "dependencies": {
+        "buffer": "^5.5.0",
+        "inherits": "^2.0.4",
+        "readable-stream": "^3.4.0"
+      }
+    },
     "node_modules/brace-expansion": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
@@ -378,6 +390,30 @@
         "balanced-match": "^1.0.0"
       }
     },
+    "node_modules/buffer": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.1.13"
+      }
+    },
     "node_modules/buffer-crc32": {
       "version": "0.2.13",
       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
@@ -435,7 +471,8 @@
     "node_modules/chownr": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
-      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
+      "license": "ISC"
     },
     "node_modules/cli-cursor": {
       "version": "4.0.0",
@@ -638,9 +675,9 @@
       }
     },
     "node_modules/dockerode": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.4.tgz",
-      "integrity": "sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==",
+      "version": "4.0.6",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz",
+      "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==",
       "license": "Apache-2.0",
       "dependencies": {
         "@balena/dockerignore": "^1.0.2",
@@ -648,7 +685,7 @@
         "@grpc/proto-loader": "^0.7.13",
         "docker-modem": "^5.0.6",
         "protobufjs": "^7.3.2",
-        "tar-fs": "~2.0.1",
+        "tar-fs": "~2.1.2",
         "uuid": "^10.0.0"
       },
       "engines": {
@@ -776,7 +813,8 @@
     "node_modules/fs-constants": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
-      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
+      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
+      "license": "MIT"
     },
     "node_modules/fs-extra": {
       "version": "11.3.0",
@@ -895,7 +933,8 @@
           "type": "consulting",
           "url": "https://feross.org/support"
         }
-      ]
+      ],
+      "license": "BSD-3-Clause"
     },
     "node_modules/inherits": {
       "version": "2.0.4",
@@ -1186,9 +1225,10 @@
       }
     },
     "node_modules/luxon": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
-      "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==",
+      "version": "3.6.1",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.6.1.tgz",
+      "integrity": "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ==",
+      "license": "MIT",
       "engines": {
         "node": ">=12"
       }
@@ -1263,7 +1303,8 @@
     "node_modules/mkdirp-classic": {
       "version": "0.5.3",
       "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
-      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
+      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
+      "license": "MIT"
     },
     "node_modules/ms": {
       "version": "2.1.2",
@@ -1662,20 +1703,22 @@
       }
     },
     "node_modules/tar-fs": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz",
-      "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz",
+      "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==",
+      "license": "MIT",
       "dependencies": {
         "chownr": "^1.1.1",
         "mkdirp-classic": "^0.5.2",
         "pump": "^3.0.0",
-        "tar-stream": "^2.0.0"
+        "tar-stream": "^2.1.4"
       }
     },
     "node_modules/tar-stream": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
       "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
+      "license": "MIT",
       "dependencies": {
         "bl": "^4.0.3",
         "end-of-stream": "^1.4.1",
@@ -1687,39 +1730,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/tar-stream/node_modules/bl": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
-      "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
-      "dependencies": {
-        "buffer": "^5.5.0",
-        "inherits": "^2.0.4",
-        "readable-stream": "^3.4.0"
-      }
-    },
-    "node_modules/tar-stream/node_modules/buffer": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
-      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "dependencies": {
-        "base64-js": "^1.3.1",
-        "ieee754": "^1.1.13"
-      }
-    },
     "node_modules/tar/node_modules/chownr": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
@@ -2115,6 +2125,16 @@
         "tweetnacl": "^0.14.3"
       }
     },
+    "bl": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
+      "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
+      "requires": {
+        "buffer": "^5.5.0",
+        "inherits": "^2.0.4",
+        "readable-stream": "^3.4.0"
+      }
+    },
     "brace-expansion": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
@@ -2123,6 +2143,15 @@
         "balanced-match": "^1.0.0"
       }
     },
+    "buffer": {
+      "version": "5.7.1",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
+      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
+      "requires": {
+        "base64-js": "^1.3.1",
+        "ieee754": "^1.1.13"
+      }
+    },
     "buffer-crc32": {
       "version": "0.2.13",
       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
@@ -2298,16 +2327,16 @@
       }
     },
     "dockerode": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.4.tgz",
-      "integrity": "sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==",
+      "version": "4.0.6",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz",
+      "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==",
       "requires": {
         "@balena/dockerignore": "^1.0.2",
         "@grpc/grpc-js": "^1.11.1",
         "@grpc/proto-loader": "^0.7.13",
         "docker-modem": "^5.0.6",
         "protobufjs": "^7.3.2",
-        "tar-fs": "~2.0.1",
+        "tar-fs": "~2.1.2",
         "uuid": "^10.0.0"
       }
     },
@@ -2669,9 +2698,9 @@
       "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ=="
     },
     "luxon": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
-      "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ=="
+      "version": "3.6.1",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.6.1.tgz",
+      "integrity": "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ=="
     },
     "mimic-fn": {
       "version": "2.1.0",
@@ -2989,14 +3018,14 @@
       }
     },
     "tar-fs": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz",
-      "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz",
+      "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==",
       "requires": {
         "chownr": "^1.1.1",
         "mkdirp-classic": "^0.5.2",
         "pump": "^3.0.0",
-        "tar-stream": "^2.0.0"
+        "tar-stream": "^2.1.4"
       }
     },
     "tar-stream": {
@@ -3009,27 +3038,6 @@
         "fs-constants": "^1.0.0",
         "inherits": "^2.0.3",
         "readable-stream": "^3.1.1"
-      },
-      "dependencies": {
-        "bl": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
-          "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
-          "requires": {
-            "buffer": "^5.5.0",
-            "inherits": "^2.0.4",
-            "readable-stream": "^3.4.0"
-          }
-        },
-        "buffer": {
-          "version": "5.7.1",
-          "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
-          "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
-          "requires": {
-            "base64-js": "^1.3.1",
-            "ieee754": "^1.1.13"
-          }
-        }
       }
     },
     "tweetnacl": {
diff --git a/dev/diff/package.json b/dev/diff/package.json
index 875e242d3d..a5a7beb0df 100644
--- a/dev/diff/package.json
+++ b/dev/diff/package.json
@@ -3,7 +3,7 @@
   "type": "module",
   "dependencies": {
     "chalk": "^5.4.1",
-    "dockerode": "^4.0.4",
+    "dockerode": "^4.0.6",
     "enquirer": "^2.4.1",
     "extract-zip": "^2.0.1",
     "fs-extra": "^11.3.0",
@@ -11,7 +11,7 @@
     "keypress": "^0.2.1",
     "listr2": "^6.6.1",
     "lodash-es": "^4.17.21",
-    "luxon": "^3.5.0",
+    "luxon": "^3.6.1",
     "pretty-bytes": "^6.1.1",
     "tar": "^7.4.3",
     "yargs": "^17.7.2"

From ae542fb0d55cfdcba401f6b427e07f4fc2f6d8e7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 28 Apr 2025 12:19:29 -0400
Subject: [PATCH 335/656] chore(deps): bump luxon in /dev/coverage-action in
 the npm group (#8757)

Bumps the npm group in /dev/coverage-action with 1 update: [luxon](https://github.com/moment/luxon).


Updates `luxon` from 3.5.0 to 3.6.0
- [Changelog](https://github.com/moment/luxon/blob/master/CHANGELOG.md)
- [Commits](https://github.com/moment/luxon/compare/3.5.0...3.6.0)

---
updated-dependencies:
- dependency-name: luxon
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/coverage-action/package-lock.json | 15 ++++++++-------
 dev/coverage-action/package.json      |  2 +-
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/dev/coverage-action/package-lock.json b/dev/coverage-action/package-lock.json
index f23a9b66c7..33aed07072 100644
--- a/dev/coverage-action/package-lock.json
+++ b/dev/coverage-action/package-lock.json
@@ -12,7 +12,7 @@
         "@actions/core": "1.11.1",
         "@actions/github": "6.0.0",
         "lodash": "4.17.21",
-        "luxon": "3.5.0"
+        "luxon": "3.6.1"
       }
     },
     "node_modules/@actions/core": {
@@ -209,9 +209,10 @@
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
     },
     "node_modules/luxon": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
-      "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==",
+      "version": "3.6.1",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.6.1.tgz",
+      "integrity": "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ==",
+      "license": "MIT",
       "engines": {
         "node": ">=12"
       }
@@ -413,9 +414,9 @@
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
     },
     "luxon": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
-      "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ=="
+      "version": "3.6.1",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.6.1.tgz",
+      "integrity": "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ=="
     },
     "once": {
       "version": "1.4.0",
diff --git a/dev/coverage-action/package.json b/dev/coverage-action/package.json
index 1d03fe3dab..9e1433f1ce 100644
--- a/dev/coverage-action/package.json
+++ b/dev/coverage-action/package.json
@@ -9,6 +9,6 @@
     "@actions/core": "1.11.1",
     "@actions/github": "6.0.0",
     "lodash": "4.17.21",
-    "luxon": "3.5.0"
+    "luxon": "3.6.1"
   }
 }

From f505ce4946c3d4fc4656084bbca3790ff39f30c8 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 30 Apr 2025 12:13:34 -0300
Subject: [PATCH 336/656] chore(deps): pin django-celery-beat (#8870)

Related to https://github.com/celery/django-celery-beat/issues/875
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b00a21ab49..eb72600fe3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,7 +17,7 @@ Django>4.2,<5
 django-admin-rangefilter>=0.13.2
 django-analytical>=3.1.0
 django-bootstrap5>=21.3
-django-celery-beat>=2.3.0
+django-celery-beat>=2.3.0,<2.8.0  # pin until https://github.com/celery/django-celery-beat/issues/875 is resolved, then revisit
 django-celery-results>=2.5.1
 django-csp>=3.7
 django-cors-headers>=3.11.0

From 5b78311256560cc0c9c4be6e6ab077ad901bc336 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 30 Apr 2025 11:27:10 -0400
Subject: [PATCH 337/656] ci: update build.yml to allow chore for semver bump

---
 .github/workflows/build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0273fa4302..bc9680cd9d 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -79,6 +79,7 @@ jobs:
         token: ${{ github.token }}
         branch: release
         skipInvalidTags: true
+        patchList: fix, bugfix, perf, refactor, test, tests, chore
 
     - name: Get Dev Version
       if: ${{ github.ref_name != 'release' }}

From 7765ff1106d10808314fc0c2175606e92f9956c1 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 1 May 2025 15:49:49 -0300
Subject: [PATCH 338/656] feat: set BEAT_CRON_STARTING_DEADLINE (#8876)

https://docs.celeryq.dev/en/main/userguide/configuration.html#beat-cron-starting-deadline
---
 ietf/settings.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ietf/settings.py b/ietf/settings.py
index 8005aca8cc..c21120f77a 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -1268,6 +1268,7 @@ def skip_unreadable_post(record):
 CELERY_BROKER_URL = 'amqp://mq/'
 CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
 CELERY_BEAT_SYNC_EVERY = 1  # update DB after every event
+CELERY_BEAT_CRON_STARTING_DEADLINE = 1800  # seconds after a missed deadline before abandoning a cron task
 CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True  # the default, but setting it squelches a warning
 # Use a result backend so we can chain tasks. This uses the rpc backend, see
 # https://docs.celeryq.dev/en/stable/userguide/tasks.html#rpc-result-backend-rabbitmq-qpid

From a62387d635c70765d3a446dac9d2b9f122244669 Mon Sep 17 00:00:00 2001
From: Mark Nottingham <mnot@mnot.net>
Date: Fri, 2 May 2025 04:57:17 +1000
Subject: [PATCH 339/656] Make submission text less passive (#8845)

Fixes #8807.
---
 ietf/templates/base/menu.html          | 2 +-
 ietf/templates/submit/submit_base.html | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html
index 1d7bad8724..3ae0d3dc57 100644
--- a/ietf/templates/base/menu.html
+++ b/ietf/templates/base/menu.html
@@ -95,7 +95,7 @@
     <li>
         <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
            href="{% url "ietf.submit.views.upload_submission" %}">
-            I-D submission
+            Submit an Internet-Draft
         </a>
     </li>
     <li>
diff --git a/ietf/templates/submit/submit_base.html b/ietf/templates/submit/submit_base.html
index 0613cf5d6a..a2c7be1a3b 100644
--- a/ietf/templates/submit/submit_base.html
+++ b/ietf/templates/submit/submit_base.html
@@ -4,7 +4,7 @@
 {% block pagehead %}{{ block.super }}{% endblock %}
 {% block content %}
     {% origin %}
-    <h1>Internet-Draft submission</h1>
+    <h1>Submit an Internet-Draft</h1>
     <ul class="nav nav-tabs my-3">
         <li class="nav-item">
             <a class="nav-link {% if selected == "index" %}active{% endif %}"

From 5b77ba6b716d5da0ef8ea60de365e75952d0e1d0 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 13 May 2025 12:21:01 -0300
Subject: [PATCH 340/656] chore(deps): bump to fixed django-celery-beat (#8896)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index eb72600fe3..6f72bd784d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,7 +17,7 @@ Django>4.2,<5
 django-admin-rangefilter>=0.13.2
 django-analytical>=3.1.0
 django-bootstrap5>=21.3
-django-celery-beat>=2.3.0,<2.8.0  # pin until https://github.com/celery/django-celery-beat/issues/875 is resolved, then revisit
+django-celery-beat>=2.8.1
 django-celery-results>=2.5.1
 django-csp>=3.7
 django-cors-headers>=3.11.0

From ae67dfc550903ce58db51d5450754f75e1b798a8 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Tue, 13 May 2025 15:34:42 +0000
Subject: [PATCH 341/656] ci: update base image target version to 20250513T1521

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index bdf3cff0e4..21304781dc 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250421T1600
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250513T1521
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index bdfdb2eed1..b59f430fcb 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250421T1600
+20250513T1521

From 9d91471942409fb5e853d4a7164065dcbb24942f Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 14 May 2025 00:03:06 -0400
Subject: [PATCH 342/656] chore: add types to issue templates

---
 .github/ISSUE_TEMPLATE/new-feature.yml  | 1 +
 .github/ISSUE_TEMPLATE/report-a-bug.yml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/ISSUE_TEMPLATE/new-feature.yml b/.github/ISSUE_TEMPLATE/new-feature.yml
index ddf0b575e8..285081e1c8 100644
--- a/.github/ISSUE_TEMPLATE/new-feature.yml
+++ b/.github/ISSUE_TEMPLATE/new-feature.yml
@@ -1,6 +1,7 @@
 name: Suggest new feature or enhancement
 description: Propose a new idea to be implemented.
 labels: ["enhancement"]
+type: Feature
 body:
   - type: markdown
     attributes:
diff --git a/.github/ISSUE_TEMPLATE/report-a-bug.yml b/.github/ISSUE_TEMPLATE/report-a-bug.yml
index d5b725d721..47fa1185b4 100644
--- a/.github/ISSUE_TEMPLATE/report-a-bug.yml
+++ b/.github/ISSUE_TEMPLATE/report-a-bug.yml
@@ -1,6 +1,7 @@
 name: Report a Datatracker bug
 description: Something in the datatracker's behavior isn't right? File a bug report. Don't use this to report RFC errata or issues with the content of Internet-Drafts.
 labels: ["bug"]
+type: Bug
 body:
   - type: markdown
     attributes:

From 9237758228ea4df36cda37389fc0d294345f6163 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 14 May 2025 13:27:34 -0300
Subject: [PATCH 343/656] chore(deps): revert "bump to fixed
 django-celery-beat" (#8899)

This reverts commit 5b77ba6b716d5da0ef8ea60de365e75952d0e1d0. There are still reports of trouble with django-celery-beat 2.8.1.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 6f72bd784d..eb72600fe3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,7 +17,7 @@ Django>4.2,<5
 django-admin-rangefilter>=0.13.2
 django-analytical>=3.1.0
 django-bootstrap5>=21.3
-django-celery-beat>=2.8.1
+django-celery-beat>=2.3.0,<2.8.0  # pin until https://github.com/celery/django-celery-beat/issues/875 is resolved, then revisit
 django-celery-results>=2.5.1
 django-csp>=3.7
 django-cors-headers>=3.11.0

From 51f5957a40c69ad3d858429d26c2f7bc38e6b0d4 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Wed, 14 May 2025 16:40:08 +0000
Subject: [PATCH 344/656] ci: update base image target version to 20250514T1627

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 21304781dc..3ad11e4d9e 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250513T1521
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250514T1627
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index b59f430fcb..4bd4db561e 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250513T1521
+20250514T1627

From 5fd8ae6bb828222d03ccf1431faad6b151dcae2e Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 14 May 2025 15:34:29 -0500
Subject: [PATCH 345/656] feat: summarize all documents in iesg processing
 (#8878)

* feat: more detailed view of all docs in iesg processing (#8838)

* feat: more detailed view of all docs in iesg processing

* fix: commit new template

* feat: cache the new page for 5m in slowpages

* fix: add endcache

* fix: load cache tag definition

* fix: exclude things not in progress

* fix: link to the newer all ad dashboard from the base dashboard

* fix: reorder ad go-ahead to before iesg eval on search result pages

* fix: add a deprecation warning to the older docs in IESG processing view
---
 ietf/doc/tests.py                             | 24 ++++++
 ietf/doc/urls.py                              |  1 +
 ietf/doc/utils_search.py                      |  2 +-
 ietf/doc/views_search.py                      | 84 +++++++++++++++++++
 ietf/templates/doc/ad_list.html               |  1 +
 ietf/templates/doc/drafts_for_iesg.html       | 20 +++++
 .../templates/doc/drafts_in_iesg_process.html |  1 +
 7 files changed, 132 insertions(+), 1 deletion(-)
 create mode 100644 ietf/templates/doc/drafts_for_iesg.html

diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index d74688f3f6..1229df46c5 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -403,6 +403,30 @@ def test_docs_for_ad(self):
         self.assertContains(r, discuss_other.doc.name)
         self.assertContains(r, block_other.doc.name)
 
+    def test_docs_for_iesg(self):
+        ad1 = RoleFactory(name_id='ad',group__type_id='area',group__state_id='active').person
+        ad2 = RoleFactory(name_id='ad',group__type_id='area',group__state_id='active').person
+
+        draft = IndividualDraftFactory(ad=ad1)
+        draft.action_holders.set([PersonFactory()])
+        draft.set_state(State.objects.get(type='draft-iesg', slug='lc'))
+        rfc = IndividualRfcFactory(ad=ad2)
+        conflrev = DocumentFactory(type_id='conflrev',ad=ad1)
+        conflrev.set_state(State.objects.get(type='conflrev', slug='iesgeval'))
+        statchg = DocumentFactory(type_id='statchg',ad=ad2)
+        statchg.set_state(State.objects.get(type='statchg', slug='iesgeval'))
+        charter = CharterFactory(name='charter-ietf-ames',ad=ad1)
+        charter.set_state(State.objects.get(type='charter', slug='iesgrev'))
+
+        r = self.client.get(urlreverse('ietf.doc.views_search.docs_for_iesg'))
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, draft.name)
+        self.assertContains(r, escape(draft.action_holders.first().name))
+        self.assertNotContains(r, rfc.name)
+        self.assertContains(r, conflrev.name)
+        self.assertContains(r, statchg.name)
+        self.assertContains(r, charter.name)
+
     def test_auth48_doc_for_ad(self):
         """Docs in AUTH48 state should have a decoration"""
         ad = RoleFactory(name_id='ad', group__type_id='area', group__state_id='active').person
diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py
index 6e2f03a3dc..0fa1a04b49 100644
--- a/ietf/doc/urls.py
+++ b/ietf/doc/urls.py
@@ -53,6 +53,7 @@
     url(r'^ad/?$', views_search.ad_workload),
     url(r'^ad/(?P<name>[^/]+)/?$', views_search.docs_for_ad),
     url(r'^ad2/(?P<name>[\w.-]+)/$', RedirectView.as_view(url='/doc/ad/%(name)s/', permanent=True)),
+    url(r'^for_iesg/?$', views_search.docs_for_iesg),
     url(r'^rfc-status-changes/?$', views_status_change.rfc_status_changes),
     url(r'^start-rfc-status-change/(?:%(name)s/)?$' % settings.URL_REGEXPS, views_status_change.start_rfc_status_change),
     url(r'^bof-requests/?$', views_bofreq.bof_requests),
diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py
index 59b64ad307..cfc8a872f8 100644
--- a/ietf/doc/utils_search.py
+++ b/ietf/doc/utils_search.py
@@ -299,10 +299,10 @@ def num(i):
         "ad-eval",
         "lc-req",
         "lc",
+        "goaheadw",
         "writeupw",
         # "defer",  # probably not a useful state to show, since it's rare
         "iesg-eva",
-        "goaheadw",
         "approved",
         "ann",
     ],
diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py
index 9e9b5e88dd..3433a9ca31 100644
--- a/ietf/doc/views_search.py
+++ b/ietf/doc/views_search.py
@@ -752,6 +752,90 @@ def sort_key(doc):
     )
 
 
+def docs_for_iesg(request):
+    def sort_key(doc):
+        dt = doc_type(doc)
+        dt_key = list(AD_WORKLOAD.keys()).index(dt)
+        ds = doc_state(doc)
+        ds_key = AD_WORKLOAD[dt].index(ds) if ds in AD_WORKLOAD[dt] else 99
+        return dt_key * 100 + ds_key
+
+    results, meta = prepare_document_table(
+        request,
+        Document.objects.filter(
+            ad__in=Person.objects.filter(
+                Q(
+                    role__name__in=("pre-ad", "ad"),
+                    role__group__type="area",
+                    role__group__state="active",
+                )
+            )
+        ).exclude(
+            type_id="rfc",
+        ).exclude(
+            type_id="draft",
+            states__type="draft", 
+            states__slug__in=["repl", "rfc"],
+        ).exclude(
+            type_id="draft",
+            states__type="draft-iesg",
+            states__slug__in=["idexists", "rfcqueue"],
+        ).exclude(
+            type_id="conflrev",
+            states__type="conflrev",
+            states__slug__in=["appr-noprob-sent", "appr-reqnopub-sent", "withdraw", "dead"],
+        ).exclude(
+            type_id="statchg",
+            states__type="statchg",
+            states__slug__in=["appr-sent", "dead"],
+        ).exclude(
+            type_id="charter",
+            states__type="charter",
+            states__slug__in=["notrev", "infrev", "approved", "replaced"],
+        ),
+        max_results=1000,
+        show_ad_and_shepherd=True,
+    )
+    results.sort(key=lambda d: sort_key(d))
+
+    # filter out some results
+    results = [
+        r
+        for r in results
+        if not (
+            r.type_id == "charter"
+            and (
+                r.group.state_id == "abandon"
+                or r.get_state_slug("charter") == "replaced"
+            )
+        )
+        and not (
+            r.type_id == "draft"
+            and (
+                r.get_state_slug("draft-iesg") == "dead"
+                or r.get_state_slug("draft") == "repl"
+                or r.get_state_slug("draft") == "rfc"
+            )
+        )
+    ]
+
+    _calculate_state_name = get_state_name_calculator()
+    for d in results:
+        dt = d.type.slug
+        d.search_heading = _calculate_state_name(dt, doc_state(d))
+        if d.search_heading != "RFC":
+            d.search_heading += f" {doc_type_name(dt)}"
+
+    return render(
+        request,
+        "doc/drafts_for_iesg.html",
+        {
+            "docs": results,
+            "meta": meta,
+        },
+    )
+
+
 def drafts_in_last_call(request):
     lc_state = State.objects.get(type="draft-iesg", slug="lc").pk
     form = SearchForm({'by':'state','state': lc_state, 'rfcs':'on', 'activedrafts':'on'})
diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html
index a73264c0f3..1d7b6e2b54 100644
--- a/ietf/templates/doc/ad_list.html
+++ b/ietf/templates/doc/ad_list.html
@@ -29,6 +29,7 @@ <h1>IESG Dashboard</h1>
         are only shown to logged-in Area Directors.
     </div>
     {% endif %}
+    <p><a href="{% url 'ietf.doc.views_search.docs_for_iesg' %}">Documents in IESG Processing</a></p>
     {% for dt in metadata %}
         <h2 class="mt-5" id="{{ dt.type.0 }}">{{ dt.type.1 }} State Counts</h2>
         <table class="table table-sm table-striped table-bordered tablesorter navskip">
diff --git a/ietf/templates/doc/drafts_for_iesg.html b/ietf/templates/doc/drafts_for_iesg.html
new file mode 100644
index 0000000000..d7a79fb714
--- /dev/null
+++ b/ietf/templates/doc/drafts_for_iesg.html
@@ -0,0 +1,20 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{% load origin static %}
+{% load cache %}
+{% load ietf_filters %}
+{% load person_filters %}
+{% block pagehead %}
+    <link rel="stylesheet" href="{% static "ietf/css/list.css" %}">
+{% endblock %}
+{% block title %}Documents for the IESG{% endblock %}
+{% block content %}
+    {% cache 300 ietf_doc_drafts_for_iesg using="slowpages" %}
+    {% origin %}
+    <h1 class="mt-4">Documents for the IESG</h1>
+    {% include "doc/search/search_results.html" with start_table=True end_table=True %}
+    {% endcache %}
+{% endblock %}
+{% block js %}
+    <script src="{% static "ietf/js/list.js" %}"></script>
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/doc/drafts_in_iesg_process.html b/ietf/templates/doc/drafts_in_iesg_process.html
index f128e528a5..d9b09e984e 100644
--- a/ietf/templates/doc/drafts_in_iesg_process.html
+++ b/ietf/templates/doc/drafts_in_iesg_process.html
@@ -10,6 +10,7 @@
 {% block content %}
     {% origin %}
     <h1>{{ title }}</h1>
+    <h2><i class="bi bi-exclamation-triangle"></i>This view is deprecated, and will soon redirect to a <a href="{% url 'ietf.doc.views_search.docs_for_iesg' %}">different representation</a></h2>
     <table class="table table-sm table-striped tablesorter">
         <thead>
             <tr>

From 06de5dda67610a296c3e1a70255ce4030216834a Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 15 May 2025 13:28:41 -0500
Subject: [PATCH 346/656] fix: adjust docevent text for docs moving telechat
 dates. (#8905)

---
 ietf/doc/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index 3ddd904c75..115b28b09b 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -771,7 +771,7 @@ def update_telechat(request, doc, by, new_telechat_date, new_returning_item=None
         else:
             e.desc = "Removed from agenda for telechat"
     elif on_agenda and new_telechat_date != prev_telechat:
-        e.desc = "Telechat date has been changed to <b>%s</b> from <b>%s</b>" % (
+        e.desc = "Telechat date has been changed to <b>%s</b> (Previous date was <b>%s</b>)" % (
             new_telechat_date, prev_telechat)
     else:
         # we didn't reschedule but flipped returning item bit - let's

From 5afea6256aed9a2a8941563d23569d847229ccdc Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 27 May 2025 10:19:37 -0300
Subject: [PATCH 347/656] fix: paginate search results correctly (#8927)

* fix: paginate search results correctly

* style: magic number -> named constant
---
 ietf/person/views.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ietf/person/views.py b/ietf/person/views.py
index bb1fa79f84..a37b164311 100644
--- a/ietf/person/views.py
+++ b/ietf/person/views.py
@@ -62,8 +62,9 @@ def ajax_select2_search(request, model_name):
         page = int(request.GET.get("p", 1)) - 1
     except ValueError:
         page = 0
-
-    objs = objs.distinct()[page:page + 10]
+    PAGE_SIZE = 10
+    first_item = page * PAGE_SIZE
+    objs = objs.distinct()[first_item:first_item + PAGE_SIZE]
 
     return HttpResponse(select2_id_name_json(objs), content_type='application/json')
 

From 01170dfea1f417dedcda0787d67b5035ecfb22fa Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Thu, 12 Jun 2025 04:41:50 +1200
Subject: [PATCH 348/656] test: Add test for api.ietf_utils (#8965)

* test: Add test for api.ietf_utils

* test: Use ietf.utils.test_utils instead of django.test

* test: Add test case for request without a token

* test: Test for misconfigured endpoint

* test: Improve tests

* test: Add test for a API call without X_API_KEY header
---
 ietf/api/tests_ietf_utils.py | 86 ++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 ietf/api/tests_ietf_utils.py

diff --git a/ietf/api/tests_ietf_utils.py b/ietf/api/tests_ietf_utils.py
new file mode 100644
index 0000000000..b8d7fea7b4
--- /dev/null
+++ b/ietf/api/tests_ietf_utils.py
@@ -0,0 +1,86 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.test import RequestFactory
+from django.test.utils import override_settings
+
+from ietf.api.ietf_utils import is_valid_token, requires_api_token
+from ietf.utils.test_utils import TestCase
+
+
+class IetfUtilsTests(TestCase):
+    @override_settings(
+        APP_API_TOKENS={
+            "ietf.api.foobar": ["valid-token"],
+            "ietf.api.misconfigured": "valid-token",  # misconfigured
+        }
+    )
+    def test_is_valid_token(self):
+        self.assertFalse(is_valid_token("ietf.fake.endpoint", "valid-token"))
+        self.assertFalse(is_valid_token("ietf.api.foobar", "invalid-token"))
+        self.assertFalse(is_valid_token("ietf.api.foobar", None))
+        self.assertTrue(is_valid_token("ietf.api.foobar", "valid-token"))
+
+        # misconfiguration
+        self.assertFalse(is_valid_token("ietf.api.misconfigured", "v"))
+        self.assertFalse(is_valid_token("ietf.api.misconfigured", None))
+        self.assertTrue(is_valid_token("ietf.api.misconfigured", "valid-token"))
+
+    @override_settings(
+        APP_API_TOKENS={
+            "ietf.api.foo": ["valid-token"],
+            "ietf.api.bar": ["another-token"],
+            "ietf.api.misconfigured": "valid-token",  # misconfigured
+        }
+    )
+    def test_requires_api_token(self):
+        @requires_api_token("ietf.api.foo")
+        def protected_function(request):
+            return f"Access granted: {request.method}"
+
+        # request with a valid token
+        request = RequestFactory().get(
+            "/some/url", headers={"X_API_KEY": "valid-token"}
+        )
+        result = protected_function(request)
+        self.assertEqual(result, "Access granted: GET")
+
+        # request with an invalid token
+        request = RequestFactory().get(
+            "/some/url", headers={"X_API_KEY": "invalid-token"}
+        )
+        result = protected_function(request)
+        self.assertEqual(result.status_code, 403)
+
+        # request without a token
+        request = RequestFactory().get("/some/url", headers={"X_API_KEY": ""})
+        result = protected_function(request)
+        self.assertEqual(result.status_code, 403)
+
+        # request without a X_API_KEY token
+        request = RequestFactory().get("/some/url")
+        result = protected_function(request)
+        self.assertEqual(result.status_code, 403)
+
+        # request with a valid token for another API endpoint
+        request = RequestFactory().get(
+            "/some/url", headers={"X_API_KEY": "another-token"}
+        )
+        result = protected_function(request)
+        self.assertEqual(result.status_code, 403)
+
+        # requests for a misconfigured endpoint
+        @requires_api_token("ietf.api.misconfigured")
+        def another_protected_function(request):
+            return f"Access granted: {request.method}"
+
+        # request with valid token
+        request = RequestFactory().get(
+            "/some/url", headers={"X_API_KEY": "valid-token"}
+        )
+        result = another_protected_function(request)
+        self.assertEqual(result, "Access granted: GET")
+
+        # request with invalid token with the correct initial character
+        request = RequestFactory().get("/some/url", headers={"X_API_KEY": "v"})
+        result = another_protected_function(request)
+        self.assertEqual(result.status_code, 403)

From 425cbe2fa35271cd39f5836d85713d84e58aedeb Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Thu, 12 Jun 2025 07:35:23 -0700
Subject: [PATCH 349/656] feat: use new meeting.Registration model (#8983)

* refactor: use new meeting.Registration model for OidcExtraScopeClaims

* refactor: use new meeting.Registration model for nomcom eligibility calculations

* refactor: use new meeting.Registration model for various meeting views

* fix: revert to on_delete=PROTECT

* refactor: update new APIs and corresponding tests

* fix: tweaks for clarity

* fix: fix assert syntax

* fix: fix minor issue with MeetingRegistrtion migration

* fix: add missing meeting.tasks.fetch_meeting_attendance_task (#8987)

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/api/tests.py                             | 205 +++++-------
 ietf/api/views.py                             | 122 +------
 ietf/ietfauth/tests.py                        |  66 ++--
 ietf/ietfauth/utils.py                        |  20 +-
 ietf/meeting/factories.py                     |  23 +-
 .../migrations/0013_correct_reg_checkedin.py  |  41 +++
 ietf/meeting/models.py                        |  30 +-
 ietf/meeting/tasks.py                         |  19 ++
 ietf/meeting/tests_models.py                  |  34 +-
 ietf/meeting/tests_tasks.py                   |  30 ++
 ietf/meeting/tests_utils.py                   | 276 ++++++++++++++-
 ietf/meeting/tests_views.py                   |  56 ++--
 ietf/meeting/utils.py                         | 314 +++++++++++++++++-
 ietf/meeting/views.py                         |  18 +-
 ietf/nomcom/tests.py                          |  47 +--
 ietf/nomcom/utils.py                          |   6 +-
 ietf/settings.py                              |   3 +
 .../find_meetingregistration_person_issues.py |  37 ---
 .../repair_meetingregistration_person.py      |  18 -
 ietf/stats/utils.py                           |  60 +---
 .../meeting/proceedings_attendees.html        |   4 +-
 21 files changed, 951 insertions(+), 478 deletions(-)
 create mode 100644 ietf/meeting/migrations/0013_correct_reg_checkedin.py
 delete mode 100644 ietf/stats/management/commands/find_meetingregistration_person_issues.py
 delete mode 100644 ietf/stats/management/commands/repair_meetingregistration_person.py

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 809b45cc2b..32f33dcc63 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -833,111 +833,91 @@ def test_api_new_meeting_registration_nomcom_volunteer(self):
     def test_api_new_meeting_registration_v2(self):
         meeting = MeetingFactory(type_id='ietf')
         person = PersonFactory()
-        regs = [
-            {
-                'affiliation': "Alguma Corporação",
-                'country_code': 'PT',
-                'email': person.email().address,
-                'first_name': person.first_name(),
-                'last_name': person.last_name(),
-                'meeting': str(meeting.number),
-                'reg_type': 'onsite',
-                'ticket_type': 'week_pass',
-                'checkedin': False,
-                'is_nomcom_volunteer': False,
-                'cancelled': False,
-            }
-        ]
-
+        reg_detail = {
+            'email': person.email().address,
+            'first_name': person.first_name(),
+            'last_name': person.last_name(),
+            'meeting': meeting.number,
+            'affiliation': "Alguma Corporação",
+            'country_code': 'PT',
+            'checkedin': False,
+            'is_nomcom_volunteer': False,
+            'cancelled': False,
+            'tickets': [{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}],
+        }
+        reg_data = {'objects': {reg_detail['email']: reg_detail}}
         url = urlreverse('ietf.api.views.api_new_meeting_registration_v2')
         #
         # Test invalid key
-        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "invalid-token"})
+        r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "invalid-token"})
         self.assertEqual(r.status_code, 403)
         #
         # Test invalid data
-        bad_regs = copy.deepcopy(regs)
-        del(bad_regs[0]['email'])
-        r = self.client.post(url, data=json.dumps(bad_regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        bad_reg_data = copy.deepcopy(reg_data)
+        del bad_reg_data['objects'][reg_detail['email']]['email']
+        r = self.client.post(url, data=json.dumps(bad_reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"})
         self.assertEqual(r.status_code, 400)
         #
         # Test valid POST
-        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"})
         self.assertContains(r, "Success", status_code=202)
         #
         # Check record
-        reg = regs[0]
-        objects = Registration.objects.filter(email=reg['email'], meeting__number=reg['meeting'])
+        objects = Registration.objects.filter(email=reg_detail['email'], meeting__number=reg_detail['meeting'])
         self.assertEqual(objects.count(), 1)
         obj = objects[0]
         for key in ['affiliation', 'country_code', 'first_name', 'last_name', 'checkedin']:
-            self.assertEqual(getattr(obj, key), False if key=='checkedin' else reg.get(key) , "Bad data for field '%s'" % key)
+            self.assertEqual(getattr(obj, key), False if key == 'checkedin' else reg_detail.get(key), f"Bad data for field {key}")
         self.assertEqual(obj.tickets.count(), 1)
         ticket = obj.tickets.first()
-        self.assertEqual(ticket.ticket_type.slug, regs[0]['ticket_type'])
-        self.assertEqual(ticket.attendance_type.slug, regs[0]['reg_type'])
+        self.assertEqual(ticket.ticket_type.slug, reg_detail['tickets'][0]['ticket_type'])
+        self.assertEqual(ticket.attendance_type.slug, reg_detail['tickets'][0]['attendance_type'])
         self.assertEqual(obj.person, person)
         #
         # Test update (switch to remote)
-        regs = [
-            {
-                'affiliation': "Alguma Corporação",
-                'country_code': 'PT',
-                'email': person.email().address,
-                'first_name': person.first_name(),
-                'last_name': person.last_name(),
-                'meeting': str(meeting.number),
-                'reg_type': 'remote',
-                'ticket_type': 'week_pass',
-                'checkedin': False,
-                'is_nomcom_volunteer': False,
-                'cancelled': False,
-            }
-        ]
-        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        reg_detail = {
+            'affiliation': "Alguma Corporação",
+            'country_code': 'PT',
+            'email': person.email().address,
+            'first_name': person.first_name(),
+            'last_name': person.last_name(),
+            'meeting': meeting.number,
+            'checkedin': False,
+            'is_nomcom_volunteer': False,
+            'cancelled': False,
+            'tickets': [{'attendance_type': 'remote', 'ticket_type': 'week_pass'}],
+        }
+        reg_data = {'objects': {reg_detail['email']: reg_detail}}
+        r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"})
         self.assertContains(r, "Success", status_code=202)
-        objects = Registration.objects.filter(email=reg['email'], meeting__number=reg['meeting'])
+        objects = Registration.objects.filter(email=reg_detail['email'], meeting__number=reg_detail['meeting'])
         self.assertEqual(objects.count(), 1)
         obj = objects[0]
         self.assertEqual(obj.tickets.count(), 1)
         ticket = obj.tickets.first()
-        self.assertEqual(ticket.ticket_type.slug, regs[0]['ticket_type'])
-        self.assertEqual(ticket.attendance_type.slug, regs[0]['reg_type'])
+        self.assertEqual(ticket.ticket_type.slug, reg_detail['tickets'][0]['ticket_type'])
+        self.assertEqual(ticket.attendance_type.slug, reg_detail['tickets'][0]['attendance_type'])
         #
         # Test multiple
-        regs = [
-            {
-                'affiliation': "Alguma Corporação",
-                'country_code': 'PT',
-                'email': person.email().address,
-                'first_name': person.first_name(),
-                'last_name': person.last_name(),
-                'meeting': str(meeting.number),
-                'reg_type': 'onsite',
-                'ticket_type': 'one_day',
-                'checkedin': False,
-                'is_nomcom_volunteer': False,
-                'cancelled': False,
-            },
-
-            {
-                'affiliation': "Alguma Corporação",
-                'country_code': 'PT',
-                'email': person.email().address,
-                'first_name': person.first_name(),
-                'last_name': person.last_name(),
-                'meeting': str(meeting.number),
-                'reg_type': 'remote',
-                'ticket_type': 'week_pass',
-                'checkedin': False,
-                'is_nomcom_volunteer': False,
-                'cancelled': False,
-            }
-        ]
-
-        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        reg_detail = {
+            'affiliation': "Alguma Corporação",
+            'country_code': 'PT',
+            'email': person.email().address,
+            'first_name': person.first_name(),
+            'last_name': person.last_name(),
+            'meeting': meeting.number,
+            'checkedin': False,
+            'is_nomcom_volunteer': False,
+            'cancelled': False,
+            'tickets': [
+                {'attendance_type': 'onsite', 'ticket_type': 'one_day'},
+                {'attendance_type': 'remote', 'ticket_type': 'week_pass'},
+            ],
+        }
+        reg_data = {'objects': {reg_detail['email']: reg_detail}}
+        r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"})
         self.assertContains(r, "Success", status_code=202)
-        objects = Registration.objects.filter(email=reg['email'], meeting__number=reg['meeting'])
+        objects = Registration.objects.filter(email=reg_detail['email'], meeting__number=reg_detail['meeting'])
         self.assertEqual(objects.count(), 1)
         obj = objects[0]
         self.assertEqual(obj.tickets.count(), 2)
@@ -948,51 +928,46 @@ def test_api_new_meeting_registration_v2(self):
     def test_api_new_meeting_registration_v2_cancelled(self):
         meeting = MeetingFactory(type_id='ietf')
         person = PersonFactory()
-        regs = [
-            {
-                'affiliation': "Acme",
-                'country_code': 'US',
-                'email': person.email().address,
-                'first_name': person.first_name(),
-                'last_name': person.last_name(),
-                'meeting': str(meeting.number),
-                'reg_type': 'onsite',
-                'ticket_type': 'week_pass',
-                'checkedin': False,
-                'is_nomcom_volunteer': False,
-                'cancelled': False,
-            }
-        ]
+        reg_detail = {
+            'affiliation': "Acme",
+            'country_code': 'US',
+            'email': person.email().address,
+            'first_name': person.first_name(),
+            'last_name': person.last_name(),
+            'meeting': meeting.number,
+            'checkedin': False,
+            'is_nomcom_volunteer': False,
+            'cancelled': False,
+            'tickets': [{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}],
+        }
+        reg_data = {'objects': {reg_detail['email']: reg_detail}}
         url = urlreverse('ietf.api.views.api_new_meeting_registration_v2')
         self.assertEqual(Registration.objects.count(), 0)
-        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"})
         self.assertContains(r, "Success", status_code=202)
         self.assertEqual(Registration.objects.count(), 1)
-        regs[0]['cancelled'] = True
-        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        reg_detail['cancelled'] = True
+        r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"})
         self.assertContains(r, "Success", status_code=202)
         self.assertEqual(Registration.objects.count(), 0)
 
-    @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]})    
+    @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]})
     def test_api_new_meeting_registration_v2_nomcom(self):
         meeting = MeetingFactory(type_id='ietf')
         person = PersonFactory()
-        regs = [
-            {
-                'affiliation': "Acme",
-                'country_code': 'US',
-                'email': person.email().address,
-                'first_name': person.first_name(),
-                'last_name': person.last_name(),
-                'meeting': str(meeting.number),
-                'reg_type': 'onsite',
-                'ticket_type': 'week_pass',
-                'checkedin': False,
-                'is_nomcom_volunteer': False,
-                'cancelled': False,
-            }
-        ]
-
+        reg_detail = {
+            'affiliation': "Acme",
+            'country_code': 'US',
+            'email': person.email().address,
+            'first_name': person.first_name(),
+            'last_name': person.last_name(),
+            'meeting': meeting.number,
+            'checkedin': False,
+            'is_nomcom_volunteer': False,
+            'cancelled': False,
+            'tickets': [{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}],
+        }
+        reg_data = {'objects': {reg_detail['email']: reg_detail}}
         url = urlreverse('ietf.api.views.api_new_meeting_registration_v2')
         now = datetime.datetime.now()
         if now.month > 10:
@@ -1003,14 +978,14 @@ def test_api_new_meeting_registration_v2_nomcom(self):
         nomcom = NomComFactory.create(is_accepting_volunteers=True, **nomcom_kwargs_for_year(year))
 
         # first test is_nomcom_volunteer False
-        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"})
         self.assertContains(r, "Success", status_code=202)
         # assert no Volunteers exists
         self.assertEqual(Volunteer.objects.count(), 0)
 
         # test is_nomcom_volunteer True
-        regs[0]['is_nomcom_volunteer'] = True
-        r = self.client.post(url, data=json.dumps(regs), content_type='application/json', headers={"X-Api-Key": "valid-token"})
+        reg_detail['is_nomcom_volunteer'] = True
+        r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"})
         self.assertContains(r, "Success", status_code=202)
         # assert Volunteer exists
         self.assertEqual(Volunteer.objects.count(), 1)
diff --git a/ietf/api/views.py b/ietf/api/views.py
index 97b9793048..06a386b2e3 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -45,7 +45,8 @@
 from ietf.ietfauth.utils import role_required
 from ietf.ietfauth.views import send_account_creation_email
 from ietf.ipr.utils import ingest_response_email as ipr_ingest_response_email
-from ietf.meeting.models import Meeting, Registration
+from ietf.meeting.models import Meeting
+from ietf.meeting.utils import import_registration_json_validator, process_single_registration
 from ietf.nomcom.models import Volunteer, NomCom
 from ietf.nomcom.utils import ingest_feedback_email as nomcom_ingest_feedback_email
 from ietf.person.models import Person, Email
@@ -241,31 +242,6 @@ def err(code, text):
         return HttpResponse(status=405)
 
 
-_new_registration_json_validator = jsonschema.Draft202012Validator(
-    schema={
-        "type": "array",
-        "items": {
-            "type": "object",
-            "properties": {
-                "meeting": {"type": "string"},
-                "first_name": {"type": "string"},
-                "last_name": {"type": "string"},
-                "affiliation": {"type": "string"},
-                "country_code": {"type": "string"},
-                "email": {"type": "string"},
-                "reg_type": {"type": "string"},
-                "ticket_type": {"type": "string"},
-                "checkedin": {"type": "boolean"},
-                "is_nomcom_volunteer": {"type": "boolean"},
-                "cancelled": {"type": "boolean"},
-            },
-            "required": ["meeting", "first_name", "last_name", "affiliation", "country_code", "email", "reg_type", "ticket_type", "checkedin", "is_nomcom_volunteer", "cancelled"],
-            "additionalProperties": "false"
-        }
-    }
-)
-
-
 @requires_api_token
 @csrf_exempt
 def api_new_meeting_registration_v2(request):
@@ -285,7 +261,7 @@ def _api_response(result):
     # Validate
     try:
         payload = json.loads(request.body)
-        _new_registration_json_validator.validate(payload)
+        import_registration_json_validator.validate(payload)
     except json.decoder.JSONDecodeError as err:
         return _http_err(400, f"JSON parse error at line {err.lineno} col {err.colno}: {err.msg}")
     except jsonschema.exceptions.ValidationError as err:
@@ -293,91 +269,23 @@ def _api_response(result):
     except Exception:
         return _http_err(400, "Invalid request format")
 
-    # Validate consistency
-    # - if receive multiple records they should be for same meeting, same person (email)
-    if len(payload) > 1:
-        if len(set([r['meeting'] for r in payload])) != 1:
-            return _http_err(400, "Different meeting values")
-        if len(set([r['email'] for r in payload])) != 1:
-            return _http_err(400, "Different email values")
-
-    # Validate meeting
-    number = payload[0]['meeting']
+    # Get the meeting ID from the first registration, the API only deals with one meeting at a time
+    first_email = next(iter(payload['objects']))
+    meeting_number = payload['objects'][first_email]['meeting']
     try:
-        meeting = Meeting.objects.get(number=number)
+        meeting = Meeting.objects.get(number=meeting_number)
     except Meeting.DoesNotExist:
-        return _http_err(400, "Invalid meeting value: '%s'" % (number, ))
+        return _http_err(400, f"Invalid meeting value: {meeting_number}")
 
-    # Validate email
-    email = payload[0]['email']
+    # confirm email exists
     try:
-        validate_email(email)
-    except ValidationError:
-        return _http_err(400, "Invalid email value: '%s'" % (email, ))
-
-    # get person
-    person = Person.objects.filter(email__address=email).first()
-    if not person:
-        log.log(f"api_new_meeting_registration_v2 no Person found for {email}")
-
-    registration = payload[0]
-    # handle cancelled
-    if registration['cancelled']:
-        if len(payload) > 1:
-            return _http_err(400, "Error. Received cancelled registration notification with more than one record. ({})".format(email))
-        try:
-            obj = Registration.objects.get(meeting=meeting, email=email)
-        except Registration.DoesNotExist:
-            return _http_err(400, "Error. Received cancelled registration notification for non-existing registration. ({})".format(email))
-        if obj.tickets.count() == 1:
-            obj.delete()
-        else:
-            obj.tickets.filter(
-                attendance_type__slug=registration.reg_type,
-                ticket_type__slug=registration.ticket_type).delete()
-        return HttpResponse('Success', status=202, content_type='text/plain')
+        Email.objects.get(address=first_email)
+    except Email.DoesNotExist:
+        return _http_err(400, f"Unknown email: {first_email}")
 
-    # create or update MeetingRegistration
-    update_fields = ['first_name', 'last_name', 'affiliation', 'country_code', 'checkedin', 'is_nomcom_volunteer']
-    try:
-        reg = Registration.objects.get(meeting=meeting, email=email)
-        for key, value in registration.items():
-            if key in update_fields:
-                setattr(reg, key, value)
-        reg.save()
-    except Registration.DoesNotExist:
-        reg = Registration.objects.create(
-            meeting_id=meeting.pk,
-            person=person,
-            email=email,
-            first_name=registration['first_name'],
-            last_name=registration['last_name'],
-            affiliation=registration['affiliation'],
-            country_code=registration['country_code'],
-            checkedin=registration['checkedin'])
-
-    # handle registration tickets
-    reg.tickets.all().delete()
-    for registration in payload:
-        reg.tickets.create(
-            attendance_type_id=registration['reg_type'],
-            ticket_type_id=registration['ticket_type'],
-        )
-        # handle nomcom volunteer
-        if registration['is_nomcom_volunteer'] and person:
-            try:
-                nomcom = NomCom.objects.get(is_accepting_volunteers=True)
-            except (NomCom.DoesNotExist, NomCom.MultipleObjectsReturned):
-                nomcom = None
-            if nomcom:
-                Volunteer.objects.get_or_create(
-                    nomcom=nomcom,
-                    person=person,
-                    defaults={
-                        "affiliation": registration["affiliation"],
-                        "origin": "registration"
-                    }
-                )
+    reg_data = payload['objects'][first_email]
+
+    process_single_registration(reg_data, meeting)
 
     return HttpResponse('Success', status=202, content_type='text/plain')
 
diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py
index fed764e8bd..f6d7671bc9 100644
--- a/ietf/ietfauth/tests.py
+++ b/ietf/ietfauth/tests.py
@@ -32,14 +32,13 @@
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.group.models import Group, Role, RoleName
 from ietf.ietfauth.utils import has_role
-from ietf.meeting.factories import MeetingFactory
+from ietf.meeting.factories import MeetingFactory, RegistrationFactory, RegistrationTicketFactory
 from ietf.nomcom.factories import NomComFactory
 from ietf.person.factories import PersonFactory, EmailFactory, UserFactory, PersonalApiKeyFactory
 from ietf.person.models import Person, Email
 from ietf.person.tasks import send_apikey_usage_emails_task
 from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory
 from ietf.review.models import ReviewWish, UnavailablePeriod
-from ietf.stats.models import MeetingRegistration
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized
 from ietf.utils.timezone import date_today
@@ -1016,11 +1015,15 @@ def test_oidc_code_auth(self):
             EmailFactory(person=person)
             email_list = person.email_set.all().values_list('address', flat=True)
             meeting = MeetingFactory(type_id='ietf', date=date_today())
-            MeetingRegistration.objects.create(
-                    meeting=meeting, person=None, first_name=person.first_name(), last_name=person.last_name(),
-                    email=email_list[0], ticket_type='full_week', reg_type='remote', affiliation='Some Company',
-                )
-
+            reg_person = RegistrationFactory(
+                meeting=meeting,
+                person=person,
+                first_name=person.first_name(),
+                last_name=person.last_name(),
+                email=email_list[0],
+                affiliation='Some Company',
+                with_ticket={'attendance_type_id': 'remote', 'ticket_type_id': 'week_pass'},
+            )
             # Get access authorisation
             session = {}
             session["state"] = rndstr()
@@ -1073,35 +1076,48 @@ def test_oidc_code_auth(self):
             for key in ['iss', 'sub', 'aud', 'exp', 'iat', 'auth_time', 'nonce', 'at_hash']:
                 self.assertIn(key, access_token_info['id_token'])
 
-            # Get userinfo, check keys present
+            # Get userinfo, check keys present, most common scenario
             userinfo = client.do_user_info_request(state=params["state"], scope=args['scope'])
             for key in [ 'email', 'family_name', 'given_name', 'meeting', 'name', 'pronouns', 'roles',
                          'ticket_type', 'reg_type', 'affiliation', 'picture', 'dots', ]:
                 self.assertIn(key, userinfo)
                 self.assertTrue(userinfo[key])
             self.assertIn('remote', set(userinfo['reg_type'].split()))
-            self.assertNotIn('hackathon', set(userinfo['reg_type'].split()))
+            self.assertNotIn('hackathon_onsite', set(userinfo['reg_type'].split()))
             self.assertIn(active_group.acronym, [i[1] for i in userinfo['roles']])
             self.assertNotIn(closed_group.acronym, [i[1] for i in userinfo['roles']])
 
-            # Create another registration, with a different email
-            MeetingRegistration.objects.create(
-                    meeting=meeting, person=None, first_name=person.first_name(), last_name=person.last_name(),
-                    email=email_list[1], ticket_type='one_day', reg_type='hackathon', affiliation='Some Company, Inc',
-                )
+            # Create a registration, with only email, no person (rare if at all)
+            reg_person.delete()
+            reg_email = RegistrationFactory(
+                meeting=meeting,
+                person=None,
+                first_name=person.first_name(),
+                last_name=person.last_name(),
+                email=email_list[1],
+                affiliation='Some Company, Inc',
+                with_ticket={'attendance_type_id': 'hackathon_onsite', 'ticket_type_id': 'one_day'},
+            )
             userinfo = client.do_user_info_request(state=params["state"], scope=args['scope'])
-            self.assertIn('hackathon', set(userinfo['reg_type'].split()))
-            self.assertIn('remote', set(userinfo['reg_type'].split()))
-            self.assertIn('full_week', set(userinfo['ticket_type'].split()))
-            self.assertIn('Some Company', userinfo['affiliation'])
-
-            # Create a third registration, with a composite reg type
-            MeetingRegistration.objects.create(
-                    meeting=meeting, person=None, first_name=person.first_name(), last_name=person.last_name(),
-                    email=email_list[1], ticket_type='one_day', reg_type='hackathon remote', affiliation='Some Company, Inc',
-                )
+            self.assertIn('hackathon_onsite', set(userinfo['reg_type'].split()))
+            self.assertNotIn('remote', set(userinfo['reg_type'].split()))
+            self.assertIn('one_day', set(userinfo['ticket_type'].split()))
+            self.assertIn('Some Company, Inc', userinfo['affiliation'])
+
+            # Test with multiple tickets
+            reg_email.delete()
+            creg = RegistrationFactory(
+                meeting=meeting,
+                person=None,
+                first_name=person.first_name(),
+                last_name=person.last_name(),
+                email=email_list[1],
+                affiliation='Some Company, Inc',
+                with_ticket={'attendance_type_id': 'hackathon_remote', 'ticket_type_id': 'week_pass'},
+            )
+            RegistrationTicketFactory(registration=creg, attendance_type_id='remote', ticket_type_id='week_pass')
             userinfo = client.do_user_info_request(state=params["state"], scope=args['scope'])
-            self.assertEqual(set(userinfo['reg_type'].split()), set(['remote', 'hackathon']))
+            self.assertEqual(set(userinfo['reg_type'].split()), set(['remote', 'hackathon_remote']))
 
             # Check that ending a session works
             r = client.do_end_session_request(state=params["state"], scope=args['scope'])
diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py
index b4c6da14ea..efdd6f3ea6 100644
--- a/ietf/ietfauth/utils.py
+++ b/ietf/ietfauth/utils.py
@@ -346,13 +346,14 @@ def scope_pronouns(self):
         )
 
     def scope_registration(self):
+        # import here to avoid circular imports
         from ietf.meeting.helpers import get_current_ietf_meeting
-        from ietf.stats.models import MeetingRegistration
+        from ietf.meeting.models import Registration
         meeting = get_current_ietf_meeting()
         person = self.user.person
         email_list = person.email_set.values_list('address')
         q = Q(person=person, meeting=meeting) | Q(email__in=email_list, meeting=meeting)
-        regs = MeetingRegistration.objects.filter(q).distinct()
+        regs = Registration.objects.filter(q).distinct()
         for reg in regs:
             if not reg.person_id:
                 reg.person = person
@@ -363,19 +364,20 @@ def scope_registration(self):
             ticket_types = set([])
             reg_types = set([])
             for reg in regs:
-                ticket_types.add(reg.ticket_type)
-                reg_types.add(reg.reg_type)
+                for ticket in reg.tickets.all():
+                    ticket_types.add(ticket.ticket_type.slug)
+                    reg_types.add(ticket.attendance_type.slug)
             info = {
-                'meeting':      meeting.number,
+                'meeting': meeting.number,
                 # full_week, one_day, student:
-                'ticket_type':  ' '.join(ticket_types),
+                'ticket_type': ' '.join(ticket_types),
                 # onsite, remote, hackathon_onsite, hackathon_remote:
-                'reg_type':     ' '.join(reg_types),
-                'affiliation':  ([ reg.affiliation for reg in regs if reg.affiliation ] or [''])[0],
+                'reg_type': ' '.join(reg_types),
+                'affiliation': ([reg.affiliation for reg in regs if reg.affiliation] or [''])[0],
             }
 
         return info
-            
+
 def can_request_rfc_publication(user, doc):
     """Answers whether this user has an appropriate role to send this document to the RFC Editor for publication as an RFC.
 
diff --git a/ietf/meeting/factories.py b/ietf/meeting/factories.py
index b3d21830fe..fc0ce8387c 100644
--- a/ietf/meeting/factories.py
+++ b/ietf/meeting/factories.py
@@ -321,6 +321,15 @@ class Meta:
 
 
 class RegistrationFactory(factory.django.DjangoModelFactory):
+    """
+    This will create an associated onsite week_pass ticket by default.
+    Methods of calling:
+
+    RegistrationFactory()                   create a ticket with defaults, onsite
+    RegistrationFactory(with_ticket=True)   same as above
+    RegistrationFactory(with_ticket={'attendance_type_id': 'remote'})   creates ticket with overrides
+    RegistrationFactory(with_ticket=False)  does not create a ticket
+    """
     class Meta:
         model = Registration
         skip_postgeneration_save = True
@@ -335,6 +344,16 @@ class Meta:
     attended = False
     checkedin = False
 
+    @factory.post_generation
+    def with_ticket(self, create, extracted, **kwargs):
+        if not create:
+            return
+        if extracted is False:
+            # Explicitly disable ticket creation
+            return
+        ticket_kwargs = extracted if isinstance(extracted, dict) else {}
+        RegistrationTicketFactory(registration=self, **ticket_kwargs)
+
 
 class RegistrationTicketFactory(factory.django.DjangoModelFactory):
     class Meta:
@@ -342,5 +361,5 @@ class Meta:
         skip_postgeneration_save = True
 
     registration = factory.SubFactory(RegistrationFactory)
-    attendance_type_id = 'onsite'
-    ticket_type_id = 'week_pass'
+    attendance_type_id = factory.LazyAttribute(lambda _: 'onsite')
+    ticket_type_id = factory.LazyAttribute(lambda _: 'week_pass')
diff --git a/ietf/meeting/migrations/0013_correct_reg_checkedin.py b/ietf/meeting/migrations/0013_correct_reg_checkedin.py
new file mode 100644
index 0000000000..88b3efceac
--- /dev/null
+++ b/ietf/meeting/migrations/0013_correct_reg_checkedin.py
@@ -0,0 +1,41 @@
+# Generated by Django 4.2.21 on 2025-05-20 22:28
+
+'''
+The original migration had a flaw. If a participant had both a remote and onsite
+registration, which is rare but does occur, which registration the checkedin state
+came from was indeterminate. If it came from the remote registration it would be
+False which might be wrong. This migration finds all registrations with onsite tickets
+and checkedin is False, and checks if it is correct, and fixes if needed.
+'''
+
+from django.db import migrations
+import datetime
+
+
+def forward(apps, schema_editor):
+    Registration = apps.get_model('meeting', 'Registration')
+    MeetingRegistration = apps.get_model('stats', 'MeetingRegistration')
+    today = datetime.date.today()
+    for reg in Registration.objects.filter(tickets__attendance_type__slug='onsite', checkedin=False, meeting__date__lt=today).order_by('meeting__number'):
+        # get original MeetingRegistration
+        mregs = MeetingRegistration.objects.filter(meeting=reg.meeting, email=reg.email, reg_type='onsite')
+        mregs_checkedin = [mr.checkedin for mr in mregs]
+        if any(mregs_checkedin):
+            reg.checkedin = True
+            reg.save()
+            print(f'updating {reg.meeting}:{reg.email}:{reg.pk}')
+
+
+def reverse(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("meeting", "0012_registration_registrationticket"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]
diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index 003f8cd76e..cc5241efa2 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -233,9 +233,9 @@ def get_proceedings_materials(self):
         ).order_by('type__order')
 
     def get_attendance(self):
-        """Get the meeting attendance from the MeetingRegistrations
+        """Get the meeting attendance from the Registrations
 
-        Returns a NamedTuple with onsite and online attributes. Returns None if the record is unavailable
+        Returns a NamedTuple with onsite and remote attributes. Returns None if the record is unavailable
         for this meeting.
         """
         number = self.get_number()
@@ -247,10 +247,10 @@ def get_attendance(self):
         # We've separated session attendance off to ietf.meeting.Attended, but need to report attendance at older
         # meetings correctly.
 
-        attended_per_meetingregistration = (
-            Q(meetingregistration__meeting=self) & (
-                Q(meetingregistration__attended=True) |
-                Q(meetingregistration__checkedin=True)
+        attended_per_meeting_registration = (
+            Q(registration__meeting=self) & (
+                Q(registration__attended=True) |
+                Q(registration__checkedin=True)
             )
         )
         attended_per_meeting_attended = (
@@ -260,11 +260,11 @@ def get_attendance(self):
             # is good enough, just attending e.g. a training session is also good enough
         )
         attended = Person.objects.filter(
-            attended_per_meetingregistration | attended_per_meeting_attended
+            attended_per_meeting_registration | attended_per_meeting_attended
         ).distinct()
 
-        onsite=set(attended.filter(meetingregistration__meeting=self, meetingregistration__reg_type='onsite'))
-        remote=set(attended.filter(meetingregistration__meeting=self, meetingregistration__reg_type='remote'))
+        onsite = set(attended.filter(registration__meeting=self, registration__tickets__attendance_type__slug='onsite'))
+        remote = set(attended.filter(registration__meeting=self, registration__tickets__attendance_type__slug='remote'))
         remote.difference_update(onsite)
 
         return Attendance(
@@ -1487,10 +1487,10 @@ def __str__(self):
 
 class RegistrationManager(models.Manager):
     def onsite(self):
-        return self.get_queryset().filter(registrationticket__attendance_type__slug='onsite')
+        return self.get_queryset().filter(tickets__attendance_type__slug='onsite')
 
     def remote(self):
-        return self.get_queryset().filter(registrationticket__attendance_type__slug='remote').exclude(registrationticket__attendance_type__slug='onsite')
+        return self.get_queryset().filter(tickets__attendance_type__slug='remote').exclude(tickets__attendance_type__slug='onsite')
 
 class Registration(models.Model):
     """Registration attendee records from the IETF registration system"""
@@ -1513,6 +1513,14 @@ class Registration(models.Model):
     def __str__(self):
         return "{} {}".format(self.first_name, self.last_name)
 
+    @property
+    def attendance_type(self):
+        if self.tickets.filter(attendance_type__slug='onsite').exists():
+            return 'onsite'
+        elif self.tickets.filter(attendance_type__slug='remote').exists():
+            return 'remote'
+        return None
+
 class RegistrationTicket(models.Model):
     registration = ForeignKey(Registration, related_name='tickets')
     attendance_type = ForeignKey(AttendanceTypeName, on_delete=models.PROTECT)
diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py
index 20bb602682..dc3fbc99ec 100644
--- a/ietf/meeting/tasks.py
+++ b/ietf/meeting/tasks.py
@@ -10,6 +10,7 @@
 from .utils import generate_proceedings_content
 from .views import generate_agenda_data
 from .utils import migrate_registrations, check_migrate_registrations
+from .utils import fetch_attendance_from_meetings
 
 
 @shared_task
@@ -60,3 +61,21 @@ def proceedings_content_refresh_task(*, all=False):
         elif all or (num % 24 == now.hour):
             log.log(f"Refreshing proceedings for meeting {meeting.number}...")
             generate_proceedings_content(meeting, force_refresh=True)
+
+
+@shared_task
+def fetch_meeting_attendance_task():
+    # fetch most recent two meetings
+    meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by("-date")[:2]
+    try:
+        stats = fetch_attendance_from_meetings(meetings)
+    except RuntimeError as err:
+        log.log(f"Error in fetch_meeting_attendance_task: {err}")
+    else:
+        for meeting, meeting_stats in zip(meetings, stats):
+            log.log(
+                "Fetched data for meeting {:>3}: {:4d} created, {:4d} updated, {:4d} deleted, {:4d} processed".format(
+                    meeting.number, meeting_stats['created'], meeting_stats['updated'], meeting_stats['deleted'],
+                    meeting_stats['processed']
+                )
+            )
diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py
index 03b706e1d7..e333ddad9a 100644
--- a/ietf/meeting/tests_models.py
+++ b/ietf/meeting/tests_models.py
@@ -11,8 +11,8 @@
 import ietf.meeting.models
 from ietf.group.factories import GroupFactory, GroupHistoryFactory
 from ietf.meeting.factories import MeetingFactory, SessionFactory, AttendedFactory, SessionPresentationFactory
+from ietf.meeting.factories import RegistrationFactory
 from ietf.meeting.models import Session
-from ietf.stats.factories import MeetingRegistrationFactory
 from ietf.utils.test_utils import TestCase
 from ietf.utils.timezone import date_today, datetime_today
 
@@ -21,9 +21,9 @@ class MeetingTests(TestCase):
     def test_get_attendance_pre110(self):
         """Pre-110 meetings do not calculate attendance"""
         meeting = MeetingFactory(type_id='ietf', number='109')
-        MeetingRegistrationFactory.create_batch(3, meeting=meeting, reg_type='')
-        MeetingRegistrationFactory.create_batch(4, meeting=meeting, reg_type='remote')
-        MeetingRegistrationFactory.create_batch(5, meeting=meeting, reg_type='in_person')
+        RegistrationFactory.create_batch(3, meeting=meeting, with_ticket={'attendance_type_id': 'unknown'})
+        RegistrationFactory.create_batch(4, meeting=meeting, with_ticket={'attendance_type_id': 'remote'})
+        RegistrationFactory.create_batch(5, meeting=meeting, with_ticket={'attendance_type_id': 'onsite'})
         self.assertIsNone(meeting.get_attendance())
 
     def test_get_attendance_110(self):
@@ -31,31 +31,31 @@ def test_get_attendance_110(self):
         meeting = MeetingFactory(type_id='ietf', number='110')
 
         # start with attendees that should be ignored
-        MeetingRegistrationFactory.create_batch(3, meeting=meeting, reg_type='', attended=True)
-        MeetingRegistrationFactory(meeting=meeting, reg_type='', attended=False)
+        RegistrationFactory.create_batch(3, meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}, attended=True)
+        RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}, attended=False)
         attendance = meeting.get_attendance()
         self.assertIsNotNone(attendance)
         self.assertEqual(attendance.remote, 0)
         self.assertEqual(attendance.onsite, 0)
 
         # add online attendees with at least one who registered but did not attend
-        MeetingRegistrationFactory.create_batch(4, meeting=meeting, reg_type='remote', attended=True)
-        MeetingRegistrationFactory(meeting=meeting, reg_type='remote', attended=False)
+        RegistrationFactory.create_batch(4, meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=True)
+        RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=False)
         attendance = meeting.get_attendance()
         self.assertIsNotNone(attendance)
         self.assertEqual(attendance.remote, 4)
         self.assertEqual(attendance.onsite, 0)
 
         # and the same for onsite attendees
-        MeetingRegistrationFactory.create_batch(5, meeting=meeting, reg_type='onsite', attended=True)
-        MeetingRegistrationFactory(meeting=meeting, reg_type='in_person', attended=False)
+        RegistrationFactory.create_batch(5, meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=True)
+        RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False)
         attendance = meeting.get_attendance()
         self.assertIsNotNone(attendance)
         self.assertEqual(attendance.remote, 4)
         self.assertEqual(attendance.onsite, 5)
 
         # and once more after removing all the online attendees
-        meeting.meetingregistration_set.filter(reg_type='remote').delete()
+        meeting.registration_set.remote().delete()
         attendance = meeting.get_attendance()
         self.assertIsNotNone(attendance)
         self.assertEqual(attendance.remote, 0)
@@ -64,11 +64,11 @@ def test_get_attendance_110(self):
     def test_get_attendance_113(self):
         """Simulate IETF 113 attendance gathering data"""
         meeting = MeetingFactory(type_id='ietf', number='113')
-        MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', attended=True, checkedin=False)
-        MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', attended=False, checkedin=True)
-        p1 = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', attended=False, checkedin=False).person
+        RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=True, checkedin=False)
+        RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=True)
+        p1 = RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=False).person
         AttendedFactory(session__meeting=meeting, person=p1)
-        p2 = MeetingRegistrationFactory(meeting=meeting, reg_type='remote', attended=False, checkedin=False).person
+        p2 = RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=False, checkedin=False).person
         AttendedFactory(session__meeting=meeting, person=p2)
         attendance = meeting.get_attendance()
         self.assertEqual(attendance.onsite, 3)
@@ -82,9 +82,9 @@ def test_get_attendance_keeps_meetings_distinct(self):
 
         # Create a person who attended a remote session for first_mtg and onsite for second_mtg without
         # checking in for either.
-        p = MeetingRegistrationFactory(meeting=second_mtg, reg_type='onsite', attended=False, checkedin=False).person
+        p = RegistrationFactory(meeting=second_mtg, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=False).person
         AttendedFactory(session__meeting=first_mtg, person=p)
-        MeetingRegistrationFactory(meeting=first_mtg, person=p, reg_type='remote', attended=False, checkedin=False)
+        RegistrationFactory(meeting=first_mtg, person=p, with_ticket={'attendance_type_id': 'remote'}, attended=False, checkedin=False)
         AttendedFactory(session__meeting=second_mtg, person=p)
 
         att = first_mtg.get_attendance()
diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py
index c026a99835..66de212899 100644
--- a/ietf/meeting/tests_tasks.py
+++ b/ietf/meeting/tests_tasks.py
@@ -3,8 +3,10 @@
 import datetime
 from mock import patch, call
 from ietf.utils.test_utils import TestCase
+from ietf.utils.timezone import date_today
 from .factories import MeetingFactory
 from .tasks import proceedings_content_refresh_task, agenda_data_refresh
+from .tasks import fetch_meeting_attendance_task
 
 
 class TaskTests(TestCase):
@@ -49,3 +51,31 @@ def test_proceedings_content_refresh_task(self, mock_generate):
         with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc):
             proceedings_content_refresh_task(all=True)
         self.assertEqual(mock_generate.call_count, 2)
+
+    @patch("ietf.meeting.tasks.fetch_attendance_from_meetings")
+    def test_fetch_meeting_attendance_task(self, mock_fetch_attendance):
+        today = date_today()
+        meetings = [
+            MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=1)),
+            MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=2)),
+            MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=3)),
+        ]
+        data = {
+            'created': 1,
+            'updated': 2,
+            'deleted': 0,
+            'processed': 3,
+        }
+
+        mock_fetch_attendance.return_value = [data, data]
+
+        fetch_meeting_attendance_task()
+        self.assertEqual(mock_fetch_attendance.call_count, 1)
+        self.assertCountEqual(mock_fetch_attendance.call_args[0][0], meetings[0:2])
+
+        # test handling of RuntimeError
+        mock_fetch_attendance.reset_mock()
+        mock_fetch_attendance.side_effect = RuntimeError
+        fetch_meeting_attendance_task()
+        self.assertTrue(mock_fetch_attendance.called)
+        # Good enough that we got here without raising an exception
diff --git a/ietf/meeting/tests_utils.py b/ietf/meeting/tests_utils.py
index 4bda3a65fa..8d912158ce 100644
--- a/ietf/meeting/tests_utils.py
+++ b/ietf/meeting/tests_utils.py
@@ -1,11 +1,22 @@
 # Copyright The IETF Trust 2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
+import copy
 import datetime
 import debug  # pyflakes: ignore
-from ietf.meeting.factories import MeetingFactory # RegistrationFactory, RegistrationTicketFactory
+import json
+import jsonschema
+from json import JSONDecodeError
+from mock import patch, Mock
+
+from django.http import HttpResponse, JsonResponse
+from ietf.meeting.factories import MeetingFactory, RegistrationFactory, RegistrationTicketFactory
 from ietf.meeting.models import Registration
-from ietf.meeting.utils import migrate_registrations, get_preferred
+from ietf.meeting.utils import (migrate_registrations, get_preferred, process_single_registration,
+    get_registration_data, sync_registration_data, fetch_attendance_from_meetings)
+from ietf.nomcom.models import Volunteer
+from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year
+from ietf.person.factories import PersonFactory
 from ietf.stats.factories import MeetingRegistrationFactory
 from ietf.utils.test_utils import TestCase
 
@@ -72,7 +83,7 @@ def test_additional_ticket(self):
         migrate_registrations(initial=True)
         # new.refresh_from_db()
         self.assertEqual(new.tickets.count(), 2)
-        
+
     def test_cancelled_registration(self):
         # setup test initial conditions
         meeting = MeetingFactory(type_id='ietf', number='109')
@@ -82,10 +93,265 @@ def test_cancelled_registration(self):
         # do test
         migrate_registrations(initial=True)
         self.assertEqual(Registration.objects.count(), 0)
-    
+
     def test_get_preferred(self):
         meeting = MeetingFactory(type_id='ietf', number='109')
         onsite = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
+        remote = MeetingRegistrationFactory(meeting=meeting, reg_type='remote', ticket_type='week_pass')
         hackathon = MeetingRegistrationFactory(meeting=meeting, reg_type='hackathon_onsite', ticket_type='week_pass')
-        result = get_preferred([onsite, hackathon])
+        result = get_preferred([remote, onsite, hackathon])
         self.assertEqual(result, onsite)
+        result = get_preferred([hackathon, remote])
+        self.assertEqual(result, remote)
+        result = get_preferred([hackathon])
+        self.assertEqual(result, hackathon)
+
+
+class JsonResponseWithJson(JsonResponse):
+    def json(self):
+        return json.loads(self.content)
+
+
+class GetRegistrationsTests(TestCase):
+
+    @patch('ietf.meeting.utils.requests.get')
+    def test_get_registation_data(self, mock_get):
+        meeting = MeetingFactory(type_id='ietf', number='122')
+        person = PersonFactory()
+        reg_details = dict(
+            first_name=person.first_name(),
+            last_name=person.last_name(),
+            email=person.email().address,
+            affiliation='Microsoft',
+            country_code='US',
+            meeting=meeting.number,
+            checkedin=True,
+            is_nomcom_volunteer=False,
+            cancelled=False,
+            tickets=[{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}],
+        )
+        reg_data = {'objects': {person.email().address: reg_details}}
+        reg_data_bad = copy.deepcopy(reg_data)
+        del reg_data_bad['objects'][person.email().address]['email']
+        response1 = HttpResponse('Invalid apikey', status=403)
+        response2 = JsonResponseWithJson(reg_data)
+        response3 = Mock()
+        response3.status_code = 200
+        response3.json.side_effect = JSONDecodeError("Expecting value", doc="", pos=0)
+        response4 = JsonResponseWithJson(reg_data_bad)
+        mock_get.side_effect = [response1, response2, response3, response4]
+        # test status 403
+        with self.assertRaises(Exception):
+            get_registration_data(meeting)
+        # test status 200 good
+        returned_data = get_registration_data(meeting)
+        self.assertEqual(returned_data, reg_data)
+        # test decode error
+        with self.assertRaises(ValueError):
+            get_registration_data(meeting)
+        # test validation error
+        with self.assertRaises(jsonschema.exceptions.ValidationError):
+            get_registration_data(meeting)
+
+    @patch('ietf.meeting.utils.get_registration_data')
+    def test_sync_registation_data(self, mock_get):
+        meeting = MeetingFactory(type_id='ietf', number='122')
+        person1 = PersonFactory()
+        person2 = PersonFactory()
+        items = []
+        for person in [person1, person2]:
+            items.append(dict(
+                first_name=person.first_name(),
+                last_name=person.last_name(),
+                email=person.email().address,
+                affiliation='Microsoft',
+                country_code='US',
+                meeting=meeting.number,
+                checkedin=True,
+                is_nomcom_volunteer=False,
+                cancelled=False,
+                tickets=[{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}],
+            ))
+        reg_data = {'objects': {items[0]['email']: items[0], items[1]['email']: items[1]}}
+        mock_get.return_value = reg_data
+        self.assertEqual(Registration.objects.filter(meeting=meeting).count(), 0)
+        stats = sync_registration_data(meeting)
+        self.assertEqual(Registration.objects.filter(meeting=meeting).count(), 2)
+        self.assertEqual(stats['created'], 2)
+        # test idempotent
+        stats = sync_registration_data(meeting)
+        self.assertEqual(Registration.objects.filter(meeting=meeting).count(), 2)
+        self.assertEqual(stats['created'], 0)
+        # test delete cancelled registration
+        del reg_data['objects'][items[1]['email']]
+        stats = sync_registration_data(meeting)
+        self.assertEqual(Registration.objects.filter(meeting=meeting).count(), 1)
+        self.assertEqual(stats['deleted'], 1)
+
+    def test_process_single_registration(self):
+        # test new registration
+        meeting = MeetingFactory(type_id='ietf', number='122')
+        person = PersonFactory()
+        reg_data = dict(
+            first_name=person.first_name(),
+            last_name=person.last_name(),
+            email=person.email().address,
+            affiliation='Microsoft',
+            country_code='US',
+            meeting=meeting.number,
+            checkedin=True,
+            is_nomcom_volunteer=False,
+            cancelled=False,
+            tickets=[{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}],
+        )
+        self.assertEqual(meeting.registration_set.count(), 0)
+        new_reg, action = process_single_registration(reg_data, meeting)
+        self.assertEqual(meeting.registration_set.count(), 1)
+        reg = meeting.registration_set.first()
+        self.assertEqual(new_reg, reg)
+        self.assertEqual(action, 'created')
+        self.assertEqual(reg.first_name, person.first_name())
+        self.assertEqual(reg.last_name, person.last_name())
+        self.assertEqual(reg.email, person.email().address)
+        self.assertEqual(reg.affiliation, 'Microsoft')
+        self.assertEqual(reg.meeting, meeting)
+        self.assertEqual(reg.checkedin, True)
+        self.assertEqual(reg.tickets.count(), 1)
+        ticket = reg.tickets.first()
+        self.assertEqual(ticket.attendance_type.slug, 'onsite')
+        self.assertEqual(ticket.ticket_type.slug, 'week_pass')
+
+        # test no change
+        new_reg, action = process_single_registration(reg_data, meeting)
+        self.assertEqual(meeting.registration_set.count(), 1)
+        reg = meeting.registration_set.first()
+        self.assertEqual(new_reg, reg)
+        self.assertEqual(action, None)
+
+        # test update fields
+        reg_data['affiliation'] = 'Cisco'
+        new_reg, action = process_single_registration(reg_data, meeting)
+        self.assertEqual(meeting.registration_set.count(), 1)
+        reg = meeting.registration_set.first()
+        self.assertEqual(new_reg, reg)
+        self.assertEqual(action, 'updated')
+        self.assertEqual(reg.affiliation, 'Cisco')
+
+        # test update tickets
+        reg_data['tickets'] = [{'attendance_type': 'remote', 'ticket_type': 'week_pass'}]
+        new_reg, action = process_single_registration(reg_data, meeting)
+        self.assertEqual(meeting.registration_set.count(), 1)
+        reg = meeting.registration_set.first()
+        self.assertEqual(new_reg, reg)
+        self.assertEqual(action, 'updated')
+        self.assertEqual(reg.tickets.count(), 1)
+        ticket = reg.tickets.first()
+        self.assertEqual(ticket.attendance_type.slug, 'remote')
+
+        # test tickets, two of same
+        reg_data['tickets'] = [
+            {'attendance_type': 'onsite', 'ticket_type': 'one_day'},
+            {'attendance_type': 'onsite', 'ticket_type': 'one_day'},
+            {'attendance_type': 'remote', 'ticket_type': 'week_pass'},
+        ]
+        new_reg, action = process_single_registration(reg_data, meeting)
+        self.assertEqual(meeting.registration_set.count(), 1)
+        reg = meeting.registration_set.first()
+        self.assertEqual(new_reg, reg)
+        self.assertEqual(action, 'updated')
+        self.assertEqual(reg.tickets.count(), 3)
+        self.assertEqual(reg.tickets.filter(attendance_type__slug='onsite', ticket_type__slug='one_day').count(), 2)
+        self.assertEqual(reg.tickets.filter(attendance_type__slug='remote', ticket_type__slug='week_pass').count(), 1)
+
+        # test tickets, two of same, delete one
+        reg_data['tickets'] = [
+            {'attendance_type': 'onsite', 'ticket_type': 'one_day'},
+            {'attendance_type': 'remote', 'ticket_type': 'week_pass'},
+        ]
+        new_reg, action = process_single_registration(reg_data, meeting)
+        self.assertEqual(meeting.registration_set.count(), 1)
+        reg = meeting.registration_set.first()
+        self.assertEqual(new_reg, reg)
+        self.assertEqual(action, 'updated')
+        self.assertEqual(reg.tickets.count(), 2)
+        self.assertEqual(reg.tickets.filter(attendance_type__slug='onsite', ticket_type__slug='one_day').count(), 1)
+        self.assertEqual(reg.tickets.filter(attendance_type__slug='remote', ticket_type__slug='week_pass').count(), 1)
+
+    def test_process_single_registration_nomcom(self):
+        '''Test that Volunteer is created if is_nomcom_volunteer=True'''
+        meeting = MeetingFactory(type_id='ietf', number='122')
+        person = PersonFactory()
+        reg_data = dict(
+            first_name=person.first_name(),
+            last_name=person.last_name(),
+            email=person.email().address,
+            affiliation='Microsoft',
+            country_code='US',
+            meeting=meeting.number,
+            checkedin=True,
+            is_nomcom_volunteer=True,
+            cancelled=False,
+            tickets=[{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}],
+        )
+        now = datetime.datetime.now()
+        if now.month > 10:
+            year = now.year + 1
+        else:
+            year = now.year
+        # create appropriate group and nomcom objects
+        nomcom = NomComFactory.create(is_accepting_volunteers=True, **nomcom_kwargs_for_year(year))
+        # assert no Volunteers exists
+        self.assertEqual(Volunteer.objects.count(), 0)
+        new_reg, action = process_single_registration(reg_data, meeting)
+        self.assertEqual(action, 'created')
+        # assert Volunteer exists
+        self.assertEqual(Volunteer.objects.count(), 1)
+        volunteer = Volunteer.objects.last()
+        self.assertEqual(volunteer.person, person)
+        self.assertEqual(volunteer.nomcom, nomcom)
+        self.assertEqual(volunteer.origin, 'registration')
+
+    def test_process_single_registration_cancelled(self):
+        # test cancelled registration, one of two tickets
+        meeting = MeetingFactory(type_id='ietf', number='122')
+        person = PersonFactory()
+        reg = RegistrationFactory(meeting=meeting, person=person, checkedin=False, with_ticket={'attendance_type_id': 'onsite'})
+        RegistrationTicketFactory(registration=reg, attendance_type_id='remote', ticket_type_id='week_pass')
+        reg_data = dict(
+            first_name=person.first_name(),
+            last_name=person.last_name(),
+            email=person.email().address,
+            affiliation='Microsoft',
+            country_code='US',
+            meeting=meeting.number,
+            checkedin=False,
+            is_nomcom_volunteer=False,
+            cancelled=True,
+            tickets=[{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}],
+        )
+        self.assertEqual(meeting.registration_set.count(), 1)
+        self.assertEqual(reg.tickets.count(), 2)
+        new_reg, action = process_single_registration(reg_data, meeting)
+        self.assertEqual((new_reg, action), (None, 'deleted'))
+        self.assertEqual(meeting.registration_set.count(), 1)
+        self.assertEqual(reg.tickets.count(), 1)
+        self.assertTrue(reg.tickets.filter(attendance_type__slug='remote').exists())
+        # test cancelled registration, last ticket
+        reg_data['tickets'][0]['attendance_type'] = 'remote'
+        new_reg, action = process_single_registration(reg_data, meeting)
+        self.assertEqual((new_reg, action), (None, 'deleted'))
+        self.assertEqual(meeting.registration_set.count(), 0)
+
+    @patch("ietf.meeting.utils.sync_registration_data")
+    def test_fetch_attendance_from_meetings(self, mock_sync_reg_data):
+        mock_meetings = [object(), object(), object()]
+        d1 = dict(created=1, updated=2, deleted=0, processed=3)
+        d2 = dict(created=2, updated=2, deleted=0, processed=4)
+        d3 = dict(created=1, updated=4, deleted=1, processed=5)
+        mock_sync_reg_data.side_effect = (d1, d2, d3)
+        stats = fetch_attendance_from_meetings(mock_meetings)
+        self.assertEqual(
+            [mock_sync_reg_data.call_args_list[n][0][0] for n in range(3)],
+            mock_meetings,
+        )
+        self.assertEqual(stats, [d1, d2, d3])
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 889a1bc882..1aac2a6523 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -66,8 +66,7 @@
     SessionPresentationFactory, MeetingFactory, FloorPlanFactory,
     TimeSlotFactory, SlideSubmissionFactory, RoomFactory,
     ConstraintFactory, MeetingHostFactory, ProceedingsMaterialFactory,
-    AttendedFactory)
-from ietf.stats.factories import MeetingRegistrationFactory
+    AttendedFactory, RegistrationFactory)
 from ietf.doc.factories import DocumentFactory, WgDraftFactory
 from ietf.submit.tests import submission_file
 from ietf.utils.test_utils import assert_ical_response_is_valid
@@ -8852,25 +8851,24 @@ def test_proceedings_attendees(self):
            - prefer onsite checkedin=True to remote attended when same person has both
         """
 
-        meeting = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number="118")
+        m = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number="118")
         person_a = PersonFactory(name='Person A')
         person_b = PersonFactory(name='Person B')
         person_c = PersonFactory(name='Person C')
         person_d = PersonFactory(name='Person D')
-        MeetingRegistrationFactory(meeting=meeting, person=person_a, reg_type='onsite', checkedin=True)
-        MeetingRegistrationFactory(meeting=meeting, person=person_b, reg_type='onsite', checkedin=False)
-        MeetingRegistrationFactory(meeting=meeting, person=person_a, reg_type='remote')
-        AttendedFactory(session__meeting=meeting, session__type_id='plenary', person=person_a)
-        MeetingRegistrationFactory(meeting=meeting, person=person_c, reg_type='remote')
-        AttendedFactory(session__meeting=meeting, session__type_id='plenary', person=person_c)
-        MeetingRegistrationFactory(meeting=meeting, person=person_d, reg_type='remote')
+        areg = RegistrationFactory(meeting=m, person=person_a, checkedin=True, with_ticket={'attendance_type_id': 'onsite'})
+        RegistrationFactory(meeting=m, person=person_b, checkedin=False, with_ticket={'attendance_type_id': 'onsite'})
+        creg = RegistrationFactory(meeting=m, person=person_c, with_ticket={'attendance_type_id': 'remote'})
+        RegistrationFactory(meeting=m, person=person_d, with_ticket={'attendance_type_id': 'remote'})
+        AttendedFactory(session__meeting=m, session__type_id='plenary', person=person_a)
+        AttendedFactory(session__meeting=m, session__type_id='plenary', person=person_c)
         url = urlreverse('ietf.meeting.views.proceedings_attendees',kwargs={'num': 118})
         response = self.client.get(url)
         self.assertContains(response, 'Attendee list')
         q = PyQuery(response.content)
         self.assertEqual(2, len(q("#id_attendees tbody tr")))
         text = q('#id_attendees tbody tr').text().replace('\n', ' ')
-        self.assertEqual(text, "A Person onsite C Person remote")
+        self.assertEqual(text, f"A Person {areg.affiliation} {areg.country_code} onsite C Person {creg.affiliation} {creg.country_code} remote")
 
     def test_proceedings_overview(self):
         '''Test proceedings IETF Overview page.
@@ -9271,27 +9269,25 @@ def test_get_next_sequence(self):
         self.assertEqual(sequence,1)
 
     def test_participants_for_meeting(self):
-        person_a = PersonFactory()
-        person_b = PersonFactory()
-        person_c = PersonFactory()
-        person_d = PersonFactory()
         m = MeetingFactory.create(type_id='ietf')
-        MeetingRegistrationFactory(meeting=m, person=person_a, reg_type='onsite', checkedin=True)
-        MeetingRegistrationFactory(meeting=m, person=person_b, reg_type='onsite', checkedin=False)
-        MeetingRegistrationFactory(meeting=m, person=person_c, reg_type='remote')
-        MeetingRegistrationFactory(meeting=m, person=person_d, reg_type='remote')
-        AttendedFactory(session__meeting=m, session__type_id='plenary', person=person_c)
+        areg = RegistrationFactory(meeting=m, checkedin=True, with_ticket={'attendance_type_id': 'onsite'})
+        breg = RegistrationFactory(meeting=m, checkedin=False, with_ticket={'attendance_type_id': 'onsite'})
+        creg = RegistrationFactory(meeting=m, with_ticket={'attendance_type_id': 'remote'})
+        dreg = RegistrationFactory(meeting=m, with_ticket={'attendance_type_id': 'remote'})
+        AttendedFactory(session__meeting=m, session__type_id='plenary', person=creg.person)
         checked_in, attended = participants_for_meeting(m)
-        self.assertTrue(person_a.pk in checked_in)
-        self.assertTrue(person_b.pk not in checked_in)
-        self.assertTrue(person_c.pk in attended)
-        self.assertTrue(person_d.pk not in attended)
+        self.assertIn(areg.person.pk, checked_in)
+        self.assertNotIn(breg.person.pk, checked_in)
+        self.assertNotIn(areg.person.pk, attended)
+        self.assertNotIn(breg.person.pk, attended)
+        self.assertIn(creg.person.pk, attended)
+        self.assertNotIn(dreg.person.pk, attended)
 
     def test_session_attendance(self):
         meeting = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number='118')
         make_meeting_test_data(meeting=meeting)
         session = Session.objects.filter(meeting=meeting, group__acronym='mars').first()
-        regs = MeetingRegistrationFactory.create_batch(3, meeting=meeting)
+        regs = RegistrationFactory.create_batch(3, meeting=meeting)
         persons = [reg.person for reg in regs]
         self.assertEqual(session.attended_set.count(), 0)
 
@@ -9337,7 +9333,7 @@ def _test_button(person, expected):
         # person0 is already on the bluesheet
         _test_button(persons[0], False)
         # person3 attests he was there
-        persons.append(MeetingRegistrationFactory(meeting=meeting).person)
+        persons.append(RegistrationFactory(meeting=meeting).person)
         # button isn't shown if we're outside the corrections windows
         meeting.importantdate_set.create(name_id='revsub',date=date_today() - datetime.timedelta(days=20))
         _test_button(persons[3], False)
@@ -9395,12 +9391,12 @@ def _test_button(person, expected):
 
     def test_bluesheet_data(self):
         session = SessionFactory(meeting__type_id="ietf") 
-        attended_with_affil = MeetingRegistrationFactory(meeting=session.meeting, affiliation="Somewhere")
+        attended_with_affil = RegistrationFactory(meeting=session.meeting, affiliation="Somewhere")
         AttendedFactory(session=session, person=attended_with_affil.person, time="2023-03-13T01:24:00Z")  # joined 2nd
-        attended_no_affil = MeetingRegistrationFactory(meeting=session.meeting)
+        attended_no_affil = RegistrationFactory(meeting=session.meeting, affiliation="")
         AttendedFactory(session=session, person=attended_no_affil.person, time="2023-03-13T01:23:00Z")  # joined 1st
-        MeetingRegistrationFactory(meeting=session.meeting)  # did not attend
-        
+        RegistrationFactory(meeting=session.meeting)  # did not attend
+
         data = bluesheet_data(session)
         self.assertEqual(
             data,
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index af12b6a274..db67f79b93 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import datetime
 import itertools
+import jsonschema
 import os
+import requests
 from hashlib import sha384
 
 import pytz
@@ -15,6 +17,7 @@
 from django.contrib import messages
 from django.core.cache import caches
 from django.core.files.base import ContentFile
+from django.db import IntegrityError
 from django.db.models import OuterRef, Subquery, TextField, Q, Value, Max
 from django.db.models.functions import Coalesce
 from django.template.loader import render_to_string
@@ -27,7 +30,7 @@
 from ietf.doc.storage_utils import store_bytes, store_str
 from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot,
     Constraint, SchedTimeSessAssignment, SessionPresentation, Attended,
-    Registration, Meeting)
+    Registration, Meeting, RegistrationTicket)
 from ietf.doc.models import Document, State, NewRevisionDocEvent, StateDocEvent
 from ietf.doc.models import DocEvent
 from ietf.group.models import Group
@@ -160,7 +163,7 @@ def bluesheet_data(session):
         .annotate(
             affiliation=Coalesce(
                 Subquery(
-                    MeetingRegistration.objects.filter(
+                    Registration.objects.filter(
                         Q(meeting=session.meeting),
                         Q(person=OuterRef("person")) | Q(email=OuterRef("person__email")),
                     ).values("affiliation")[:1]
@@ -1008,20 +1011,25 @@ def participants_for_meeting(meeting):
         checked_in = queryset of onsite, checkedin participants values_list('person')
         attended = queryset of remote participants who attended a session values_list('person')
     """
-    checked_in = meeting.meetingregistration_set.filter(reg_type='onsite', checkedin=True).values_list('person', flat=True).distinct()
+    checked_in = meeting.registration_set.onsite().filter(checkedin=True).values_list('person', flat=True).distinct()
     sessions = meeting.session_set.filter(Q(type='plenary') | Q(group__type__in=['wg', 'rg']))
     attended = Attended.objects.filter(session__in=sessions).values_list('person', flat=True).distinct()
     return (checked_in, attended)
 
 
 def get_preferred(regs):
-    """ Return a preferred regular registration (non hackathon) from 
-        a list of registrations if there is one, otherwise any.
+    """ If there are multiple registrations return preferred in
+        this order: onsite, remote, any (ie hackathon_onsite)
     """
-    for reg in regs:
-        if reg.reg_type in ['onsite', 'remote']:
-            return reg
-    return reg
+    if len(regs) == 1:
+        return regs[0]
+    reg_types = [r.reg_type for r in regs]
+    if 'onsite' in reg_types:
+        return regs[reg_types.index('onsite')]
+    elif 'remote' in reg_types:
+        return regs[reg_types.index('remote')]
+    else:
+        return regs[0]
 
 
 def migrate_registrations(initial=False):
@@ -1287,3 +1295,291 @@ def _format_materials(items):
             else:
                 not_meeting_groups.append(entry)
     return meeting_groups, not_meeting_groups
+
+
+import_registration_json_validator = jsonschema.Draft202012Validator(
+    schema={
+        "type": "object",
+        "properties": {
+            "objects": {
+                "type": "object",
+                "patternProperties": {
+                    # Email address as key (simplified pattern or just allow any key)
+                    ".*": {
+                        "type": "object",
+                        "properties": {
+                            "first_name": {"type": "string"},
+                            "last_name": {"type": "string"},
+                            "email": {"type": "string", "format": "email"},
+                            "affiliation": {"type": "string"},
+                            "country_code": {"type": "string", "minLength": 2, "maxLength": 2},
+                            "meeting": {"type": "string"},
+                            "checkedin": {"type": "boolean"},
+                            "cancelled": {"type": "boolean"},
+                            "is_nomcom_volunteer": {"type": "boolean"},
+                            "tickets": {
+                                "type": "array",
+                                "items": {
+                                    "type": "object",
+                                    "properties": {
+                                        "attendance_type": {"type": "string"},
+                                        "ticket_type": {"type": "string"}
+                                    },
+                                    "required": ["attendance_type", "ticket_type"]
+                                }
+                            }
+                        },
+                        "required": [
+                            "first_name", "last_name", "email",
+                            "country_code", "meeting", 'affiliation',
+                            "checkedin", "is_nomcom_volunteer", "tickets",
+                            "cancelled",
+                        ]
+                    }
+                },
+                "additionalProperties": False
+            }
+        },
+        "required": ["objects"]
+    }
+)
+
+
+def get_registration_data(meeting):
+    '''Retrieve data from registation system for meeting'''
+    url = settings.REGISTRATION_PARTICIPANTS_API_URL
+    key = settings.REGISTRATION_PARTICIPANTS_API_KEY
+    params = {'meeting': meeting.number, 'apikey': key}
+    try:
+        response = requests.get(url, params=params, timeout=settings.DEFAULT_REQUESTS_TIMEOUT)
+    except requests.Timeout as e:
+        log(f'GET request timed out for [{url}]: {e}')
+        raise Exception("Timeout retrieving data from registration API") from e
+    if response.status_code == 200:
+        try:
+            decoded = response.json()
+        except ValueError as e:
+            raise ValueError(f'Could not decode response from registration API: {e}')
+    else:
+        raise Exception(f'Bad response from registration API: {response.status_code}, {response.content[:64]}')
+
+    # validate registration data
+    import_registration_json_validator.validate(decoded)
+    return decoded
+
+
+def sync_registration_data(meeting):
+    """"Sync meeting.Registration with registration system.
+
+    Registration records are created in realtime as people register for a
+    meeting. This function serves as an audit / reconciliation. Most records are
+    expected to already exist. The function has been optimized with this in mind.
+
+    - Creates new registrations if they don't exist
+    - Updates existing registrations if fields differ
+    - Updates tickets as needed
+    - Deletes registrations that exist in the database but not in the JSON data
+
+    Returns:
+        dict: Summary of changes made (created, updated, deleted counts)
+    """
+    reg_data = get_registration_data(meeting)
+
+    # Get the meeting ID from the first registration, the API only deals with one meeting at a time
+    first_email = next(iter(reg_data['objects']))
+    meeting_number = reg_data['objects'][first_email]['meeting']
+    try:
+        Meeting.objects.get(number=meeting_number)
+    except Meeting.DoesNotExist:
+        raise Exception(f'meeting does not exist {meeting_number}')
+
+    # Get all existing registrations for this meeting
+    existing_registrations = meeting.registration_set.all()
+    existing_emails = set(reg.email for reg in existing_registrations if reg.email)
+
+    # Track changes for reporting
+    stats = {
+        'created': 0,
+        'updated': 0,
+        'deleted': 0,
+        'processed': 0,
+    }
+
+    # Process registrations from reg_data
+    reg_emails = set()
+    for email, data in reg_data['objects'].items():
+        stats['processed'] += 1
+        reg_emails.add(email)
+
+        # Process this registration
+        _, action_taken = process_single_registration(data, meeting)
+
+        # Update stats
+        if action_taken == 'created':
+            stats['created'] += 1
+        elif action_taken == 'updated':
+            stats['updated'] += 1
+
+    # Delete registrations that exist in the DB but not in registration data, they've been cancelled
+    emails_to_delete = existing_emails - reg_emails
+    if emails_to_delete:
+        result = Registration.objects.filter(
+            email__in=emails_to_delete,
+            meeting=meeting
+        ).delete()
+        if 'meeting.Registration' in result[1]:
+            deleted_count = result[1]['meeting.Registration']
+        else:
+            deleted_count = 0
+        stats['deleted'] = deleted_count
+
+    # set meeting.attendees
+    count = Registration.objects.onsite().filter(meeting=meeting, checkedin=True).count()
+    if meeting.attendees != count:
+        meeting.attendees = count
+        meeting.save()
+
+    return stats
+
+
+def process_single_registration(reg_data, meeting):
+    """
+    Process a single registration record - create, update, or leave unchanged as needed.
+
+    Args:
+        reg_data (dict): Registration data
+        meeting (obj): The IETF meeting
+
+    Returns:
+        tuple: (registration, action_taken)
+            - registration: Registration object
+            - action_taken: String indicating 'created', 'updated', or None
+    """
+    # import here to avoid circular imports
+    from ietf.nomcom.models import Volunteer, NomCom
+
+    action_taken = None
+    fields_updated = False
+    tickets_modified = False
+
+    # handle deleted
+    # should not see cancelled records during nightly sync but can see
+    # them from realtime notifications
+    if reg_data['cancelled']:
+        try:
+            registration = Registration.objects.get(meeting=meeting, email=reg_data['email'])
+        except Registration.DoesNotExist:
+            return (None, None)
+        for ticket in reg_data['tickets']:
+            target = registration.tickets.filter(
+                attendance_type__slug=ticket['attendance_type'],
+                ticket_type__slug=ticket['ticket_type']).first()
+            target.delete()
+        if registration.tickets.count() == 0:
+            registration.delete()
+        return (None, 'deleted')
+
+    person = Person.objects.filter(email__address=reg_data['email']).first()
+    if not person:
+        log.log(f"ERROR: meeting registration email unknown {reg_data['email']}")
+
+    registration, created = Registration.objects.get_or_create(
+        email=reg_data['email'],
+        meeting=meeting,
+        defaults={
+            'first_name': reg_data['first_name'],
+            'last_name': reg_data['last_name'],
+            'person': person,
+            'affiliation': reg_data['affiliation'],
+            'country_code': reg_data['country_code'],
+            'checkedin': reg_data['checkedin'],
+        }
+    )
+
+    # If not created, check if we need to update
+    if not created:
+        for field in ['first_name', 'last_name', 'affiliation', 'country_code', 'checkedin']:
+            if getattr(registration, field) != reg_data[field]:
+                setattr(registration, field, reg_data[field])
+                fields_updated = True
+
+    if fields_updated:
+        registration.save()
+
+    # Process tickets - handle counting properly for multiple same-type tickets
+    # Build count dictionaries for existing and new tickets
+    existing_ticket_counts = {}
+    for ticket in registration.tickets.all():
+        key = (ticket.attendance_type.slug, ticket.ticket_type.slug)
+        existing_ticket_counts[key] = existing_ticket_counts.get(key, 0) + 1
+
+    # Get new tickets from reg_data and count them
+    reg_data_ticket_counts = {}
+    for ticket_data in reg_data.get('tickets', []):
+        key = (ticket_data['attendance_type'], ticket_data['ticket_type'])
+        reg_data_ticket_counts[key] = reg_data_ticket_counts.get(key, 0) + 1
+
+    # Calculate tickets to add and remove
+    all_ticket_types = set(existing_ticket_counts.keys()) | set(reg_data_ticket_counts.keys())
+
+    for ticket_type in all_ticket_types:
+        existing_count = existing_ticket_counts.get(ticket_type, 0)
+        new_count = reg_data_ticket_counts.get(ticket_type, 0)
+
+        # Delete excess tickets
+        if existing_count > new_count:
+            tickets_to_delete = existing_count - new_count
+            # Get all tickets of this type
+            matching_tickets = registration.tickets.filter(
+                attendance_type__slug=ticket_type[0],
+                ticket_type__slug=ticket_type[1]
+            ).order_by('id')  # Use a consistent order for deterministic deletion
+
+            # Delete the required number
+            for ticket in matching_tickets[:tickets_to_delete]:
+                ticket.delete()
+            tickets_modified = True
+
+        # Add missing tickets
+        elif new_count > existing_count:
+            tickets_to_add = new_count - existing_count
+
+            # Create the new tickets
+            for _ in range(tickets_to_add):
+                try:
+                    RegistrationTicket.objects.create(
+                        registration=registration,
+                        attendance_type_id=ticket_type[0],
+                        ticket_type_id=ticket_type[1],
+                    )
+                    tickets_modified = True
+                except IntegrityError as e:
+                    log(f"Error adding RegistrationTicket {e}")
+
+    # handle nomcom volunteer
+    if reg_data['is_nomcom_volunteer'] and person:
+        try:
+            nomcom = NomCom.objects.get(is_accepting_volunteers=True)
+        except (NomCom.DoesNotExist, NomCom.MultipleObjectsReturned):
+            nomcom = None
+        if nomcom:
+            Volunteer.objects.get_or_create(
+                nomcom=nomcom,
+                person=person,
+                defaults={
+                    "affiliation": reg_data["affiliation"],
+                    "origin": "registration"
+                }
+            )
+
+    # set action_taken
+    if created:
+        action_taken = 'created'
+    elif fields_updated or tickets_modified:
+        action_taken = 'updated'
+
+    return registration, action_taken
+
+
+def fetch_attendance_from_meetings(meetings):
+    return [sync_registration_data(meeting) for meeting in meetings]
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 591b934b58..8bd70a3733 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -61,6 +61,7 @@
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, SessionPresentation, TimeSlot, SlideSubmission, Attended
 from ietf.meeting.models import ImportantDate, SessionStatusName, SchedulingEvent, SchedTimeSessAssignment, Room, TimeSlotTypeName
+from ietf.meeting.models import Registration
 from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm, ImportMinutesForm,
                                  TimeSlotCreateForm, TimeSlotEditForm, SessionCancelForm, SessionEditForm )
 from ietf.meeting.helpers import get_person_by_email, get_schedule_by_name
@@ -98,7 +99,6 @@
 from ietf.meeting.utils import participants_for_meeting, generate_bluesheet, bluesheet_data, save_bluesheet
 from ietf.message.utils import infer_message
 from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName
-from ietf.stats.models import MeetingRegistration
 from ietf.utils import markdown
 from ietf.utils.decorators import require_api_key
 from ietf.utils.hedgedoc import Note, NoteError
@@ -2710,7 +2710,7 @@ def session_attendance(request, session_id, num):
             was_there = Attended.objects.filter(session=session, person=person).exists()
             can_add = (
                 today_utc <= cor_cut_off_date
-                and MeetingRegistration.objects.filter(
+                and Registration.objects.filter(
                     meeting=session.meeting, person=person
                 ).exists()
                 and not was_there
@@ -4210,17 +4210,17 @@ def proceedings_attendees(request, num=None):
         return HttpResponseRedirect(f'{settings.PROCEEDINGS_V1_BASE_URL.format(meeting=meeting)}/attendee.html')
 
     template = None
-    meeting_registrations = None
+    registrations = None
 
     if int(meeting.number) >= 118:
         checked_in, attended = participants_for_meeting(meeting)
-        regs = list(MeetingRegistration.objects.filter(meeting__number=num, reg_type='onsite', checkedin=True))
+        regs = list(Registration.objects.onsite().filter(meeting__number=num, checkedin=True))
 
-        for mr in MeetingRegistration.objects.filter(meeting__number=num, reg_type='remote').select_related('person'):
-            if mr.person.pk in attended and mr.person.pk not in checked_in:
-                regs.append(mr)
+        for reg in Registration.objects.remote().filter(meeting__number=num).select_related('person'):
+            if reg.person.pk in attended and reg.person.pk not in checked_in:
+                regs.append(reg)
 
-        meeting_registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name))
+        registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name))
     else:
         overview_template = "/meeting/proceedings/%s/attendees.html" % meeting.number
         try:
@@ -4230,7 +4230,7 @@ def proceedings_attendees(request, num=None):
 
     return render(request, "meeting/proceedings_attendees.html", {
         'meeting': meeting,
-        'meeting_registrations': meeting_registrations,
+        'registrations': registrations,
         'template': template,
     })
 
diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py
index 8f94cc7fc5..ea17da6707 100644
--- a/ietf/nomcom/tests.py
+++ b/ietf/nomcom/tests.py
@@ -31,7 +31,8 @@
                                NewRevisionDocEventFactory, DocumentAuthorFactory
 from ietf.group.factories import GroupFactory, GroupHistoryFactory, RoleFactory, RoleHistoryFactory
 from ietf.group.models import Group, Role
-from ietf.meeting.factories import MeetingFactory, AttendedFactory
+from ietf.meeting.factories import MeetingFactory, AttendedFactory, RegistrationFactory
+from ietf.meeting.models import Registration
 from ietf.message.models import Message
 from ietf.nomcom.test_data import nomcom_test_data, generate_cert, check_comments, \
                                   COMMUNITY_USER, CHAIR_USER, \
@@ -50,8 +51,6 @@
                               decorate_volunteers_with_qualifications, send_reminders, _is_time_to_send_reminder
 from ietf.person.factories import PersonFactory, EmailFactory
 from ietf.person.models import Email, Person
-from ietf.stats.models import MeetingRegistration
-from ietf.stats.factories import MeetingRegistrationFactory
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent
 from ietf.utils.timezone import date_today, datetime_today, datetime_from_date, DEADLINE_TZINFO
@@ -2061,7 +2060,15 @@ def first_meeting_of_year(year):
                 if not ' ' in ascii:
                     continue
                 first_name, last_name = ascii.rsplit(None, 1)
-                MeetingRegistration.objects.create(meeting=meeting, first_name=first_name, last_name=last_name, person=person, country_code='WO', email=email, attended=True)
+                RegistrationFactory(
+                    meeting=meeting,
+                    first_name=first_name,
+                    last_name=last_name,
+                    person=person,
+                    country_code='WO',
+                    email=email,
+                    attended=True
+                )
         for view in ('public_eligible','private_eligible'):
             url = reverse(f'ietf.nomcom.views.{view}',kwargs={'year':self.nc.year()})
             for username in (self.chair.user.username,'secretary'):
@@ -2084,7 +2091,7 @@ def first_meeting_of_year(year):
         for number in range(meeting_start, meeting_start+8):
             m = MeetingFactory.create(type_id='ietf', number=number)
             for p in people:
-                m.meetingregistration_set.create(person=p, reg_type="onsite", checkedin=True, attended=True)
+                RegistrationFactory(meeting=m, person=p, checkedin=True, attended=True)
         for p in people:
             self.nc.volunteer_set.create(person=p,affiliation='something')
         for view in ('public_volunteers','private_volunteers'):
@@ -2110,10 +2117,6 @@ def first_meeting_of_year(year):
         self.assertContains(response, people[-1].plain_name(), status_code=200)
         self.assertNotContains(response, unqualified_person.plain_name())
 
-
-
-
-
 class NomComIndexTests(TestCase):
     def setUp(self):
         super().setUp()
@@ -2460,7 +2463,7 @@ def setUp(self):
             for combo in combinations(meetings,combo_len):
                 p = PersonFactory()
                 for m in combo:
-                    MeetingRegistrationFactory(person=p, meeting=m, attended=True)
+                    RegistrationFactory(person=p, meeting=m, attended=True)
                 if combo_len<3:
                     self.ineligible_people.append(p)
                 else:
@@ -2470,7 +2473,7 @@ def setUp(self):
         def ineligible_person_with_role(**kwargs):
             p = RoleFactory(**kwargs).person
             for m in meetings:
-                MeetingRegistrationFactory(person=p, meeting=m, attended=True)
+                RegistrationFactory(person=p, meeting=m, attended=True)
             self.ineligible_people.append(p)
         for group in ['isocbot', 'ietf-trust', 'llc-board', 'iab']:
             for role in ['member', 'chair']:
@@ -2485,8 +2488,7 @@ def ineligible_person_with_role(**kwargs):
         self.other_date = datetime.date(2009,5,1)
         self.other_people = PersonFactory.create_batch(1)
         for date in (datetime.date(2009,3,1), datetime.date(2008,11,1), datetime.date(2008,7,1)):
-            MeetingRegistrationFactory(person=self.other_people[0],meeting__date=date, meeting__type_id='ietf', attended=True)
-
+            RegistrationFactory(person=self.other_people[0], meeting__date=date, meeting__type_id='ietf', attended=True)
 
     def test_is_person_eligible(self):
         for person in self.eligible_people:
@@ -2530,7 +2532,7 @@ def setUp(self):
             for combo in combinations(meetings,combo_len):
                 p = PersonFactory()
                 for m in combo:
-                    MeetingRegistrationFactory(person=p, meeting=m, attended=True)
+                    RegistrationFactory(person=p, meeting=m, attended=True)
                 if combo_len<3:
                     self.ineligible_people.append(p)
                 else:
@@ -2578,7 +2580,7 @@ def test_elig_by_meetings(self):
                 for combo in combinations(prev_five,combo_len):
                     p = PersonFactory()
                     for m in combo:
-                        MeetingRegistrationFactory(person=p, meeting=m, attended=True) # not checkedin because this forces looking at older meetings
+                        RegistrationFactory(person=p, meeting=m, attended=True) # not checkedin because this forces looking at older meetings
                         AttendedFactory(session__meeting=m, session__type_id='plenary',person=p)
                     if combo_len<3:
                         ineligible_people.append(p)
@@ -2593,8 +2595,9 @@ def test_elig_by_meetings(self):
             for person in ineligible_people:
                 self.assertFalse(is_eligible(person,nomcom))
 
-            Person.objects.filter(pk__in=[p.pk for p in eligible_people+ineligible_people]).delete()
-
+            people = Person.objects.filter(pk__in=[p.pk for p in eligible_people + ineligible_people])
+            Registration.objects.filter(person__in=people).delete()
+            people.delete()
 
     def test_elig_by_office_active_groups(self):
 
@@ -2778,7 +2781,7 @@ def setUp(self):
     def test_registration_is_not_enough(self):
         p = PersonFactory()
         for meeting in self.meetings:
-            MeetingRegistrationFactory(person=p, meeting=meeting, checkedin=False)
+            RegistrationFactory(person=p, meeting=meeting, checkedin=False)
         self.assertFalse(is_eligible(p, self.nomcom))
 
     def test_elig_by_meetings(self):
@@ -2795,7 +2798,7 @@ def test_elig_by_meetings(self):
                 for method in attendance_methods:
                     p = PersonFactory()
                     for meeting in combo:
-                        MeetingRegistrationFactory(person=p, meeting=meeting, reg_type='onsite', checkedin=(method in ('checkedin', 'both')))
+                        RegistrationFactory(person=p, meeting=meeting, checkedin=(method in ('checkedin', 'both')))
                         if method in ('session', 'both'):
                             AttendedFactory(session__meeting=meeting, session__type_id='plenary',person=p)
                         if combo_len<3:
@@ -2828,7 +2831,7 @@ def test_volunteer(self):
         self.assertContains(r, 'NomCom is not accepting volunteers at this time', status_code=200)
         nomcom.is_accepting_volunteers = True
         nomcom.save()
-        MeetingRegistrationFactory(person=person, affiliation='mtg_affiliation', checkedin=True)
+        RegistrationFactory(person=person, affiliation='mtg_affiliation', checkedin=True)
         r = self.client.get(url)
         self.assertContains(r, 'Volunteer for NomCom', status_code=200)
         self.assertContains(r, 'mtg_affiliation')
@@ -2882,7 +2885,7 @@ def test_suggest_affiliation(self):
         nc = NomComFactory()
         nc.volunteer_set.create(person=person,affiliation='volunteer_affil')
         self.assertEqual(suggest_affiliation(person), 'volunteer_affil')
-        MeetingRegistrationFactory(person=person, affiliation='meeting_affil')
+        RegistrationFactory(person=person, affiliation='meeting_affil')
         self.assertEqual(suggest_affiliation(person), 'meeting_affil')
 
 class VolunteerDecoratorUnitTests(TestCase):
@@ -2900,7 +2903,7 @@ def test_decorate_volunteers_with_qualifications(self):
             ('106', datetime.date(2019, 11, 16)),
         ]]
         for m in meetings:
-            MeetingRegistrationFactory(meeting=m, person=meeting_person, attended=True)
+            RegistrationFactory(meeting=m, person=meeting_person, attended=True)
             AttendedFactory(session__meeting=m, session__type_id='plenary', person=meeting_person)
         nomcom.volunteer_set.create(person=meeting_person)
 
diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py
index ab155ef1d5..10494d323f 100644
--- a/ietf/nomcom/utils.py
+++ b/ietf/nomcom/utils.py
@@ -666,14 +666,14 @@ def previous_five_meetings(date = None):
     return Meeting.objects.filter(type='ietf',date__lte=date).order_by('-date')[:5]
 
 def three_of_five_eligible_8713(previous_five, queryset=None):
-    """ Return a list of Person records who attended at least 
+    """ Return a list of Person records who attended at least
         3 of the 5 type_id='ietf' meetings before the given
         date. Does not disqualify anyone based on held roles.
         This variant bases the calculation on MeetingRegistration.attended
     """
     if queryset is None:
         queryset = Person.objects.all()
-    return queryset.filter(meetingregistration__meeting__in=list(previous_five),meetingregistration__attended=True).annotate(mtg_count=Count('meetingregistration')).filter(mtg_count__gte=3)
+    return queryset.filter(registration__meeting__in=list(previous_five), registration__attended=True).annotate(mtg_count=Count('registration')).filter(mtg_count__gte=3)
 
 def three_of_five_eligible_9389(previous_five, queryset=None):
     """ Return a list of Person records who attended at least
@@ -692,7 +692,7 @@ def three_of_five_eligible_9389(previous_five, queryset=None):
     return queryset.filter(pk__in=[id for id, count in counts.items() if count >= 3])
 
 def suggest_affiliation(person):
-    recent_meeting = person.meetingregistration_set.order_by('-meeting__date').first()
+    recent_meeting = person.registration_set.order_by('-meeting__date').first()
     affiliation = recent_meeting.affiliation if recent_meeting else ''
     if not affiliation:
         recent_volunteer = person.volunteer_set.order_by('-nomcom__group__acronym').first()
diff --git a/ietf/settings.py b/ietf/settings.py
index c21120f77a..3f66e6abfe 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -1114,6 +1114,9 @@ def skip_unreadable_post(record):
 ]
 
 STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/'
+REGISTRATION_PARTICIPANTS_API_URL = 'https://registration.ietf.org/api/v1/participants-dt/'
+REGISTRATION_PARTICIPANTS_API_KEY = 'changeme'
+
 PROCEEDINGS_VERSION_CHANGES = [
     0,   # version 1
     97,  # version 2: meeting 97 and later (was number was NEW_PROCEEDINGS_START)
diff --git a/ietf/stats/management/commands/find_meetingregistration_person_issues.py b/ietf/stats/management/commands/find_meetingregistration_person_issues.py
deleted file mode 100644
index 4eaf6ac238..0000000000
--- a/ietf/stats/management/commands/find_meetingregistration_person_issues.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright The IETF Trust 2021, All Rights Reserved
-
-import debug # pyflakes:ignore
-
-from django.core.management.base import BaseCommand
-
-from ietf.stats.utils import find_meetingregistration_person_issues
-
-class Command(BaseCommand):
-    help = "Find possible Person/Email objects to repair based on MeetingRegistration objects"
-
-    def add_arguments(self, parser):
-        parser.add_argument('--meeting',action='append')
-
-    def handle(self, *args, **options):
-        meetings = options['meeting'] or None
-        summary = find_meetingregistration_person_issues(meetings)
-
-        print(f'{summary.ok_records} records are OK')
-
-        for msg in summary.could_be_fixed:
-            print(msg)
-
-        for msg in summary.maybe_address:
-            print(msg)
-
-        for msg in summary.different_person:
-            print(msg)
-
-        for msg in summary.no_person:
-            print(msg)
-
-        for msg in summary.maybe_person:
-            print(msg)
-
-        for msg in summary.no_email:
-            print(msg)
diff --git a/ietf/stats/management/commands/repair_meetingregistration_person.py b/ietf/stats/management/commands/repair_meetingregistration_person.py
deleted file mode 100644
index 74b1a807a6..0000000000
--- a/ietf/stats/management/commands/repair_meetingregistration_person.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright The IETF Trust 2021, All Rights Reserved
-
-import debug # pyflakes:ignore
-
-from django.core.management.base import BaseCommand
-
-from ietf.stats.utils import repair_meetingregistration_person
-
-class Command(BaseCommand):
-    help = "Repair MeetingRegistration objects that have no person but an email matching a person"
-
-    def add_arguments(self, parser):
-        parser.add_argument('--meeting',action='append')
-
-    def handle(self, *args, **options):
-        meetings = options['meeting'] or None
-        repaired = repair_meetingregistration_person(meetings)
-        print(f'Repaired {repaired} MeetingRegistration objects')
\ No newline at end of file
diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py
index 2e418eb0eb..f2e1d9801d 100644
--- a/ietf/stats/utils.py
+++ b/ietf/stats/utils.py
@@ -13,7 +13,7 @@
 
 from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias, MeetingRegistration
 from ietf.name.models import CountryName
-from ietf.person.models import Person, Email
+from ietf.person.models import Email
 from ietf.utils.log import log
 
 import logging
@@ -226,8 +226,8 @@ def compute_hirsch_index(citation_counts):
 def get_meeting_registration_data(meeting):
     """"Retrieve registration attendee data and summary statistics.  Returns number
     of Registration records created.
-    
-    MeetingRegistration records are created in realtime as people register for a 
+
+    MeetingRegistration records are created in realtime as people register for a
     meeting. This function serves as an audit / reconciliation. Most records are
     expected to already exist. The function has been optimized with this in mind.
     """
@@ -329,60 +329,6 @@ def get_meeting_registration_data(meeting):
         meeting.save()
     return num_created, num_processed, num_total
 
-def repair_meetingregistration_person(meetings=None):
-    repaired_records = 0
-    qs = MeetingRegistration.objects.all()
-    if meetings:
-        qs = qs.filter(meeting__number__in=meetings)
-    for mr in qs:
-        if mr.email and not mr.person:
-            email_person = Person.objects.filter(email__address=mr.email).first()
-            if email_person:
-                mr.person = email_person
-                mr.save()
-                repaired_records += 1            
-    return repaired_records
-
-class MeetingRegistrationIssuesSummary(object):
-    pass
-
-def find_meetingregistration_person_issues(meetings=None):
-    summary = MeetingRegistrationIssuesSummary()
-
-    summary.could_be_fixed = set()
-    summary.maybe_address = set()
-    summary.different_person = set()
-    summary.no_person = set()
-    summary.maybe_person = set()
-    summary.no_email = set()
-    summary.ok_records = 0
-
-    qs = MeetingRegistration.objects.all()
-    if meetings:
-        qs = qs.filter(meeting__number__in=meetings)
-    for mr in qs:
-        if mr.person and mr.email and mr.email in mr.person.email_set.values_list('address',flat=True):
-            summary.ok_records += 1
-        elif mr.email:
-            email_person = Person.objects.filter(email__address=mr.email).first()
-            if mr.person:
-                if not email_person:
-                    summary.maybe_address.add(f'{mr.email} is not present in any Email object. The MeetingRegistration object implies this is an address for {mr.person} ({mr.person.pk})')
-                elif email_person != mr.person:
-                    summary.different_person.add(f'{mr} ({mr.pk}) has person {mr.person} ({mr.person.pk}) but an email {mr.email} attached to a different person {email_person} ({email_person.pk}).')
-            elif email_person:
-                summary.could_be_fixed.add(f'{mr} ({mr.pk}) has no person, but email {mr.email} matches {email_person} ({email_person.pk})')
-            else:
-                maybe_person_qs = Person.objects.filter(name__icontains=mr.last_name).filter(name__icontains=mr.first_name)
-                if maybe_person_qs.exists():
-                    summary.maybe_person.add(f'{mr} ({mr.pk}) has email address {mr.email} which cannot be associated with any Person. Consider these possible people {[(p,p.pk) for p in maybe_person_qs]}')
-                else:
-                    summary.no_person.add(f'{mr} ({mr.pk}) has email address {mr.email} which cannot be associated with any Person')
-        else:
-            summary.no_email.add(f'{mr} ({mr.pk}) provides no email address')
-
-    return summary
-
 
 FetchStats = namedtuple("FetchStats", "added processed total")
 
diff --git a/ietf/templates/meeting/proceedings_attendees.html b/ietf/templates/meeting/proceedings_attendees.html
index 6c51a4546e..390ce00cad 100644
--- a/ietf/templates/meeting/proceedings_attendees.html
+++ b/ietf/templates/meeting/proceedings_attendees.html
@@ -29,13 +29,13 @@ <h2>Attendee list of IETF {{ meeting.number }} meeting</h2>
             </tr>
         </thead>
         <tbody>
-            {% for reg in meeting_registrations %}
+            {% for reg in registrations %}
             <tr>
                 <td>{{ reg.last_name }}</td>
                 <td>{{ reg.first_name }}</td>
                 <td>{{ reg.affiliation }}</td>
                 <td>{{ reg.country_code }}</td>
-                <td>{{ reg.reg_type }}</td>
+                <td>{{ reg.attendance_type }}</td>
             </tr>
             {% endfor %}
         </tbody>

From 015d6f789bbc100cb182aa77a7f7df77913b20c3 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 12 Jun 2025 14:31:15 -0300
Subject: [PATCH 350/656] feat: database-backed artifact blob storage (#8992)

* feat: blobdb app

* feat: blob model

* fix: app name

* feat: admin for blob model

* feat: database router

* feat: BlobdbStorage storage class

* refactor: storage metadata via File subclass

* refactor: eliminate in_flight_custom_metadata

* chore: raise blobstore exceptions in development mode

* feat: StagedBlobStore + StorageObjectStorageMixin

* feat: StoredObject.committed timestamp

* refactor: some refactoring (wip)

* refactor: better separation of StorageObject and Storage

Partial revert of 5d6bd49afd2d130fa687617e5d3efc002a4d96f3

* feat: basic 2-blobstore commit

* fix: StorageObject should have been StoredObject

* fix: propagate metadata to final_storage

* feat: easier config of StagedBlobStorage

* refactor: save-specific commit

* chore(dev): add dev database for blob storage

* chore(dev): migrate blobdb in app-init.sh

* chore(dev): use staged blob storages for dev env

* feat: deletion for StagedBlobStorage

* refactor: single-query update of committed field

* fix: better commit behavior for celery

Make async commit optional so that celery tasks don't spawn
other celery tasks. This is a definite fix.

Reverses order of StoredObject.create() and storage.save().
This avoids a race condition between an async commit and the
creation of a StoredObject, but need to review whether this
introduces other problems.

* style: black + copyright stmts

* fix: dont test for v1 apis into the blobdb app models

* chore: fix storage configs (esp for tests) (#8664)

* chore: remove empty blobdb.views

* feat: mtime/content-type for blobdb

* test: test BlobdbStorage

* refactor: split storage utils from Storage class

This is a rather consequential refactor. Moves the utility
methods (store_file(), remove_from_storage(), etc) out of
the Storage subclass. They are now only in storage_utils.py.
Drops the StoredObjectStorageMixin, instead moving the
Storage-like parts directly into
StoredObjectStagedBlobStorage. The CustomS3Storage class
is dropped, at least for now.

* fix: circular import

* chore: CRLFs in ISSUE_TEMPLATE/config.yml

* chore: add resources.py

* refactor: extract metadata from BlobFile

* refactor: reconcile dueling MetadataFiles

* refactor: plumb metadata and reorg code

* fix: handle content type properly

* chore: remove blobdb/resources.py

* test: ietf.blobdb->OMITTED_APPS; drop ad hoc ignore

* chore: copyright

* chore: Remove CR

* refactor: storage_backends.py -> storage.py

Matches standard naming convention

* refactor: remove StagedBlobStorage

* chore: fix mypy lint

* chore: remove StagedBlobStorage tasks

* refactor: committed -> replicated

* feat: admin improvements

* chore: configure storages for dev

 * configure separate replica storage
 * enable retries
 * switch to "standard" mode instead of legacy

* feat: copy blobs to "replica" storage when stored

* feat: propagate blob deletion, too

* fix: reset replicated flag on delete

* chore: remove replication

* refactor: unify artifact storage configs

All environments now use the same storage config,
so set it up directly in settings.py instead of
scattered throughout.

* fix: mypy lint

* fix: more mypy lint...

* fix: overwrite, don't rename, in BlobdbStorage

* fix: more lint

* feat: hooks for Blobdb watcher notifications

* feat: pybob the blob replicator

Simulation of an external replicator task.

* chore: fix quoting in celery docker-init.sh

* fix: adjust replicator/storage to work together

* chore: concurrency=1 worker for blobdb queue

* feat: blobdb replication module + settings

* feat: en/disable replication per settings

* feat: per-bucket replication en/disable

* fix: use correct type for EXCLUDE_BUCKETS

* style: reorder methods

* feat: logging/exceptions in replication.py

* feat: retry replication

* fix: valid storage pattern setting

* chore: consistent naming

* feat: detect missing replica storage configs

* fix: ensure mtime is set for the replica

Use blob modified timestamp as mtime in replicated
metadata. This is the intended behavior and matches
BlobdbStorage.

* fix: log message grammar

* feat: blobdb + k8s deployment (#8946)

* chore(dev): remove assert

I like the idea, but settings are sometimes loaded
more than once so this approach doesn't work

* chore: blobdb migrations in k8s

* chore: fix typo in commet

* chore: add replicator pod for k8s deploy

* chore: drop unnecessary ports from celery/beat

* chore: fix tyop

* chore: settings for blobdb / replication

* chore: put quotes back around $@

* ci: typo in k8s/settings_local.py

* chore: additional blobdb replicator logging (#8950)

* refactor: minor refactor + ruff style

* chore: additional logging

* fix: db transactions for blobdb save/delete (#8951)

* fix: use correct DB for transaction (#8952)

* fix: use correct DB for transaction

* chore: stale comment

* fix: on_commit needs "using" also (#8953)

* chore: unused imports in settings_locals

* refactor: typing-friendly blob handling in replication.py

* chore: more lint

* chore: fix lint affecting GHA tests

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 dev/build/migration-start.sh              |   3 +
 dev/build/start.sh                        |   5 +-
 dev/celery/docker-init.sh                 |   2 +-
 dev/deploy-to-container/settings_local.py |  21 ---
 dev/diff/settings_local.py                |  21 ---
 dev/tests/settings_local.py               |  21 ---
 docker-compose.yml                        |  34 +++-
 docker/configs/settings_local.py          |  60 ++++---
 docker/configs/settings_postgresqldb.py   |   8 +
 docker/scripts/app-configure-blobstore.py |   4 +-
 docker/scripts/app-init.sh                |   4 +-
 ietf/api/tests.py                         |   1 +
 ietf/blobdb/__init__.py                   |   0
 ietf/blobdb/admin.py                      |  31 ++++
 ietf/blobdb/apps.py                       |  30 ++++
 ietf/blobdb/factories.py                  |  13 ++
 ietf/blobdb/migrations/0001_initial.py    |  78 +++++++++
 ietf/blobdb/migrations/__init__.py        |   0
 ietf/blobdb/models.py                     |  98 +++++++++++
 ietf/blobdb/replication.py                | 178 ++++++++++++++++++++
 ietf/blobdb/routers.py                    |  58 +++++++
 ietf/blobdb/storage.py                    |  96 +++++++++++
 ietf/blobdb/tasks.py                      |  17 ++
 ietf/blobdb/tests.py                      |  80 +++++++++
 ietf/doc/admin.py                         |  20 ++-
 ietf/doc/models.py                        |   5 +-
 ietf/doc/storage.py                       | 178 ++++++++++++++++++++
 ietf/doc/storage_backends.py              | 192 ----------------------
 ietf/doc/storage_utils.py                 |  96 +++++++++--
 ietf/doc/tasks.py                         |   2 +-
 ietf/message/views.py                     |   1 +
 ietf/settings.py                          |  21 ++-
 ietf/settings_test.py                     |  25 +--
 ietf/utils/storage.py                     |  39 +++++
 ietf/utils/test_runner.py                 |   2 +-
 k8s/beat.yaml                             |   6 +-
 k8s/celery.yaml                           |   6 +-
 k8s/kustomization.yaml                    |   1 +
 k8s/replicator.yaml                       |  82 +++++++++
 k8s/settings_local.py                     |  66 ++++++--
 40 files changed, 1252 insertions(+), 353 deletions(-)
 create mode 100644 ietf/blobdb/__init__.py
 create mode 100644 ietf/blobdb/admin.py
 create mode 100644 ietf/blobdb/apps.py
 create mode 100644 ietf/blobdb/factories.py
 create mode 100644 ietf/blobdb/migrations/0001_initial.py
 create mode 100644 ietf/blobdb/migrations/__init__.py
 create mode 100644 ietf/blobdb/models.py
 create mode 100644 ietf/blobdb/replication.py
 create mode 100644 ietf/blobdb/routers.py
 create mode 100644 ietf/blobdb/storage.py
 create mode 100644 ietf/blobdb/tasks.py
 create mode 100644 ietf/blobdb/tests.py
 create mode 100644 ietf/doc/storage.py
 delete mode 100644 ietf/doc/storage_backends.py
 create mode 100644 k8s/replicator.yaml

diff --git a/dev/build/migration-start.sh b/dev/build/migration-start.sh
index 211638aec6..901026e53b 100644
--- a/dev/build/migration-start.sh
+++ b/dev/build/migration-start.sh
@@ -3,4 +3,7 @@
 echo "Running Datatracker migrations..."
 ./ietf/manage.py migrate --settings=settings_local
 
+echo "Running Blobdb migrations ..."
+./ietf/manage.py migrate --settings=settings_local --database=blobdb
+
 echo "Done!"
diff --git a/dev/build/start.sh b/dev/build/start.sh
index ef6c7fc0d9..3b03637068 100644
--- a/dev/build/start.sh
+++ b/dev/build/start.sh
@@ -2,7 +2,7 @@
 #
 # Environment config:
 #
-#  CONTAINER_ROLE - datatracker, celery, or beat (defaults to datatracker)
+#  CONTAINER_ROLE - datatracker, celery, beat, migrations, or replicator (defaults to datatracker)
 #
 case "${CONTAINER_ROLE:-datatracker}" in
     auth)
@@ -20,6 +20,9 @@ case "${CONTAINER_ROLE:-datatracker}" in
     migrations)
         exec ./migration-start.sh
         ;;
+    replicator)
+        exec ./celery-start.sh --app=ietf worker --queues=blobdb --concurrency=1
+        ;;
     *)
         echo "Unknown role '${CONTAINER_ROLE}'"
         exit 255       
diff --git a/dev/celery/docker-init.sh b/dev/celery/docker-init.sh
index 9940dfd7d0..6a36cb6d78 100755
--- a/dev/celery/docker-init.sh
+++ b/dev/celery/docker-init.sh
@@ -99,7 +99,7 @@ if [[ -n "${DEV_MODE}" ]]; then
             --recursive \
             --debounce-interval 5 \
             -- \
-            celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" "$@" &
+            celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" $@ &
   celery_pid=$!
 else
   celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" "$@" &
diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py
index e878206bd5..aacf000093 100644
--- a/dev/deploy-to-container/settings_local.py
+++ b/dev/deploy-to-container/settings_local.py
@@ -2,8 +2,6 @@
 # -*- coding: utf-8 -*-
 
 from ietf.settings import *   # pyflakes:ignore
-from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
-import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
@@ -81,22 +79,3 @@
 
 # OIDC configuration
 SITE_URL = 'https://__HOSTNAME__'
-
-for storagename in MORE_STORAGE_NAMES:
-    STORAGES[storagename] = {
-        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
-        "OPTIONS": dict(
-            endpoint_url="http://blobstore:9000",
-            access_key="minio_root",
-            secret_key="minio_pass",
-            security_token=None,
-            client_config=botocore.config.Config(
-                signature_version="s3v4",
-                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
-                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
-                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
-            ),
-            verify=False,
-            bucket_name=f"test-{storagename}",
-        ),
-    }
diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py
index 9e0806a8a6..c255cac23d 100644
--- a/dev/diff/settings_local.py
+++ b/dev/diff/settings_local.py
@@ -2,8 +2,6 @@
 # -*- coding: utf-8 -*-
 
 from ietf.settings import *  # pyflakes:ignore
-from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
-import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
@@ -68,22 +66,3 @@
 SLIDE_STAGING_PATH = 'test/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
-
-for storagename in MORE_STORAGE_NAMES:
-    STORAGES[storagename] = {
-        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
-        "OPTIONS": dict(
-            endpoint_url="http://blobstore:9000",
-            access_key="minio_root",
-            secret_key="minio_pass",
-            security_token=None,
-            client_config=botocore.config.Config(
-                signature_version="s3v4",
-                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
-                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
-                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
-            ),
-            verify=False,
-            bucket_name=f"test-{storagename}",
-        ),
-    }
diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py
index f2166053a7..e1ffd60edb 100644
--- a/dev/tests/settings_local.py
+++ b/dev/tests/settings_local.py
@@ -2,8 +2,6 @@
 # -*- coding: utf-8 -*-
 
 from ietf.settings import *  # pyflakes:ignore
-from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
-import botocore.config
 
 ALLOWED_HOSTS = ['*']
 
@@ -67,22 +65,3 @@
 SLIDE_STAGING_PATH = 'test/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
-
-for storagename in MORE_STORAGE_NAMES:
-    STORAGES[storagename] = {
-        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
-        "OPTIONS": dict(
-            endpoint_url="http://blobstore:9000",
-            access_key="minio_root",
-            secret_key="minio_pass",
-            security_token=None,
-            client_config=botocore.config.Config(
-                signature_version="s3v4",
-                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
-                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
-                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
-            ),
-            verify=False,
-            bucket_name=f"test-{storagename}",
-        ),
-    }
diff --git a/docker-compose.yml b/docker-compose.yml
index 9910c02a99..100119c464 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -86,13 +86,44 @@ services:
             - .:/workspace
             - app-assets:/assets
 
+    replicator:
+        build:
+            context: .
+            dockerfile: docker/celery.Dockerfile
+        init: true
+        environment:
+            CELERY_APP: ietf
+            CELERY_ROLE: worker
+            UPDATE_REQUIREMENTS_FROM: requirements.txt
+            DEV_MODE: "yes"
+        command:
+            - '--loglevel=INFO'
+            - '--queues=blobdb'
+            - '--concurrency=1'
+                 
+        depends_on:
+            - db
+        restart: unless-stopped
+        stop_grace_period: 1m
+        volumes:
+            - .:/workspace
+            - app-assets:/assets
+
     blobstore:
         image: ghcr.io/ietf-tools/datatracker-devblobstore:latest
         restart: unless-stopped
         volumes:
             - "minio-data:/data"
 
-
+    blobdb:
+        image: postgres:16
+        restart: unless-stopped
+        environment:
+            POSTGRES_DB: blob
+            POSTGRES_USER: dt
+            POSTGRES_PASSWORD: abcd1234
+        volumes:
+            - blobdb-data:/var/lib/postgresql/data
 
 #  Celery Beat is a periodic task runner. It is not normally needed for development,
 #  but can be enabled by uncommenting the following.
@@ -118,3 +149,4 @@ volumes:
     postgresdb-data:
     app-assets:
     minio-data:
+    blobdb-data:
diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py
index 46833451c1..ca51871463 100644
--- a/docker/configs/settings_local.py
+++ b/docker/configs/settings_local.py
@@ -2,12 +2,26 @@
 # -*- coding: utf-8 -*-
 
 from ietf.settings import *  # pyflakes:ignore
-from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
-import botocore.config
+from ietf.settings import (
+    ARTIFACT_STORAGE_NAMES,
+    STORAGES,
+    BLOBSTORAGE_MAX_ATTEMPTS,
+    BLOBSTORAGE_READ_TIMEOUT,
+    BLOBSTORAGE_CONNECT_TIMEOUT,
+)
 
 ALLOWED_HOSTS = ['*']
 
 from ietf.settings_postgresqldb import DATABASES  # pyflakes:ignore
+DATABASE_ROUTERS = ["ietf.blobdb.routers.BlobdbStorageRouter"]
+BLOBDB_DATABASE = "blobdb"
+BLOBDB_REPLICATION = {
+    "ENABLED": True,
+    "DEST_STORAGE_PATTERN": "r2-{bucket}",
+    "INCLUDE_BUCKETS": ARTIFACT_STORAGE_NAMES,
+    "EXCLUDE_BUCKETS": ["staging"],
+    "VERBOSE_LOGGING": True,
+}
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/"
@@ -39,25 +53,6 @@
 # DEV_TEMPLATE_CONTEXT_PROCESSORS = [
 #    'ietf.context_processors.sql_debug',
 # ]
-for storagename in MORE_STORAGE_NAMES:
-    STORAGES[storagename] = {
-        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
-        "OPTIONS": dict(
-            endpoint_url="http://blobstore:9000",
-            access_key="minio_root",
-            secret_key="minio_pass",
-            security_token=None,
-            client_config=botocore.config.Config(
-                signature_version="s3v4",
-                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
-                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
-                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
-            ),
-            verify=False,
-            bucket_name=storagename,
-        ),
-    }
-
 
 DOCUMENT_PATH_PATTERN = '/assets/ietfdata/doc/{doc.type_id}/'
 INTERNET_DRAFT_PATH = '/assets/ietf-ftp/internet-drafts/'
@@ -80,3 +75,26 @@
 
 STATIC_IETF_ORG = "/_static"
 STATIC_IETF_ORG_INTERNAL = "http://static"
+
+
+# Blob replication storage for dev
+import botocore.config
+for storagename in ARTIFACT_STORAGE_NAMES:
+    replica_storagename = f"r2-{storagename}"
+    STORAGES[replica_storagename] = {
+        "BACKEND": "ietf.doc.storage.MetadataS3Storage",
+        "OPTIONS": dict(
+            endpoint_url="http://blobstore:9000",
+            access_key="minio_root",
+            secret_key="minio_pass",
+            security_token=None,
+            client_config=botocore.config.Config(
+                signature_version="s3v4",
+                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
+                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
+                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
+            ),
+            verify=False,
+            bucket_name=f"{storagename}",
+        ),
+    }
diff --git a/docker/configs/settings_postgresqldb.py b/docker/configs/settings_postgresqldb.py
index 05d19b9a86..9b98586658 100644
--- a/docker/configs/settings_postgresqldb.py
+++ b/docker/configs/settings_postgresqldb.py
@@ -7,4 +7,12 @@
         'USER': 'django',
         'PASSWORD': 'RkTkDPFnKpko',
     },
+    'blobdb': {
+        'HOST': 'blobdb',
+        'PORT': 5432,
+        'NAME': 'blob',
+        'ENGINE': 'django.db.backends.postgresql',
+        'USER': 'dt',
+        'PASSWORD': 'abcd1234',
+    },
 }
diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py
index ae87bf1afe..df4685b246 100755
--- a/docker/scripts/app-configure-blobstore.py
+++ b/docker/scripts/app-configure-blobstore.py
@@ -7,7 +7,7 @@
 import os
 import sys
 
-from ietf.settings import MORE_STORAGE_NAMES
+from ietf.settings import ARTIFACT_STORAGE_NAMES
 
 
 def init_blobstore():
@@ -19,7 +19,7 @@ def init_blobstore():
         aws_session_token=None,
         config=botocore.config.Config(signature_version="s3v4"),
     )
-    for bucketname in MORE_STORAGE_NAMES:
+    for bucketname in ARTIFACT_STORAGE_NAMES:
         try:
             blobstore.create_bucket(
                 Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip()
diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh
index e970398ac2..17e0c6c764 100755
--- a/docker/scripts/app-init.sh
+++ b/docker/scripts/app-init.sh
@@ -101,9 +101,11 @@ echo "Running initial checks..."
 /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check --settings=settings_local
 
 # Migrate, adjusting to what the current state of the underlying database might be:
-
 /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --fake-initial --settings=settings_local
 
+# Apply migrations to the blobdb database as well (most are skipped)
+/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local --database=blobdb
+
 if [ -z "$EDITOR_VSCODE" ]; then
     CODE=0
     python -m smtpd -n -c DebuggingServer localhost:2025 &
diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 32f33dcc63..6754f5c3de 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -50,6 +50,7 @@
     'ietf.secr.proceedings',
     'ietf.ipr',
     'ietf.status',
+    'ietf.blobdb',
 )
 
 class CustomApiTests(TestCase):
diff --git a/ietf/blobdb/__init__.py b/ietf/blobdb/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/ietf/blobdb/admin.py b/ietf/blobdb/admin.py
new file mode 100644
index 0000000000..f4cd002e07
--- /dev/null
+++ b/ietf/blobdb/admin.py
@@ -0,0 +1,31 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from django.contrib import admin
+from django.db.models.functions import Length
+from rangefilter.filters import DateRangeQuickSelectListFilterBuilder
+
+from .models import Blob
+
+
+@admin.register(Blob)
+class BlobAdmin(admin.ModelAdmin):
+    list_display = ["bucket", "name", "object_size", "modified", "mtime", "content_type"]
+    list_filter = [
+        "bucket",
+        "content_type",
+        ("modified", DateRangeQuickSelectListFilterBuilder()),
+        ("mtime", DateRangeQuickSelectListFilterBuilder()),
+    ]
+    search_fields = ["name"]
+    list_display_links = ["name"]
+
+    def get_queryset(self, request):
+        return (
+            super().get_queryset(request)
+            .defer("content")  # don't load this unless we want it
+            .annotate(object_size=Length("content"))  # accessed via object_size()
+        )
+
+    @admin.display(ordering="object_size")
+    def object_size(self, instance):
+        """Get the size of the object"""
+        return instance.object_size  # annotation added in get_queryset()
diff --git a/ietf/blobdb/apps.py b/ietf/blobdb/apps.py
new file mode 100644
index 0000000000..c2513b6819
--- /dev/null
+++ b/ietf/blobdb/apps.py
@@ -0,0 +1,30 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from django.apps import AppConfig
+
+
+class BlobdbConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "ietf.blobdb"
+
+    def ready(self):
+        """Initialize app once the registries / settings are populated"""
+        from django.conf import settings
+
+        # Validate that the DB is set up
+        db = get_blobdb()  # depends on settings.BLOBDB_DATABASE
+        if db is not None and db not in settings.DATABASES:
+            raise RuntimeError(
+                f"settings.BLOBDB_DATABASE is '{db}' but that is not present in settings.DATABASES"
+            )
+
+        # Validate replication settings
+        from .replication import validate_replication_settings
+
+        validate_replication_settings()
+
+
+def get_blobdb():
+    """Retrieve the blobdb setting from Django's settings"""
+    from django.conf import settings
+
+    return getattr(settings, "BLOBDB_DATABASE", None)
diff --git a/ietf/blobdb/factories.py b/ietf/blobdb/factories.py
new file mode 100644
index 0000000000..fdcebb2b4e
--- /dev/null
+++ b/ietf/blobdb/factories.py
@@ -0,0 +1,13 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+import factory
+
+from .models import Blob
+
+
+class BlobFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = Blob
+
+    name = factory.Faker("file_path")
+    bucket = factory.Faker("word")
+    content = factory.Faker("binary", length=32)  # careful, default length is 1e6
diff --git a/ietf/blobdb/migrations/0001_initial.py b/ietf/blobdb/migrations/0001_initial.py
new file mode 100644
index 0000000000..2fcb08e5ea
--- /dev/null
+++ b/ietf/blobdb/migrations/0001_initial.py
@@ -0,0 +1,78 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="Blob",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "name",
+                    models.CharField(help_text="Name of the blob", max_length=1024),
+                ),
+                (
+                    "bucket",
+                    models.CharField(
+                        help_text="Name of the bucket containing this blob",
+                        max_length=1024,
+                    ),
+                ),
+                (
+                    "modified",
+                    models.DateTimeField(
+                        default=django.utils.timezone.now,
+                        help_text="Last modification time of the blob",
+                    ),
+                ),
+                ("content", models.BinaryField(help_text="Content of the blob")),
+                (
+                    "checksum",
+                    models.CharField(
+                        editable=False,
+                        help_text="SHA-384 digest of the content",
+                        max_length=96,
+                    ),
+                ),
+                (
+                    "mtime",
+                    models.DateTimeField(
+                        blank=True,
+                        default=None,
+                        help_text="mtime associated with the blob as a filesystem object",
+                        null=True,
+                    ),
+                ),
+                (
+                    "content_type",
+                    models.CharField(
+                        blank=True,
+                        help_text="content-type header value for the blob contents",
+                        max_length=1024,
+                    ),
+                ),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name="blob",
+            constraint=models.UniqueConstraint(
+                fields=("bucket", "name"), name="unique_name_per_bucket"
+            ),
+        ),
+    ]
diff --git a/ietf/blobdb/migrations/__init__.py b/ietf/blobdb/migrations/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/ietf/blobdb/models.py b/ietf/blobdb/models.py
new file mode 100644
index 0000000000..8f423d9f6c
--- /dev/null
+++ b/ietf/blobdb/models.py
@@ -0,0 +1,98 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+import json
+from functools import partial
+from hashlib import sha384
+
+from django.db import models, transaction
+from django.utils import timezone
+
+from .apps import get_blobdb
+from .replication import replication_enabled
+from .tasks import pybob_the_blob_replicator_task
+
+
+class BlobQuerySet(models.QuerySet):
+    """QuerySet customized for Blob management
+
+    Operations that bypass save() / delete() won't correctly notify watchers of changes
+    to the blob contents. Disallow them.
+    """
+
+    def delete(self):
+        raise NotImplementedError("Only deleting individual Blobs is supported")
+
+    def bulk_create(self, *args, **kwargs):
+        raise NotImplementedError("Only creating individual Blobs is supported")
+
+    def update(self, *args, **kwargs):
+        # n.b., update_or_create() _does_ call save()
+        raise NotImplementedError("Updating BlobQuerySets is not supported")
+
+    def bulk_update(self, *args, **kwargs):
+        raise NotImplementedError("Updating Blobs in bulk is not supported")
+
+
+class Blob(models.Model):
+    objects = BlobQuerySet.as_manager()
+    name = models.CharField(max_length=1024, help_text="Name of the blob")
+    bucket = models.CharField(
+        max_length=1024, help_text="Name of the bucket containing this blob"
+    )
+    modified = models.DateTimeField(
+        default=timezone.now, help_text="Last modification time of the blob"
+    )
+    content = models.BinaryField(help_text="Content of the blob")
+    checksum = models.CharField(
+        max_length=96, help_text="SHA-384 digest of the content", editable=False
+    )
+    mtime = models.DateTimeField(
+        default=None,
+        blank=True,
+        null=True,
+        help_text="mtime associated with the blob as a filesystem object",
+    )
+    content_type = models.CharField(
+        max_length=1024,
+        blank=True,
+        help_text="content-type header value for the blob contents",
+    )
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["bucket", "name"], name="unique_name_per_bucket"
+            ),
+        ]
+
+    def save(self, **kwargs):
+        db = get_blobdb()
+        with transaction.atomic(using=db):
+            self.checksum = sha384(self.content, usedforsecurity=False).hexdigest()
+            super().save(**kwargs)
+            self._emit_blob_change_event(using=db)
+
+    def delete(self, **kwargs):
+        db = get_blobdb()
+        with transaction.atomic(using=db):
+            retval = super().delete(**kwargs)
+            self._emit_blob_change_event(using=db)
+        return retval
+
+    def _emit_blob_change_event(self, using=None):
+        if not replication_enabled(self.bucket):
+            return
+
+        # For now, fire a celery task we've arranged to guarantee in-order processing.
+        # Later becomes pushing an event onto a queue to a dedicated worker.
+        transaction.on_commit(
+            partial(
+                pybob_the_blob_replicator_task.delay,
+                json.dumps(
+                    {
+                        "name": self.name,
+                        "bucket": self.bucket,
+                    }
+                )
+            ),
+            using=using,
+        )
diff --git a/ietf/blobdb/replication.py b/ietf/blobdb/replication.py
new file mode 100644
index 0000000000..b9d55c9498
--- /dev/null
+++ b/ietf/blobdb/replication.py
@@ -0,0 +1,178 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+import datetime
+from dataclasses import dataclass
+from io import BytesIO
+from typing import Optional
+
+from django.conf import settings
+from django.core.files import File
+from django.core.files.storage import storages, InvalidStorageError
+from django.db import connections
+
+from ietf.utils import log
+
+DEFAULT_SETTINGS = {
+    "ENABLED": False,
+    "DEST_STORAGE_PATTERN": "r2-{bucket}",
+    "INCLUDE_BUCKETS": (),  # empty means include all
+    "EXCLUDE_BUCKETS": (),  # empty means exclude none
+    "VERBOSE_LOGGING": False,
+}
+
+
+class SimpleMetadataFile(File):
+    def __init__(self, file, name=None):
+        super().__init__(file, name)
+        self.custom_metadata = {}
+        self.content_type = ""
+
+
+def get_replication_settings():
+    return DEFAULT_SETTINGS | getattr(settings, "BLOBDB_REPLICATION", {})
+
+
+def validate_replication_settings():
+    replicator_settings = get_replication_settings()
+    # No extra settings allowed
+    unknown_settings = set(DEFAULT_SETTINGS.keys()) - set(replicator_settings.keys())
+    if len(unknown_settings) > 0:
+        raise RuntimeError(
+            f"Unrecognized BLOBDB_REPLICATOR settings: {', '.join(str(unknown_settings))}"
+        )
+    # destination storage pattern must be a string that includes {bucket}
+    pattern = replicator_settings["DEST_STORAGE_PATTERN"]
+    if not isinstance(pattern, str):
+        raise RuntimeError(
+            f"DEST_STORAGE_PATTERN must be a str, not {type(pattern).__name__}"
+        )
+    if "{bucket}" not in pattern:
+        raise RuntimeError(
+            f"DEST_STORAGE_PATTERN must contain the substring '{{bucket}}' (found '{pattern}')"
+        )
+    # include/exclude buckets must be list-like
+    include_buckets = replicator_settings["INCLUDE_BUCKETS"]
+    if not isinstance(include_buckets, (list, tuple, set)):
+        raise RuntimeError("INCLUDE_BUCKETS must be a list, tuple, or set")
+    exclude_buckets = replicator_settings["EXCLUDE_BUCKETS"]
+    if not isinstance(exclude_buckets, (list, tuple, set)):
+        raise RuntimeError("EXCLUDE_BUCKETS must be a list, tuple, or set")
+    # if we have explicit include_buckets, make sure the necessary storages exist
+    if len(include_buckets) > 0:
+        include_storages = {destination_storage_name_for(b) for b in include_buckets}
+        exclude_storages = {destination_storage_name_for(b) for b in exclude_buckets}
+        configured_storages = set(settings.STORAGES.keys())
+        missing_storages = include_storages - exclude_storages - configured_storages
+        if len(missing_storages) > 0:
+            raise RuntimeError(
+                f"Replication requires unknown storage(s): {', '.join(missing_storages)}"
+            )
+
+
+def destination_storage_name_for(bucket: str):
+    pattern = get_replication_settings()["DEST_STORAGE_PATTERN"]
+    return pattern.format(bucket=bucket)
+
+
+def destination_storage_for(bucket: str):
+    storage_name = destination_storage_name_for(bucket)
+    return storages[storage_name]
+
+
+def replication_enabled(bucket: str):
+    replication_settings = get_replication_settings()
+    if not replication_settings["ENABLED"]:
+        return False
+    # Default is all buckets are included
+    included = (
+        len(replication_settings["INCLUDE_BUCKETS"]) == 0
+        or bucket in replication_settings["INCLUDE_BUCKETS"]
+    )
+    # Default is no buckets are excluded
+    excluded = (
+        len(replication_settings["EXCLUDE_BUCKETS"]) > 0
+        and bucket in replication_settings["EXCLUDE_BUCKETS"]
+    )
+    return included and not excluded
+
+
+def verbose_logging_enabled():
+    return bool(get_replication_settings()["VERBOSE_LOGGING"])
+
+
+@dataclass
+class SqlBlob:
+    content: bytes
+    checksum: str
+    modified: datetime.datetime
+    mtime: Optional[datetime.datetime]
+    content_type: str
+
+
+def fetch_blob_via_sql(bucket: str, name: str) -> Optional[SqlBlob]:
+    blobdb_connection = connections["blobdb"]
+    cursor = blobdb_connection.cursor()
+    cursor.execute(
+        """
+        SELECT content, checksum, modified, mtime, content_type FROM blobdb_blob
+        WHERE bucket=%s AND name=%s LIMIT 1
+        """,
+        [bucket, name],
+    )
+    row = cursor.fetchone()
+    col_names = [col[0] for col in cursor.description]
+    return None if row is None else SqlBlob(**{
+        col_name: row_val
+        for col_name, row_val in zip(col_names, row)
+    })
+
+
+def replicate_blob(bucket, name):
+    """Replicate a Blobdb blob to a Storage"""
+    if not replication_enabled(bucket):
+        if verbose_logging_enabled():
+            log.log(
+                f"Not replicating {bucket}:{name} because replication is not enabled for {bucket}"
+            )
+        return
+
+    try:
+        destination_storage = destination_storage_for(bucket)
+    except InvalidStorageError as e:
+        log.log(
+            f"Failed to replicate {bucket}:{name} because destination storage for {bucket} is not configured"
+        )
+        raise ReplicationError from e
+
+    blob = fetch_blob_via_sql(bucket, name)
+    if blob is None:
+        if verbose_logging_enabled():
+            log.log("Deleting {bucket}:{name} from replica")
+        try:
+            destination_storage.delete(name)
+        except Exception as e:
+            log.log("Failed to delete {bucket}:{name} from replica: {e}")
+            raise ReplicationError from e
+    else:
+        # Add metadata expected by the MetadataS3Storage
+        file_with_metadata = SimpleMetadataFile(file=BytesIO(blob.content))
+        file_with_metadata.content_type = blob.content_type
+        file_with_metadata.custom_metadata = {
+            "sha384": blob.checksum,
+            "mtime": (blob.mtime or blob.modified).isoformat(),
+        }
+        if verbose_logging_enabled():
+            log.log(
+                f"Saving {bucket}:{name} to replica ("
+                f"sha384: '{file_with_metadata.custom_metadata['sha384'][:16]}...', "
+                f"content_type: '{file_with_metadata.content_type}', "
+                f"mtime: '{file_with_metadata.custom_metadata['mtime']})"
+            )
+        try:
+            destination_storage.save(name, file_with_metadata)
+        except Exception as e:
+            log.log("Failed to save {bucket}:{name} to replica: {e}")
+            raise ReplicationError from e
+
+
+class ReplicationError(Exception):
+    pass
diff --git a/ietf/blobdb/routers.py b/ietf/blobdb/routers.py
new file mode 100644
index 0000000000..319c0fbc71
--- /dev/null
+++ b/ietf/blobdb/routers.py
@@ -0,0 +1,58 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from django.apps import apps
+
+from .apps import BlobdbConfig, get_blobdb
+
+
+class BlobdbStorageRouter:
+    """Database router for the Blobdb"""
+
+    _app_label = None
+
+    @property
+    def app_label(self):
+        if self._app_label is None:
+            for app in apps.get_app_configs():
+                if isinstance(app, BlobdbConfig):
+                    self._app_label = app.label
+                    break
+            if self._app_label is None:
+                raise RuntimeError(
+                    f"{BlobdbConfig.name} is not present in the Django app registry"
+                )
+        return self._app_label
+
+    @property
+    def db(self):
+        return get_blobdb()
+
+    def db_for_read(self, model, **hints):
+        """Suggest the database that should be used for read operations for objects of type model
+
+        Returns None if there is no suggestion.
+        """
+        if model._meta.app_label == self.app_label:
+            return self.db
+        return None  # no suggestion
+
+    def db_for_write(self, model, **hints):
+        """Suggest the database that should be used for write of objects of type model
+
+        Returns None if there is no suggestion.
+        """
+        if model._meta.app_label == self.app_label:
+            return self.db
+        return None  # no suggestion
+
+    def allow_migrate(self, db, app_label, model_name=None, **hints):
+        """Determine if the migration operation is allowed to run on the database with alias db
+
+        Return True if the operation should run, False if it shouldn’t run, or
+        None if the router has no opinion.
+        """
+        if self.db is None:
+            return None  # no opinion, use the default db
+        is_our_app = app_label == self.app_label
+        is_our_db = db == self.db
+        if is_our_app or is_our_db:
+            return is_our_app and is_our_db
diff --git a/ietf/blobdb/storage.py b/ietf/blobdb/storage.py
new file mode 100644
index 0000000000..4213ec801d
--- /dev/null
+++ b/ietf/blobdb/storage.py
@@ -0,0 +1,96 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from typing import Optional
+
+from django.core.exceptions import SuspiciousFileOperation
+from django.core.files.base import ContentFile
+from django.core.files.storage import Storage
+from django.db.models.functions import Length
+from django.utils.deconstruct import deconstructible
+from django.utils import timezone
+
+from ietf.utils.storage import MetadataFile
+from .models import Blob
+
+
+class BlobFile(MetadataFile):
+
+    def __init__(self, content, name=None, mtime=None, content_type=""):
+        super().__init__(
+            file=ContentFile(content),
+            name=name,
+            mtime=mtime,
+            content_type=content_type,
+        )
+
+
+@deconstructible
+class BlobdbStorage(Storage):
+
+    def __init__(self, bucket_name: Optional[str]=None):
+        if bucket_name is None:
+            raise ValueError("BlobdbStorage bucket_name must be specified")
+        self.bucket_name = bucket_name
+
+    def get_queryset(self):
+        return Blob.objects.filter(bucket=self.bucket_name)
+
+    def delete(self, name):
+        blob = self.get_queryset().filter(name=name).first()
+        if blob is not None:
+            blob.delete()
+
+    def exists(self, name):
+        return self.get_queryset().filter(name=name).exists()
+
+    def size(self, name):
+        sizes = (
+            self.get_queryset()
+            .filter(name=name)
+            .annotate(object_size=Length("content"))
+            .values_list("object_size", flat=True)
+        )
+        if len(sizes) == 0:
+            raise FileNotFoundError(
+                f"No object '{name}' exists in bucket '{self.bucket_name}'"
+            )
+        return sizes[0]  # unique constraint guarantees 0 or 1 entry
+
+    def _open(self, name, mode="rb"):
+        try:
+            blob = self.get_queryset().get(name=name)
+        except Blob.DoesNotExist:
+            raise FileNotFoundError(
+                f"No object '{name}' exists in bucket '{self.bucket_name}'"
+            )
+        return BlobFile(
+            content=blob.content,
+            name=blob.name,
+            mtime=blob.mtime or blob.modified,  # fall back to modified time
+            content_type=blob.content_type,
+        )
+
+    def _save(self, name, content):
+        """Perform the save operation
+        
+        The storage API allows _save() to save to a different name than was requested. This method will
+        never do that, instead overwriting the existing blob.
+        """
+        Blob.objects.update_or_create(
+            name=name,
+            bucket=self.bucket_name,
+            defaults={
+                "content": content.read(),
+                "modified": timezone.now(),
+                "mtime": getattr(content, "mtime", None),
+                "content_type": getattr(content, "content_type", ""),
+            },
+        )
+        return name
+
+    def get_available_name(self, name, max_length=None):
+        if max_length is not None and len(name) > max_length:
+            raise SuspiciousFileOperation(
+                f"BlobdbStorage only allows names up to {max_length}, but was"
+                f"asked to store the name '{name[:5]}...{name[-5:]} of length {len(name)}"
+            )
+        return name  # overwrite is permitted
diff --git a/ietf/blobdb/tasks.py b/ietf/blobdb/tasks.py
new file mode 100644
index 0000000000..538d415830
--- /dev/null
+++ b/ietf/blobdb/tasks.py
@@ -0,0 +1,17 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+import json
+
+from celery import shared_task
+
+from .replication import replicate_blob, ReplicationError
+
+
+@shared_task(
+    autoretry_for=(ReplicationError,), retry_backoff=10, retry_kwargs={"max_retries": 5}
+)
+def pybob_the_blob_replicator_task(body: str):
+    request = json.loads(body)
+    bucket = request["bucket"]
+    name = request["name"]
+    replicate_blob(bucket, name)
diff --git a/ietf/blobdb/tests.py b/ietf/blobdb/tests.py
new file mode 100644
index 0000000000..0eadad0a1f
--- /dev/null
+++ b/ietf/blobdb/tests.py
@@ -0,0 +1,80 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+import datetime
+
+from django.core.files.base import ContentFile
+
+from ietf.utils.test_utils import TestCase
+from .factories import BlobFactory
+from .models import Blob
+from .storage import BlobFile, BlobdbStorage
+
+
+class StorageTests(TestCase):
+    def test_save(self):
+        storage = BlobdbStorage(bucket_name="my-bucket")
+        timestamp = datetime.datetime(
+            2025,
+            3,
+            17,
+            1,
+            2,
+            3,
+            tzinfo=datetime.timezone.utc,
+        )
+        # Create file to save
+        my_file = BlobFile(
+            content=b"These are my bytes.",
+            mtime=timestamp,
+            content_type="application/x-my-content-type",
+        )
+        # save the file
+        saved_name = storage.save("myfile.txt", my_file)
+        # validate the outcome
+        self.assertEqual(saved_name, "myfile.txt")
+        blob = Blob.objects.filter(bucket="my-bucket", name="myfile.txt").first()
+        self.assertIsNotNone(blob)  # validates bucket and name
+        self.assertEqual(bytes(blob.content), b"These are my bytes.")
+        self.assertEqual(blob.mtime, timestamp)
+        self.assertEqual(blob.content_type, "application/x-my-content-type")
+        
+    def test_save_naive_file(self):
+        storage = BlobdbStorage(bucket_name="my-bucket")
+        my_naive_file = ContentFile(content=b"These are my naive bytes.")
+        # save the file
+        saved_name = storage.save("myfile.txt", my_naive_file)
+        # validate the outcome
+        self.assertEqual(saved_name, "myfile.txt")
+        blob = Blob.objects.filter(bucket="my-bucket", name="myfile.txt").first()
+        self.assertIsNotNone(blob)  # validates bucket and name
+        self.assertEqual(bytes(blob.content), b"These are my naive bytes.")
+        self.assertIsNone(blob.mtime)
+        self.assertEqual(blob.content_type, "")
+
+    def test_open(self):
+        """BlobdbStorage open yields a BlobFile with specific mtime and content_type"""
+        mtime = datetime.datetime(2021, 1, 2, 3, 45, tzinfo=datetime.timezone.utc)
+        blob = BlobFactory(mtime=mtime, content_type="application/x-oh-no-you-didnt")
+        storage = BlobdbStorage(bucket_name=blob.bucket)
+        with storage.open(blob.name, "rb") as f:
+            self.assertTrue(isinstance(f, BlobFile))
+            assert isinstance(f, BlobFile)  # redundant, narrows type for linter
+            self.assertEqual(f.read(), bytes(blob.content))
+            self.assertEqual(f.mtime, mtime)
+            self.assertEqual(f.content_type, "application/x-oh-no-you-didnt")
+
+    def test_open_null_mtime(self):
+        """BlobdbStorage open yields a BlobFile with default mtime and content_type"""
+        blob = BlobFactory(content_type="application/x-oh-no-you-didnt")  # does not set mtime
+        storage = BlobdbStorage(bucket_name=blob.bucket)
+        with storage.open(blob.name, "rb") as f:
+            self.assertTrue(isinstance(f, BlobFile))
+            assert isinstance(f, BlobFile)  # redundant, narrows type for linter
+            self.assertEqual(f.read(), bytes(blob.content))
+            self.assertIsNotNone(f.mtime)
+            self.assertEqual(f.mtime, blob.modified)
+            self.assertEqual(f.content_type, "application/x-oh-no-you-didnt")
+
+    def test_open_file_not_found(self):
+        storage = BlobdbStorage(bucket_name="not-a-bucket")
+        with self.assertRaises(FileNotFoundError):
+            storage.open("definitely/not-a-file.txt")
diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py
index db3b24b2d2..b492aa3423 100644
--- a/ietf/doc/admin.py
+++ b/ietf/doc/admin.py
@@ -1,10 +1,11 @@
-# Copyright The IETF Trust 2010-2021, All Rights Reserved
+# Copyright The IETF Trust 2010-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
 from django.contrib import admin
 from django.db import models
 from django import forms
+from rangefilter.filters import DateRangeQuickSelectListFilterBuilder
 
 from .models import (StateType, State, RelatedDocument, DocumentAuthor, Document, RelatedDocHistory,
     DocHistoryAuthor, DocHistory, DocReminder, DocEvent, NewRevisionDocEvent,
@@ -220,7 +221,18 @@ class DocExtResourceAdmin(admin.ModelAdmin):
 admin.site.register(DocExtResource, DocExtResourceAdmin)
 
 class StoredObjectAdmin(admin.ModelAdmin):
-    list_display = ['store', 'name', 'modified', 'deleted']
-    list_filter = ['deleted']
-    search_fields = ['store', 'name', 'doc_name', 'doc_rev', 'deleted']
+    list_display = ['store', 'name', 'doc_name', 'modified', 'is_deleted']
+    list_filter = [
+        'store',
+        ('modified', DateRangeQuickSelectListFilterBuilder()),
+        ('deleted', DateRangeQuickSelectListFilterBuilder()),
+    ]
+    search_fields = ['name', 'doc_name', 'doc_rev']
+    list_display_links = ['name']
+
+    @admin.display(boolean=True, description="Deleted?", ordering="deleted")
+    def is_deleted(self, instance):
+        return instance.deleted is not None
+    
+
 admin.site.register(StoredObject, StoredObjectAdmin)
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index 55da70972c..b6f36cb8a7 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2010-2023, All Rights Reserved
+# Copyright The IETF Trust 2010-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -1615,3 +1615,6 @@ class Meta:
         indexes = [
             models.Index(fields=["doc_name", "doc_rev"]),
         ]
+
+    def __str__(self):
+        return f"{self.store}:{self.name}"
diff --git a/ietf/doc/storage.py b/ietf/doc/storage.py
new file mode 100644
index 0000000000..a234ef2d4f
--- /dev/null
+++ b/ietf/doc/storage.py
@@ -0,0 +1,178 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from typing import Optional
+
+import debug  # pyflakes:ignore
+import json
+
+from contextlib import contextmanager
+from storages.backends.s3 import S3Storage
+
+from django.core.files.base import File
+
+from ietf.blobdb.storage import BlobdbStorage
+from ietf.doc.models import StoredObject
+from ietf.utils.log import log
+from ietf.utils.storage import MetadataFile
+from ietf.utils.timezone import timezone
+
+
+class StoredObjectFile(MetadataFile):
+    """Django storage File object that represents a StoredObject"""
+    def __init__(self, file, name, mtime=None, content_type="", store=None, doc_name=None, doc_rev=None):
+        super().__init__(
+            file=file,
+            name=name,
+            mtime=mtime,
+            content_type=content_type,
+        )
+        self.store = store
+        self.doc_name = doc_name
+        self.doc_rev = doc_rev
+
+    @classmethod
+    def from_storedobject(cls, file, name, store):
+        """Alternate constructor for objects that already exist in the StoredObject table"""
+        stored_object = StoredObject.objects.filter(store=store, name=name, deleted__isnull=True).first()
+        if stored_object is None:
+            raise FileNotFoundError(f"StoredObject for {store}:{name} does not exist or was deleted")
+        file = cls(file, name, store, doc_name=stored_object.doc_name, doc_rev=stored_object.doc_rev)
+        if int(file.custom_metadata["len"]) != stored_object.len:
+            raise RuntimeError(f"File length changed unexpectedly for {store}:{name}")
+        if file.custom_metadata["sha384"] != stored_object.sha384:
+            raise RuntimeError(f"SHA-384 hash changed unexpectedly for {store}:{name}")
+        return file
+
+
+@contextmanager
+def maybe_log_timing(enabled, op, **kwargs):
+    """If enabled, log elapsed time and additional data from kwargs
+
+    Emits log even if an exception occurs
+    """
+    before = timezone.now()
+    exception = None
+    try:
+        yield
+    except Exception as err:
+        exception = err
+        raise
+    finally:
+        if enabled:
+            dt = timezone.now() - before
+            log(
+                json.dumps(
+                    {
+                        "log": "S3Storage_timing",
+                        "seconds": dt.total_seconds(),
+                        "op": op,
+                        "exception": "" if exception is None else repr(exception),
+                        **kwargs,
+                    }
+                )
+            )
+
+
+class MetadataS3Storage(S3Storage):
+    def get_default_settings(self):
+        # add a default for the ietf_log_blob_timing boolean
+        return super().get_default_settings() | {"ietf_log_blob_timing": False}
+
+    def _save(self, name, content: File):
+        with maybe_log_timing(
+            self.ietf_log_blob_timing, "_save", bucket_name=self.bucket_name, name=name
+        ):
+            return super()._save(name, content)
+
+    def _open(self, name, mode="rb"):
+        with maybe_log_timing(
+            self.ietf_log_blob_timing,
+            "_open",
+            bucket_name=self.bucket_name,
+            name=name,
+            mode=mode,
+        ):
+            return super()._open(name, mode)
+
+    def delete(self, name):
+        with maybe_log_timing(
+            self.ietf_log_blob_timing, "delete", bucket_name=self.bucket_name, name=name
+        ):
+            super().delete(name)
+
+    def _get_write_parameters(self, name, content=None):
+        # debug.show('f"getting write parameters for {name}"')
+        params = super()._get_write_parameters(name, content)
+        # If we have a non-empty explicit content type, use it
+        content_type = getattr(content, "content_type", "").strip()
+        if content_type != "":
+            params["ContentType"] = content_type
+        if "Metadata" not in params:
+            params["Metadata"] = {}
+        if hasattr(content, "custom_metadata"):
+            params["Metadata"].update(content.custom_metadata)
+        return params
+
+
+class StoredObjectBlobdbStorage(BlobdbStorage):
+    ietf_log_blob_timing = True
+    warn_if_missing = True  # TODO-BLOBSTORE make this configurable (or remove it)
+
+    def _save_stored_object(self, name, content) -> StoredObject:
+        now = timezone.now()
+        record, created = StoredObject.objects.get_or_create(
+            store=self.bucket_name,
+            name=name,
+            defaults=dict(
+                sha384=content.custom_metadata["sha384"],
+                len=int(content.custom_metadata["len"]),
+                store_created=now,
+                created=now,
+                modified=now,
+                doc_name=getattr(
+                    content,
+                    "doc_name",  # Note that these are assumed to be invariant
+                    None,  # should be blank?
+                ),
+                doc_rev=getattr(
+                    content,
+                    "doc_rev",  # for a given name
+                    None,  # should be blank?
+                ),
+            ),
+        )
+        if not created:
+            record.sha384 = content.custom_metadata["sha384"]
+            record.len = int(content.custom_metadata["len"])
+            record.modified = now
+            record.deleted = None
+            record.save()
+        return record
+
+    def _delete_stored_object(self, name) -> Optional[StoredObject]:
+        existing_record = StoredObject.objects.filter(store=self.bucket_name, name=name)
+        if not existing_record.exists() and self.warn_if_missing:
+            complaint = (
+                f"WARNING: Asked to delete {name} from {self.bucket_name} storage, "
+                f"but there was no matching StoredObject"
+            )
+            log(complaint)
+            debug.show("complaint")
+        else:
+            now = timezone.now()
+            # Note that existing_record is a queryset that will have one matching object
+            existing_record.filter(deleted__isnull=True).update(deleted=now)
+        return existing_record.first()
+
+    def _save(self, name, content):
+        """Perform the save operation 
+        
+        In principle the name could change on save to the blob store. As of now, BlobdbStorage
+        will not change it, but allow for that possibility. Callers should be prepared for this.
+        """
+        saved_name = super()._save(name, content)
+        self._save_stored_object(saved_name, content)
+        return saved_name
+
+    def delete(self, name):
+        self._delete_stored_object(name)
+        super().delete(name)
diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py
deleted file mode 100644
index 5eeab040e5..0000000000
--- a/ietf/doc/storage_backends.py
+++ /dev/null
@@ -1,192 +0,0 @@
-# Copyright The IETF Trust 2025, All Rights Reserved
-
-import debug  # pyflakes:ignore
-import json
-
-from contextlib import contextmanager
-from hashlib import sha384
-from io import BufferedReader
-from storages.backends.s3 import S3Storage
-from typing import Optional, Union
-
-from django.core.files.base import File
-
-from ietf.doc.models import StoredObject
-from ietf.utils.log import log
-from ietf.utils.timezone import timezone
-
-
-@contextmanager
-def maybe_log_timing(enabled, op, **kwargs):
-    """If enabled, log elapsed time and additional data from kwargs
-
-    Emits log even if an exception occurs
-    """
-    before = timezone.now()
-    exception = None
-    try:
-        yield
-    except Exception as err:
-        exception = err
-        raise
-    finally:
-        if enabled:
-            dt = timezone.now() - before
-            log(
-                json.dumps(
-                    {
-                        "log": "S3Storage_timing",
-                        "seconds": dt.total_seconds(),
-                        "op": op,
-                        "exception": "" if exception is None else repr(exception),
-                        **kwargs,
-                    }
-                )
-            )
-
-
-# TODO-BLOBSTORE
-# Consider overriding save directly so that
-# we capture metadata for, e.g., ImageField objects
-class CustomS3Storage(S3Storage):
-
-    def __init__(self, **settings):
-        self.in_flight_custom_metadata = {}  # type is Dict[str, Dict[str, str]]
-        super().__init__(**settings)
-
-    def get_default_settings(self):
-        # add a default for the ietf_log_blob_timing boolean
-        return super().get_default_settings() | {"ietf_log_blob_timing": False}
-
-    def _save(self, name, content):
-        with maybe_log_timing(
-            self.ietf_log_blob_timing, "_save", bucket_name=self.bucket_name, name=name
-        ):
-            return super()._save(name, content)
-
-    def _open(self, name, mode="rb"):
-        with maybe_log_timing(
-            self.ietf_log_blob_timing,
-            "_open",
-            bucket_name=self.bucket_name,
-            name=name,
-            mode=mode,
-        ):
-            return super()._open(name, mode)
-
-    def delete(self, name):
-        with maybe_log_timing(
-            self.ietf_log_blob_timing, "delete", bucket_name=self.bucket_name, name=name
-        ):
-            super().delete(name)
-
-    def store_file(
-        self,
-        kind: str,
-        name: str,
-        file: Union[File, BufferedReader],
-        allow_overwrite: bool = False,
-        doc_name: Optional[str] = None,
-        doc_rev: Optional[str] = None,
-    ):
-        is_new = not self.exists_in_storage(kind, name)
-        # debug.show('f"Asked to store {name} in {kind}: is_new={is_new}, allow_overwrite={allow_overwrite}"')
-        if not allow_overwrite and not is_new:
-            log(f"Failed to save {kind}:{name} - name already exists in store")
-            debug.show('f"Failed to save {kind}:{name} - name already exists in store"')
-            # raise Exception("Not ignoring overwrite attempts while testing")
-        else:
-            try:
-                new_name = self.save(name, file)
-                now = timezone.now()
-                record, created = StoredObject.objects.get_or_create(
-                    store=kind,
-                    name=name,
-                    defaults=dict(
-                        sha384=self.in_flight_custom_metadata[name]["sha384"],
-                        len=int(self.in_flight_custom_metadata[name]["len"]),
-                        store_created=now,
-                        created=now,
-                        modified=now,
-                        doc_name=doc_name,  # Note that these are assumed to be invariant
-                        doc_rev=doc_rev,  # for a given name
-                    ),
-                )
-                if not created:
-                    record.sha384 = self.in_flight_custom_metadata[name]["sha384"]
-                    record.len = int(self.in_flight_custom_metadata[name]["len"])
-                    record.modified = now
-                    record.deleted = None
-                    record.save()
-                if new_name != name:
-                    complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead."
-                    log(complaint)
-                    debug.show("complaint")
-                    # Note that we are otherwise ignoring this condition - it should become an error later.
-            except Exception as e:
-                # Log and then swallow the exception while we're learning.
-                # Don't let failure pass so quietly when these are the autoritative bits.
-                complaint = f"Failed to save {kind}:{name}"
-                log(complaint, e)
-                debug.show('f"{complaint}: {e}"')
-            finally:
-                del self.in_flight_custom_metadata[name]
-        return None
-
-    def exists_in_storage(self, kind: str, name: str) -> bool:
-        try:
-            # open is realized with a HEAD
-            # See https://github.com/jschneier/django-storages/blob/b79ea310201e7afd659fe47e2882fe59aae5b517/storages/backends/s3.py#L528
-            with self.open(name):
-                return True
-        except FileNotFoundError:
-            return False
-
-    def remove_from_storage(
-        self, kind: str, name: str, warn_if_missing: bool = True
-    ) -> None:
-        now = timezone.now()
-        try:
-            with self.open(name):
-                pass
-            self.delete(name)
-            # debug.show('f"deleted {name} from {kind} storage"')
-        except FileNotFoundError:
-            if warn_if_missing:
-                complaint = (
-                    f"WARNING: Asked to delete non-existent {name} from {kind} storage"
-                )
-                log(complaint)
-                debug.show("complaint")
-        existing_record = StoredObject.objects.filter(store=kind, name=name)
-        if not existing_record.exists() and warn_if_missing:
-            complaint = f"WARNING: Asked to delete {name} from {kind} storage, but there was no matching StorageObject"
-            log(complaint)
-            debug.show("complaint")
-        else:
-            # Note that existing_record is a queryset that will have one matching object
-            existing_record.filter(deleted__isnull=True).update(deleted=now)
-
-    def _get_write_parameters(self, name, content=None):
-        # debug.show('f"getting write parameters for {name}"')
-        params = super()._get_write_parameters(name, content)
-        if "Metadata" not in params:
-            params["Metadata"] = {}
-        try:
-            content.seek(0)
-        except AttributeError:  # TODO-BLOBSTORE
-            debug.say("Encountered Non-Seekable content")
-            raise NotImplementedError("cannot handle unseekable content")
-        content_bytes = content.read()
-        if not isinstance(
-            content_bytes, bytes
-        ):  # TODO-BLOBSTORE: This is sketch-development only -remove before committing
-            raise Exception(f"Expected bytes - got {type(content_bytes)}")
-        content.seek(0)
-        metadata = {
-            "len": f"{len(content_bytes)}",
-            "sha384": f"{sha384(content_bytes).hexdigest()}",
-        }
-        params["Metadata"].update(metadata)
-        self.in_flight_custom_metadata[name] = metadata
-        return params
diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py
index 012efc9071..510c98c4f5 100644
--- a/ietf/doc/storage_utils.py
+++ b/ietf/doc/storage_utils.py
@@ -1,21 +1,19 @@
 # Copyright The IETF Trust 2025, All Rights Reserved
-
+import datetime
 from io import BufferedReader
 from typing import Optional, Union
+
 import debug  # pyflakes ignore
 
 from django.conf import settings
 from django.core.files.base import ContentFile, File
-from django.core.files.storage import storages
+from django.core.files.storage import storages, Storage
 
 from ietf.utils.log import log
 
 
-# TODO-BLOBSTORE (Future, maybe after leaving 3.9) : add a return type
-def _get_storage(kind: str):
-
-    if kind in settings.MORE_STORAGE_NAMES:
-        # TODO-BLOBSTORE - add a checker that verifies configuration will only return CustomS3Storages
+def _get_storage(kind: str) -> Storage:
+    if kind in settings.ARTIFACT_STORAGE_NAMES:
         return storages[kind]
     else:
         debug.say(f"Got into not-implemented looking for {kind}")
@@ -26,23 +24,35 @@ def exists_in_storage(kind: str, name: str) -> bool:
     if settings.ENABLE_BLOBSTORAGE:
         try:
             store = _get_storage(kind)
-            return store.exists_in_storage(kind, name)
+            with store.open(name):
+                return True
+        except FileNotFoundError:
+            return False
         except Exception as err:
             log(f"Blobstore Error: Failed to test existence of {kind}:{name}: {repr(err)}")
+            if settings.SERVER_MODE == "development":
+                raise
     return False
 
 
 def remove_from_storage(kind: str, name: str, warn_if_missing: bool = True) -> None:
     if settings.ENABLE_BLOBSTORAGE:
         try:
-            store = _get_storage(kind)
-            store.remove_from_storage(kind, name, warn_if_missing)
+            if exists_in_storage(kind, name):
+                _get_storage(kind).delete(name)
+            elif warn_if_missing:
+                complaint = (
+                    f"WARNING: Asked to delete non-existent {name} from {kind} storage"
+                )
+                debug.show("complaint")
+                log(complaint)
         except Exception as err:
             log(f"Blobstore Error: Failed to remove {kind}:{name}: {repr(err)}")
+            if settings.SERVER_MODE == "development":
+                raise
     return None
 
 
-# TODO-BLOBSTORE: Try to refactor `kind` out of the signature of the methods already on the custom store (which knows its kind)
 def store_file(
     kind: str,
     name: str,
@@ -50,14 +60,36 @@ def store_file(
     allow_overwrite: bool = False,
     doc_name: Optional[str] = None,
     doc_rev: Optional[str] = None,
+    content_type: str="",
+    mtime: Optional[datetime.datetime]=None,
 ) -> None:
-    # debug.show('f"asked to store {name} into {kind}"')
+    from .storage import StoredObjectFile  # avoid circular import
     if settings.ENABLE_BLOBSTORAGE:
         try:
-            store = _get_storage(kind)
-            store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev)
+            is_new = not exists_in_storage(kind, name)
+            # debug.show('f"Asked to store {name} in {kind}: is_new={is_new}, allow_overwrite={allow_overwrite}"')
+            if not allow_overwrite and not is_new:
+                debug.show('f"Failed to save {kind}:{name} - name already exists in store"')
+                raise RuntimeError(f"Failed to save {kind}:{name} - name already exists in store")
+            new_name = _get_storage(kind).save(
+                name,
+                StoredObjectFile(
+                    file=file,
+                    name=name,
+                    doc_name=doc_name,
+                    doc_rev=doc_rev,
+                    mtime=mtime,
+                    content_type=content_type,
+                ),
+            )
+            if new_name != name:
+                complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead."
+                debug.show("complaint")
+                raise RuntimeError(complaint)
         except Exception as err:
             log(f"Blobstore Error: Failed to store file {kind}:{name}: {repr(err)}")
+            if settings.SERVER_MODE == "development":
+                raise  # TODO-BLOBSTORE eventually make this an error for all modes
     return None
 
 
@@ -68,13 +100,26 @@ def store_bytes(
     allow_overwrite: bool = False,
     doc_name: Optional[str] = None,
     doc_rev: Optional[str] = None,
+    content_type: str = "",
+    mtime: Optional[datetime.datetime] = None,
 ) -> None:
     if settings.ENABLE_BLOBSTORAGE:
         try:
-            store_file(kind, name, ContentFile(content), allow_overwrite)
+            store_file(
+                kind,
+                name,
+                ContentFile(content),
+                allow_overwrite,
+                doc_name,
+                doc_rev,
+                content_type,
+                mtime,
+            )
         except Exception as err:
             # n.b., not likely to get an exception here because store_file or store_bytes will catch it
             log(f"Blobstore Error: Failed to store bytes to {kind}:{name}: {repr(err)}")
+            if settings.SERVER_MODE == "development":
+                raise  # TODO-BLOBSTORE eventually make this an error for all modes
     return None
 
 
@@ -85,19 +130,32 @@ def store_str(
     allow_overwrite: bool = False,
     doc_name: Optional[str] = None,
     doc_rev: Optional[str] = None,
+    content_type: str = "",
+    mtime: Optional[datetime.datetime] = None,
 ) -> None:
     if settings.ENABLE_BLOBSTORAGE:
         try:
             content_bytes = content.encode("utf-8")
-            store_bytes(kind, name, content_bytes, allow_overwrite)
+            store_bytes(
+                kind,
+                name,
+                content_bytes,
+                allow_overwrite,
+                doc_name,
+                doc_rev,
+                content_type,
+                mtime,
+            )
         except Exception as err:
             # n.b., not likely to get an exception here because store_file or store_bytes will catch it
             log(f"Blobstore Error: Failed to store string to {kind}:{name}: {repr(err)}")
+            if settings.SERVER_MODE == "development":
+                raise  # TODO-BLOBSTORE eventually make this an error for all modes
     return None
 
 
 def retrieve_bytes(kind: str, name: str) -> bytes:
-    from ietf.doc.storage_backends import maybe_log_timing
+    from ietf.doc.storage import maybe_log_timing
     content = b""
     if settings.ENABLE_BLOBSTORAGE:
         try:
@@ -112,6 +170,8 @@ def retrieve_bytes(kind: str, name: str) -> bytes:
                     content = f.read()
         except Exception as err:
             log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}")
+            if settings.SERVER_MODE == "development":
+                raise
     return content
 
 
@@ -124,4 +184,6 @@ def retrieve_str(kind: str, name: str) -> str:
             content = content_bytes.decode("utf-8")
         except Exception as err:
             log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}")
+            if settings.SERVER_MODE == "development":
+                raise
     return content
diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py
index e24c58e1e7..4f7fe37782 100644
--- a/ietf/doc/tasks.py
+++ b/ietf/doc/tasks.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2024, All Rights Reserved
+# Copyright The IETF Trust 2024-2025, All Rights Reserved
 #
 # Celery task definitions
 #
diff --git a/ietf/message/views.py b/ietf/message/views.py
index e4cca63017..355dcdd8d2 100644
--- a/ietf/message/views.py
+++ b/ietf/message/views.py
@@ -1,3 +1,4 @@
+# Copyright The IETF Trust 2013-2025, All Rights Reserved
 from django.shortcuts import render, get_object_or_404
 
 from ietf.message.models import Message
diff --git a/ietf/settings.py b/ietf/settings.py
index 3f66e6abfe..0b6b3745d2 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -185,9 +185,11 @@
 
 ENABLE_BLOBSTORAGE = True
 
-BLOBSTORAGE_MAX_ATTEMPTS = 1
-BLOBSTORAGE_CONNECT_TIMEOUT = 2
-BLOBSTORAGE_READ_TIMEOUT = 2
+# "standard" retry mode is used, which does exponential backoff with a base factor of 2
+# and a cap of 20. 
+BLOBSTORAGE_MAX_ATTEMPTS = 5  # boto3 default is 3 (for "standard" retry mode)
+BLOBSTORAGE_CONNECT_TIMEOUT = 10  # seconds; boto3 default is 60
+BLOBSTORAGE_READ_TIMEOUT = 10  # seconds; boto3 default is 60
 
 WSGI_APPLICATION = "ietf.wsgi.application"
 
@@ -471,6 +473,7 @@ def skip_unreadable_post(record):
     'widget_tweaks',
     # IETF apps
     'ietf.api',
+    'ietf.blobdb',
     'ietf.community',
     'ietf.dbtemplate',
     'ietf.doc',
@@ -748,8 +751,8 @@ def skip_unreadable_post(record):
     "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"},
 }
 
-# settings_local will need to configure storages for these names
-MORE_STORAGE_NAMES: list[str] = [
+# Storages for artifacts stored as blobs
+ARTIFACT_STORAGE_NAMES: list[str] = [
     "bofreq",
     "charter",
     "conflrev",
@@ -774,6 +777,11 @@ def skip_unreadable_post(record):
     "photo",
     "review",
 ]
+for storagename in ARTIFACT_STORAGE_NAMES:
+    STORAGES[storagename] = {
+        "BACKEND": "ietf.doc.storage.StoredObjectBlobdbStorage",
+        "OPTIONS": {"bucket_name": storagename},
+    }
 
 # Override this in settings_local.py if needed
 # *_PATH variables ends with a slash/ .
@@ -1281,6 +1289,9 @@ def skip_unreadable_post(record):
 CELERY_CACHE_BACKEND = 'celery-results'  # which Django cache to use
 CELERY_RESULT_EXPIRES = datetime.timedelta(minutes=5)  # how long are results valid? (Default is 1 day)
 CELERY_TASK_IGNORE_RESULT = True  # ignore results unless specifically enabled for a task
+CELERY_TASK_ROUTES = {
+    "ietf.blobdb.tasks.pybob_the_blob_replicator_task": {"queue": "blobdb"}
+}
 
 # Meetecho API setup: Uncomment this and provide real credentials to enable
 # Meetecho conference creation for interim session requests
diff --git a/ietf/settings_test.py b/ietf/settings_test.py
index fe77152d42..644a6d5b16 100755
--- a/ietf/settings_test.py
+++ b/ietf/settings_test.py
@@ -14,8 +14,7 @@
 import shutil
 import tempfile
 from ietf.settings import *                                          # pyflakes:ignore
-from ietf.settings import STORAGES, TEST_CODE_COVERAGE_CHECKER, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
-import botocore.config
+from ietf.settings import TEST_CODE_COVERAGE_CHECKER
 import debug                            # pyflakes:ignore
 debug.debug = True
 
@@ -49,6 +48,10 @@ def __getitem__(self, item):
         },
     }
 
+# test with a single DB - do not use a DB router
+BLOBDB_DATABASE = "default"
+DATABASE_ROUTERS: list[str] = []
+
 if TEST_CODE_COVERAGE_CHECKER and not TEST_CODE_COVERAGE_CHECKER._started: # pyflakes:ignore
     TEST_CODE_COVERAGE_CHECKER.start()                          # pyflakes:ignore
 
@@ -115,21 +118,3 @@ def tempdir_with_cleanup(**kwargs):
 _blob_store_enable_profiling = (
     os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true"
 )
-for storagename in MORE_STORAGE_NAMES:
-    STORAGES[storagename] = {
-        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
-        "OPTIONS": dict(
-            endpoint_url=_blob_store_endpoint_url,
-            access_key=_blob_store_access_key,
-            secret_key=_blob_store_secret_key,
-            security_token=None,
-            client_config=botocore.config.Config(
-                signature_version="s3v4",
-                connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
-                read_timeout=BLOBSTORAGE_READ_TIMEOUT,
-                retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS},
-            ),
-            bucket_name=f"{_blob_store_bucket_prefix}{storagename}",
-            ietf_log_blob_timing=_blob_store_enable_profiling,
-        ),
-    }
diff --git a/ietf/utils/storage.py b/ietf/utils/storage.py
index 42fcf884aa..bad5af5178 100644
--- a/ietf/utils/storage.py
+++ b/ietf/utils/storage.py
@@ -1,8 +1,12 @@
 # Copyright The IETF Trust 2020-2025, All Rights Reserved
 """Django Storage classes"""
+import datetime
+from hashlib import sha384
 from pathlib import Path
+from typing import Optional
 
 from django.conf import settings
+from django.core.files.base import File
 from django.core.files.storage import FileSystemStorage
 from ietf.doc.storage_utils import store_file
 from .log import log
@@ -48,9 +52,44 @@ def save(self, name, content, max_length=None):
                     store_file(self.kind, blob_name, f, allow_overwrite=True)
             except Exception as err:
                 log(f"Blobstore Error: Failed to shadow {saved_name} at {self.kind}:{blob_name}: {repr(err)}")
+                if settings.SERVER_MODE == "development":
+                    raise
         return saved_name  # includes the path!
 
     def deconstruct(self):
         path, args, kwargs = super().deconstruct()
         kwargs["kind"] = ""  # don't record "kind" in migrations
         return path, args, kwargs
+
+
+class MetadataFile(File):
+    """File that includes metadata"""
+
+    def __init__(self, file, name=None, mtime: Optional[datetime.datetime]=None, content_type=""):
+        super().__init__(file=file, name=name)
+        self.mtime = mtime
+        self.content_type = content_type
+        self._custom_metadata = None
+
+    @property
+    def custom_metadata(self):
+        if self._custom_metadata is None:
+            self._custom_metadata = self._compute_custom_metadata()
+        return self._custom_metadata
+
+    def _compute_custom_metadata(self):
+        try:
+            self.file.seek(0)
+        except AttributeError:  # TODO-BLOBSTORE
+            raise NotImplementedError("cannot handle unseekable content")
+        content_bytes = self.file.read()
+        if not isinstance(
+            content_bytes, bytes
+        ):  # TODO-BLOBSTORE: This is sketch-development only -remove before committing
+            raise Exception(f"Expected bytes - got {type(content_bytes)}")
+        self.file.seek(0)
+        return {
+            "len": f"{len(content_bytes)}",
+            "sha384": f"{sha384(content_bytes).hexdigest()}",
+            "mtime": None if self.mtime is None else self.mtime.isoformat(),
+        }
diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py
index c06e7876db..bfe5a56597 100644
--- a/ietf/utils/test_runner.py
+++ b/ietf/utils/test_runner.py
@@ -1268,7 +1268,7 @@ class TestBlobstoreManager():
     )
 
     def createTestBlobstores(self):
-        for storagename in settings.MORE_STORAGE_NAMES:
+        for storagename in settings.ARTIFACT_STORAGE_NAMES:
             bucketname = f"test-{storagename}"
             try:
                 bucket = self.blobstore.create_bucket(Bucket=bucketname)
diff --git a/k8s/beat.yaml b/k8s/beat.yaml
index 9a8fe2f0a4..cc98beecf6 100644
--- a/k8s/beat.yaml
+++ b/k8s/beat.yaml
@@ -36,10 +36,6 @@ spec:
         - name: beat
           image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
           imagePullPolicy: Always
-          ports:
-            - containerPort: 8000
-              name: http
-              protocol: TCP
           volumeMounts:
             - name: dt-vol
               mountPath: /a
@@ -63,7 +59,7 @@ spec:
             runAsUser: 1000
             runAsGroup: 1000
       volumes:
-        # To be overriden with the actual shared volume
+        # To be overridden with the actual shared volume
         - name: dt-vol
         - name: dt-tmp
           emptyDir:
diff --git a/k8s/celery.yaml b/k8s/celery.yaml
index 15f3bf0c7c..a2799f2a6d 100644
--- a/k8s/celery.yaml
+++ b/k8s/celery.yaml
@@ -36,10 +36,6 @@ spec:
         - name: celery
           image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
           imagePullPolicy: Always
-          ports:
-            - containerPort: 8000
-              name: http
-              protocol: TCP
           volumeMounts:
             - name: dt-vol
               mountPath: /a
@@ -92,7 +88,7 @@ spec:
             runAsUser: 65534 # "nobody" user by default
             runAsGroup: 65534  # "nogroup" group by default
       volumes:
-        # To be overriden with the actual shared volume
+        # To be overridden with the actual shared volume
         - name: dt-vol
         - name: dt-tmp
           emptyDir:
diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml
index 2b623da2bd..769cb03517 100644
--- a/k8s/kustomization.yaml
+++ b/k8s/kustomization.yaml
@@ -14,3 +14,4 @@ resources:
   - datatracker.yaml
   - memcached.yaml
   - rabbitmq.yaml
+  - replicator.yaml
diff --git a/k8s/replicator.yaml b/k8s/replicator.yaml
new file mode 100644
index 0000000000..9c462bd96b
--- /dev/null
+++ b/k8s/replicator.yaml
@@ -0,0 +1,82 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: replicator
+  labels:
+    deleteBeforeUpgrade: yes
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: replicator
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: replicator
+    spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - datatracker
+              topologyKey: "kubernetes.io/hostname"
+      securityContext:
+        runAsNonRoot: true
+      containers:
+        # -----------------------------------------------------
+        # Celery Container
+        # -----------------------------------------------------
+        - name: celery
+          image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: dt-vol
+              mountPath: /a
+            - name: dt-tmp
+              mountPath: /tmp
+            - name: dt-home
+              mountPath: /home/datatracker
+            - name: dt-xml2rfc-cache
+              mountPath: /var/cache/xml2rfc
+            - name: dt-cfg
+              mountPath: /workspace/ietf/settings_local.py
+              subPath: settings_local.py
+          env:
+            - name: "CONTAINER_ROLE"
+              value: "replicator"
+          envFrom:
+            - secretRef:
+                name: dt-secrets-env
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            readOnlyRootFilesystem: true
+            runAsUser: 1000
+            runAsGroup: 1000
+      volumes:
+        # To be overridden with the actual shared volume
+        - name: dt-vol
+        - name: dt-tmp
+          emptyDir:
+            sizeLimit: "2Gi"
+        - name: dt-xml2rfc-cache
+          emptyDir:
+            sizeLimit: "2Gi"
+        - name: dt-home
+          emptyDir:
+            sizeLimit: "2Gi"
+        - name: dt-cfg
+          configMap:
+            name: files-cfgmap
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 600
diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 0f4ecaa631..074888728f 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -7,7 +7,13 @@
 
 from ietf import __release_hash__
 from ietf.settings import *  # pyflakes:ignore
-from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS
+from ietf.settings import (
+    STORAGES,
+    ARTIFACT_STORAGE_NAMES,
+    BLOBSTORAGE_CONNECT_TIMEOUT,
+    BLOBSTORAGE_READ_TIMEOUT,
+    BLOBSTORAGE_MAX_ATTEMPTS,
+)
 import botocore.config
 
 
@@ -92,8 +98,20 @@ def _multiline_to_list(s):
         "PASSWORD": os.environ.get("DATATRACKER_DB_PASS", ""),
         "OPTIONS": json.loads(os.environ.get("DATATRACKER_DB_OPTS_JSON", "{}")),
     },
+    "blobdb": {
+        "HOST": os.environ.get("BLOBDB_DB_HOST", "blobdb"),
+        "PORT": os.environ.get("BLOBDB_DB_PORT", "5432"),
+        "NAME": os.environ.get("BLOBDB_DB_NAME", "blob"),
+        "ENGINE": "django.db.backends.postgresql",
+        "USER": os.environ.get("BLOBDB_DB_USER", "django"),
+        "PASSWORD": os.environ.get("BLOBDB_DB_PASS", ""),
+        "OPTIONS": json.loads(os.environ.get("BLOBDB_DB_OPTS_JSON", "{}")),
+    },
 }
 
+DATABASE_ROUTERS = ["ietf.blobdb.routers.BlobdbStorageRouter"]
+BLOBDB_DATABASE = "blobdb"
+
 # Configure persistent connections. A setting of 0 is Django's default.
 _conn_max_age = os.environ.get("DATATRACKER_DB_CONN_MAX_AGE", "0")
 # A string "none" means unlimited age.
@@ -129,7 +147,9 @@ def _multiline_to_list(s):
 )
 
 # mailarchive API key
-_mailing_list_archive_api_key = os.environ.get("DATATRACKER_MAILING_LIST_ARCHIVE_API_KEY", None)
+_mailing_list_archive_api_key = os.environ.get(
+    "DATATRACKER_MAILING_LIST_ARCHIVE_API_KEY", None
+)
 if _mailing_list_archive_api_key is None:
     raise RuntimeError("DATATRACKER_MAILING_LIST_ARCHIVE_API_KEY must be set")
 MAILING_LIST_ARCHIVE_API_KEY = _mailing_list_archive_api_key
@@ -178,7 +198,7 @@ def _multiline_to_list(s):
 # paste the encoded secret into stdin. Copy/paste that into an editor you trust not
 # to leave a copy lying around. When done editing, copy/paste the final JSON through
 #    jq -c | base64
-# and copy/paste the output into the secret store. 
+# and copy/paste the output into the secret store.
 if "DATATRACKER_APP_API_TOKENS_JSON_B64" in os.environ:
     if "DATATRACKER_APP_API_TOKENS_JSON" in os.environ:
         raise RuntimeError(
@@ -307,7 +327,7 @@ def _multiline_to_list(s):
 # Console logs as JSON instead of plain when running in k8s
 LOGGING["handlers"]["console"]["formatter"] = "json"
 
-# Configure storages for the blob store
+# Configure storages for the replica blob store
 _blob_store_endpoint_url = os.environ.get("DATATRACKER_BLOB_STORE_ENDPOINT_URL")
 _blob_store_access_key = os.environ.get("DATATRACKER_BLOB_STORE_ACCESS_KEY")
 _blob_store_secret_key = os.environ.get("DATATRACKER_BLOB_STORE_SECRET_KEY")
@@ -316,9 +336,7 @@ def _multiline_to_list(s):
         "All of DATATRACKER_BLOB_STORE_ENDPOINT_URL, DATATRACKER_BLOB_STORE_ACCESS_KEY, "
         "and DATATRACKER_BLOB_STORE_SECRET_KEY must be set"
     )
-_blob_store_bucket_prefix = os.environ.get(
-    "DATATRACKER_BLOB_STORE_BUCKET_PREFIX", ""
-)
+_blob_store_bucket_prefix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "")
 _blob_store_enable_profiling = (
     os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true"
 )
@@ -326,14 +344,20 @@ def _multiline_to_list(s):
     os.environ.get("DATATRACKER_BLOB_STORE_MAX_ATTEMPTS", BLOBSTORAGE_MAX_ATTEMPTS)
 )
 _blob_store_connect_timeout = float(
-    os.environ.get("DATATRACKER_BLOB_STORE_CONNECT_TIMEOUT", BLOBSTORAGE_CONNECT_TIMEOUT)
+    os.environ.get(
+        "DATATRACKER_BLOB_STORE_CONNECT_TIMEOUT", BLOBSTORAGE_CONNECT_TIMEOUT
+    )
 )
 _blob_store_read_timeout = float(
     os.environ.get("DATATRACKER_BLOB_STORE_READ_TIMEOUT", BLOBSTORAGE_READ_TIMEOUT)
 )
-for storage_name in MORE_STORAGE_NAMES:
-    STORAGES[storage_name] = {
-        "BACKEND": "ietf.doc.storage_backends.CustomS3Storage",
+
+for storagename in ARTIFACT_STORAGE_NAMES:
+    if storagename in ["staging"]:
+        continue
+    replica_storagename = f"r2-{storagename}"
+    STORAGES[replica_storagename] = {
+        "BACKEND": "ietf.doc.storage.MetadataS3Storage",
         "OPTIONS": dict(
             endpoint_url=_blob_store_endpoint_url,
             access_key=_blob_store_access_key,
@@ -345,7 +369,25 @@ def _multiline_to_list(s):
                 read_timeout=_blob_store_read_timeout,
                 retries={"total_max_attempts": _blob_store_max_attempts},
             ),
-            bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(),
+            verify=False,
+            bucket_name=f"{_blob_store_bucket_prefix}{storagename}".strip(),
             ietf_log_blob_timing=_blob_store_enable_profiling,
         ),
     }
+
+# Configure the blobdb app for artifact storage
+_blobdb_replication_enabled = (
+    os.environ.get("DATATRACKER_BLOBDB_REPLICATION_ENABLED", "true").lower() == "true"
+)
+_blobdb_replication_verbose_logging = (
+    os.environ.get("DATATRACKER_BLOBDB_REPLICATION_VERBOSE_LOGGING", "false").lower()
+    == "true"
+)
+
+BLOBDB_REPLICATION = {
+    "ENABLED": _blobdb_replication_enabled,
+    "DEST_STORAGE_PATTERN": "r2-{bucket}",
+    "INCLUDE_BUCKETS": ARTIFACT_STORAGE_NAMES,
+    "EXCLUDE_BUCKETS": ["staging"],
+    "VERBOSE_LOGGING": _blobdb_replication_verbose_logging,
+}

From 7e9cd9c79caedd376e4671cf55206bd7dc409fdf Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 12 Jun 2025 16:56:51 -0300
Subject: [PATCH 351/656] chore: lint, lint everywhere (#8995)

---
 ietf/settings_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/settings_test.py b/ietf/settings_test.py
index 644a6d5b16..b5da9b833b 100755
--- a/ietf/settings_test.py
+++ b/ietf/settings_test.py
@@ -50,7 +50,7 @@ def __getitem__(self, item):
 
 # test with a single DB - do not use a DB router
 BLOBDB_DATABASE = "default"
-DATABASE_ROUTERS: list[str] = []
+DATABASE_ROUTERS = []  # type: ignore
 
 if TEST_CODE_COVERAGE_CHECKER and not TEST_CODE_COVERAGE_CHECKER._started: # pyflakes:ignore
     TEST_CODE_COVERAGE_CHECKER.start()                          # pyflakes:ignore

From 1377503077c2acd22e0d7fee2757c0308ff249ff Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 14 Jun 2025 01:54:42 -0400
Subject: [PATCH 352/656] chore(deps): bump @actions/github (#8894)

Bumps the npm group in /dev/coverage-action with 1 update: [@actions/github](https://github.com/actions/toolkit/tree/HEAD/packages/github).


Updates `@actions/github` from 6.0.0 to 6.0.1
- [Changelog](https://github.com/actions/toolkit/blob/main/packages/github/RELEASES.md)
- [Commits](https://github.com/actions/toolkit/commits/HEAD/packages/github)

---
updated-dependencies:
- dependency-name: "@actions/github"
  dependency-version: 6.0.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/coverage-action/package-lock.json | 326 ++++++++++++++++----------
 dev/coverage-action/package.json      |   2 +-
 2 files changed, 197 insertions(+), 131 deletions(-)

diff --git a/dev/coverage-action/package-lock.json b/dev/coverage-action/package-lock.json
index 33aed07072..4a1ff1e03d 100644
--- a/dev/coverage-action/package-lock.json
+++ b/dev/coverage-action/package-lock.json
@@ -10,7 +10,7 @@
       "license": "BSD-3-Clause",
       "dependencies": {
         "@actions/core": "1.11.1",
-        "@actions/github": "6.0.0",
+        "@actions/github": "6.0.1",
         "lodash": "4.17.21",
         "luxon": "3.6.1"
       }
@@ -33,14 +33,18 @@
       }
     },
     "node_modules/@actions/github": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.0.tgz",
-      "integrity": "sha512-alScpSVnYmjNEXboZjarjukQEzgCRmjMv6Xj47fsdnqGS73bjJNDpiiXmp8jr0UZLdUB6d9jW63IcmddUP+l0g==",
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.1.tgz",
+      "integrity": "sha512-xbZVcaqD4XnQAe35qSQqskb3SqIAfRyLBrHMd/8TuL7hJSz2QtbDwnNM8zWx4zO5l2fnGtseNE3MbEvD7BxVMw==",
+      "license": "MIT",
       "dependencies": {
         "@actions/http-client": "^2.2.0",
         "@octokit/core": "^5.0.1",
-        "@octokit/plugin-paginate-rest": "^9.0.0",
-        "@octokit/plugin-rest-endpoint-methods": "^10.0.0"
+        "@octokit/plugin-paginate-rest": "^9.2.2",
+        "@octokit/plugin-rest-endpoint-methods": "^10.4.0",
+        "@octokit/request": "^8.4.1",
+        "@octokit/request-error": "^5.1.1",
+        "undici": "^5.28.5"
       }
     },
     "node_modules/@actions/http-client": {
@@ -69,20 +73,22 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz",
       "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==",
+      "license": "MIT",
       "engines": {
         "node": ">= 18"
       }
     },
     "node_modules/@octokit/core": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.0.1.tgz",
-      "integrity": "sha512-lyeeeZyESFo+ffI801SaBKmCfsvarO+dgV8/0gD8u1d87clbEdWsP5yC+dSj3zLhb2eIf5SJrn6vDz9AheETHw==",
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.1.tgz",
+      "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==",
+      "license": "MIT",
       "dependencies": {
         "@octokit/auth-token": "^4.0.0",
-        "@octokit/graphql": "^7.0.0",
-        "@octokit/request": "^8.0.2",
-        "@octokit/request-error": "^5.0.0",
-        "@octokit/types": "^12.0.0",
+        "@octokit/graphql": "^7.1.0",
+        "@octokit/request": "^8.4.1",
+        "@octokit/request-error": "^5.1.1",
+        "@octokit/types": "^13.0.0",
         "before-after-hook": "^2.2.0",
         "universal-user-agent": "^6.0.0"
       },
@@ -91,12 +97,12 @@
       }
     },
     "node_modules/@octokit/endpoint": {
-      "version": "9.0.1",
-      "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.1.tgz",
-      "integrity": "sha512-hRlOKAovtINHQPYHZlfyFwaM8OyetxeoC81lAkBy34uLb8exrZB50SQdeW3EROqiY9G9yxQTpp5OHTV54QD+vA==",
+      "version": "9.0.6",
+      "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz",
+      "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==",
+      "license": "MIT",
       "dependencies": {
-        "@octokit/types": "^12.0.0",
-        "is-plain-object": "^5.0.0",
+        "@octokit/types": "^13.1.0",
         "universal-user-agent": "^6.0.0"
       },
       "engines": {
@@ -104,12 +110,13 @@
       }
     },
     "node_modules/@octokit/graphql": {
-      "version": "7.0.2",
-      "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.2.tgz",
-      "integrity": "sha512-OJ2iGMtj5Tg3s6RaXH22cJcxXRi7Y3EBqbHTBRq+PQAqfaS8f/236fUrWhfSn8P4jovyzqucxme7/vWSSZBX2Q==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz",
+      "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==",
+      "license": "MIT",
       "dependencies": {
-        "@octokit/request": "^8.0.1",
-        "@octokit/types": "^12.0.0",
+        "@octokit/request": "^8.4.1",
+        "@octokit/types": "^13.0.0",
         "universal-user-agent": "^6.0.0"
       },
       "engines": {
@@ -117,47 +124,80 @@
       }
     },
     "node_modules/@octokit/openapi-types": {
-      "version": "19.0.0",
-      "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.0.0.tgz",
-      "integrity": "sha512-PclQ6JGMTE9iUStpzMkwLCISFn/wDeRjkZFIKALpvJQNBGwDoYYi2fFvuHwssoQ1rXI5mfh6jgTgWuddeUzfWw=="
+      "version": "24.2.0",
+      "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz",
+      "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==",
+      "license": "MIT"
     },
     "node_modules/@octokit/plugin-paginate-rest": {
-      "version": "9.0.0",
-      "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.0.0.tgz",
-      "integrity": "sha512-oIJzCpttmBTlEhBmRvb+b9rlnGpmFgDtZ0bB6nq39qIod6A5DP+7RkVLMOixIgRCYSHDTeayWqmiJ2SZ6xgfdw==",
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.2.tgz",
+      "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==",
+      "license": "MIT",
       "dependencies": {
-        "@octokit/types": "^12.0.0"
+        "@octokit/types": "^12.6.0"
       },
       "engines": {
         "node": ">= 18"
       },
       "peerDependencies": {
-        "@octokit/core": ">=5"
+        "@octokit/core": "5"
+      }
+    },
+    "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/openapi-types": {
+      "version": "20.0.0",
+      "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz",
+      "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==",
+      "license": "MIT"
+    },
+    "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/types": {
+      "version": "12.6.0",
+      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz",
+      "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==",
+      "license": "MIT",
+      "dependencies": {
+        "@octokit/openapi-types": "^20.0.0"
       }
     },
     "node_modules/@octokit/plugin-rest-endpoint-methods": {
-      "version": "10.0.1",
-      "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.0.1.tgz",
-      "integrity": "sha512-fgS6HPkPvJiz8CCliewLyym9qAx0RZ/LKh3sATaPfM41y/O2wQ4Z9MrdYeGPVh04wYmHFmWiGlKPC7jWVtZXQA==",
+      "version": "10.4.1",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz",
+      "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==",
+      "license": "MIT",
       "dependencies": {
-        "@octokit/types": "^12.0.0"
+        "@octokit/types": "^12.6.0"
       },
       "engines": {
         "node": ">= 18"
       },
       "peerDependencies": {
-        "@octokit/core": ">=5"
+        "@octokit/core": "5"
+      }
+    },
+    "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/openapi-types": {
+      "version": "20.0.0",
+      "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz",
+      "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==",
+      "license": "MIT"
+    },
+    "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/types": {
+      "version": "12.6.0",
+      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz",
+      "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==",
+      "license": "MIT",
+      "dependencies": {
+        "@octokit/openapi-types": "^20.0.0"
       }
     },
     "node_modules/@octokit/request": {
-      "version": "8.1.4",
-      "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.4.tgz",
-      "integrity": "sha512-M0aaFfpGPEKrg7XoA/gwgRvc9MSXHRO2Ioki1qrPDbl1e9YhjIwVoHE7HIKmv/m3idzldj//xBujcFNqGX6ENA==",
+      "version": "8.4.1",
+      "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz",
+      "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==",
+      "license": "MIT",
       "dependencies": {
-        "@octokit/endpoint": "^9.0.0",
-        "@octokit/request-error": "^5.0.0",
-        "@octokit/types": "^12.0.0",
-        "is-plain-object": "^5.0.0",
+        "@octokit/endpoint": "^9.0.6",
+        "@octokit/request-error": "^5.1.1",
+        "@octokit/types": "^13.1.0",
         "universal-user-agent": "^6.0.0"
       },
       "engines": {
@@ -165,11 +205,12 @@
       }
     },
     "node_modules/@octokit/request-error": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.1.tgz",
-      "integrity": "sha512-X7pnyTMV7MgtGmiXBwmO6M5kIPrntOXdyKZLigNfQWSEQzVxR4a4vo49vJjTWX70mPndj8KhfT4Dx+2Ng3vnBQ==",
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz",
+      "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==",
+      "license": "MIT",
       "dependencies": {
-        "@octokit/types": "^12.0.0",
+        "@octokit/types": "^13.1.0",
         "deprecation": "^2.0.0",
         "once": "^1.4.0"
       },
@@ -178,30 +219,25 @@
       }
     },
     "node_modules/@octokit/types": {
-      "version": "12.0.0",
-      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.0.0.tgz",
-      "integrity": "sha512-EzD434aHTFifGudYAygnFlS1Tl6KhbTynEWELQXIbTY8Msvb5nEqTZIm7sbPEt4mQYLZwu3zPKVdeIrw0g7ovg==",
+      "version": "13.10.0",
+      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz",
+      "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==",
+      "license": "MIT",
       "dependencies": {
-        "@octokit/openapi-types": "^19.0.0"
+        "@octokit/openapi-types": "^24.2.0"
       }
     },
     "node_modules/before-after-hook": {
       "version": "2.2.3",
       "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz",
-      "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ=="
+      "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==",
+      "license": "Apache-2.0"
     },
     "node_modules/deprecation": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
-      "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ=="
-    },
-    "node_modules/is-plain-object": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz",
-      "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==",
-      "engines": {
-        "node": ">=0.10.0"
-      }
+      "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==",
+      "license": "ISC"
     },
     "node_modules/lodash": {
       "version": "4.17.21",
@@ -220,7 +256,8 @@
     "node_modules/once": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+      "license": "ISC",
       "dependencies": {
         "wrappy": "1"
       }
@@ -234,9 +271,10 @@
       }
     },
     "node_modules/undici": {
-      "version": "5.26.4",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-5.26.4.tgz",
-      "integrity": "sha512-OG+QOf0fTLtazL9P9X7yqWxQ+Z0395Wk6DSkyTxtaq3wQEjIroVe7Y4asCX/vcCxYpNGMnwz8F0qbRYUoaQVMw==",
+      "version": "5.29.0",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz",
+      "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==",
+      "license": "MIT",
       "dependencies": {
         "@fastify/busboy": "^2.0.0"
       },
@@ -245,14 +283,16 @@
       }
     },
     "node_modules/universal-user-agent": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz",
-      "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w=="
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz",
+      "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==",
+      "license": "ISC"
     },
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
+      "license": "ISC"
     }
   },
   "dependencies": {
@@ -274,14 +314,17 @@
       }
     },
     "@actions/github": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.0.tgz",
-      "integrity": "sha512-alScpSVnYmjNEXboZjarjukQEzgCRmjMv6Xj47fsdnqGS73bjJNDpiiXmp8jr0UZLdUB6d9jW63IcmddUP+l0g==",
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.1.tgz",
+      "integrity": "sha512-xbZVcaqD4XnQAe35qSQqskb3SqIAfRyLBrHMd/8TuL7hJSz2QtbDwnNM8zWx4zO5l2fnGtseNE3MbEvD7BxVMw==",
       "requires": {
         "@actions/http-client": "^2.2.0",
         "@octokit/core": "^5.0.1",
-        "@octokit/plugin-paginate-rest": "^9.0.0",
-        "@octokit/plugin-rest-endpoint-methods": "^10.0.0"
+        "@octokit/plugin-paginate-rest": "^9.2.2",
+        "@octokit/plugin-rest-endpoint-methods": "^10.4.0",
+        "@octokit/request": "^8.4.1",
+        "@octokit/request-error": "^5.1.1",
+        "undici": "^5.28.5"
       }
     },
     "@actions/http-client": {
@@ -309,88 +352,116 @@
       "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA=="
     },
     "@octokit/core": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.0.1.tgz",
-      "integrity": "sha512-lyeeeZyESFo+ffI801SaBKmCfsvarO+dgV8/0gD8u1d87clbEdWsP5yC+dSj3zLhb2eIf5SJrn6vDz9AheETHw==",
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.1.tgz",
+      "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==",
       "requires": {
         "@octokit/auth-token": "^4.0.0",
-        "@octokit/graphql": "^7.0.0",
-        "@octokit/request": "^8.0.2",
-        "@octokit/request-error": "^5.0.0",
-        "@octokit/types": "^12.0.0",
+        "@octokit/graphql": "^7.1.0",
+        "@octokit/request": "^8.4.1",
+        "@octokit/request-error": "^5.1.1",
+        "@octokit/types": "^13.0.0",
         "before-after-hook": "^2.2.0",
         "universal-user-agent": "^6.0.0"
       }
     },
     "@octokit/endpoint": {
-      "version": "9.0.1",
-      "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.1.tgz",
-      "integrity": "sha512-hRlOKAovtINHQPYHZlfyFwaM8OyetxeoC81lAkBy34uLb8exrZB50SQdeW3EROqiY9G9yxQTpp5OHTV54QD+vA==",
+      "version": "9.0.6",
+      "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz",
+      "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==",
       "requires": {
-        "@octokit/types": "^12.0.0",
-        "is-plain-object": "^5.0.0",
+        "@octokit/types": "^13.1.0",
         "universal-user-agent": "^6.0.0"
       }
     },
     "@octokit/graphql": {
-      "version": "7.0.2",
-      "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.2.tgz",
-      "integrity": "sha512-OJ2iGMtj5Tg3s6RaXH22cJcxXRi7Y3EBqbHTBRq+PQAqfaS8f/236fUrWhfSn8P4jovyzqucxme7/vWSSZBX2Q==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz",
+      "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==",
       "requires": {
-        "@octokit/request": "^8.0.1",
-        "@octokit/types": "^12.0.0",
+        "@octokit/request": "^8.4.1",
+        "@octokit/types": "^13.0.0",
         "universal-user-agent": "^6.0.0"
       }
     },
     "@octokit/openapi-types": {
-      "version": "19.0.0",
-      "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.0.0.tgz",
-      "integrity": "sha512-PclQ6JGMTE9iUStpzMkwLCISFn/wDeRjkZFIKALpvJQNBGwDoYYi2fFvuHwssoQ1rXI5mfh6jgTgWuddeUzfWw=="
+      "version": "24.2.0",
+      "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz",
+      "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg=="
     },
     "@octokit/plugin-paginate-rest": {
-      "version": "9.0.0",
-      "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.0.0.tgz",
-      "integrity": "sha512-oIJzCpttmBTlEhBmRvb+b9rlnGpmFgDtZ0bB6nq39qIod6A5DP+7RkVLMOixIgRCYSHDTeayWqmiJ2SZ6xgfdw==",
+      "version": "9.2.2",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.2.tgz",
+      "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==",
       "requires": {
-        "@octokit/types": "^12.0.0"
+        "@octokit/types": "^12.6.0"
+      },
+      "dependencies": {
+        "@octokit/openapi-types": {
+          "version": "20.0.0",
+          "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz",
+          "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA=="
+        },
+        "@octokit/types": {
+          "version": "12.6.0",
+          "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz",
+          "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==",
+          "requires": {
+            "@octokit/openapi-types": "^20.0.0"
+          }
+        }
       }
     },
     "@octokit/plugin-rest-endpoint-methods": {
-      "version": "10.0.1",
-      "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.0.1.tgz",
-      "integrity": "sha512-fgS6HPkPvJiz8CCliewLyym9qAx0RZ/LKh3sATaPfM41y/O2wQ4Z9MrdYeGPVh04wYmHFmWiGlKPC7jWVtZXQA==",
+      "version": "10.4.1",
+      "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz",
+      "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==",
       "requires": {
-        "@octokit/types": "^12.0.0"
+        "@octokit/types": "^12.6.0"
+      },
+      "dependencies": {
+        "@octokit/openapi-types": {
+          "version": "20.0.0",
+          "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz",
+          "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA=="
+        },
+        "@octokit/types": {
+          "version": "12.6.0",
+          "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz",
+          "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==",
+          "requires": {
+            "@octokit/openapi-types": "^20.0.0"
+          }
+        }
       }
     },
     "@octokit/request": {
-      "version": "8.1.4",
-      "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.4.tgz",
-      "integrity": "sha512-M0aaFfpGPEKrg7XoA/gwgRvc9MSXHRO2Ioki1qrPDbl1e9YhjIwVoHE7HIKmv/m3idzldj//xBujcFNqGX6ENA==",
+      "version": "8.4.1",
+      "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz",
+      "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==",
       "requires": {
-        "@octokit/endpoint": "^9.0.0",
-        "@octokit/request-error": "^5.0.0",
-        "@octokit/types": "^12.0.0",
-        "is-plain-object": "^5.0.0",
+        "@octokit/endpoint": "^9.0.6",
+        "@octokit/request-error": "^5.1.1",
+        "@octokit/types": "^13.1.0",
         "universal-user-agent": "^6.0.0"
       }
     },
     "@octokit/request-error": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.1.tgz",
-      "integrity": "sha512-X7pnyTMV7MgtGmiXBwmO6M5kIPrntOXdyKZLigNfQWSEQzVxR4a4vo49vJjTWX70mPndj8KhfT4Dx+2Ng3vnBQ==",
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz",
+      "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==",
       "requires": {
-        "@octokit/types": "^12.0.0",
+        "@octokit/types": "^13.1.0",
         "deprecation": "^2.0.0",
         "once": "^1.4.0"
       }
     },
     "@octokit/types": {
-      "version": "12.0.0",
-      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.0.0.tgz",
-      "integrity": "sha512-EzD434aHTFifGudYAygnFlS1Tl6KhbTynEWELQXIbTY8Msvb5nEqTZIm7sbPEt4mQYLZwu3zPKVdeIrw0g7ovg==",
+      "version": "13.10.0",
+      "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz",
+      "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==",
       "requires": {
-        "@octokit/openapi-types": "^19.0.0"
+        "@octokit/openapi-types": "^24.2.0"
       }
     },
     "before-after-hook": {
@@ -403,11 +474,6 @@
       "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
       "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ=="
     },
-    "is-plain-object": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz",
-      "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q=="
-    },
     "lodash": {
       "version": "4.17.21",
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
@@ -421,7 +487,7 @@
     "once": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
       "requires": {
         "wrappy": "1"
       }
@@ -432,22 +498,22 @@
       "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="
     },
     "undici": {
-      "version": "5.26.4",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-5.26.4.tgz",
-      "integrity": "sha512-OG+QOf0fTLtazL9P9X7yqWxQ+Z0395Wk6DSkyTxtaq3wQEjIroVe7Y4asCX/vcCxYpNGMnwz8F0qbRYUoaQVMw==",
+      "version": "5.29.0",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz",
+      "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==",
       "requires": {
         "@fastify/busboy": "^2.0.0"
       }
     },
     "universal-user-agent": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz",
-      "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w=="
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz",
+      "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ=="
     },
     "wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
     }
   }
 }
diff --git a/dev/coverage-action/package.json b/dev/coverage-action/package.json
index 9e1433f1ce..d59a8de22c 100644
--- a/dev/coverage-action/package.json
+++ b/dev/coverage-action/package.json
@@ -7,7 +7,7 @@
   "license": "BSD-3-Clause",
   "dependencies": {
     "@actions/core": "1.11.1",
-    "@actions/github": "6.0.0",
+    "@actions/github": "6.0.1",
     "lodash": "4.17.21",
     "luxon": "3.6.1"
   }

From d5b07d1a205b0ac8ca610cc3efcab64314fb4dd4 Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Wed, 18 Jun 2025 10:55:19 -0700
Subject: [PATCH 353/656] fix: change related_email API to return all addresses
 (#9013)

---
 ietf/api/tests.py | 2 +-
 ietf/api/views.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 6754f5c3de..29edd99b2b 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -1161,7 +1161,7 @@ def test_related_email_list(self):
         self.assertEqual(r.headers["Content-Type"], "application/json")
         result = json.loads(r.content)
         self.assertCountEqual(result.keys(), ["addresses"])
-        self.assertCountEqual(result["addresses"], joe.person.email_set.exclude(address='joe@home.com').values_list("address", flat=True))
+        self.assertCountEqual(result["addresses"], joe.person.email_set.values_list("address", flat=True))
         # non-ascii
         non_ascii_url = urlreverse("ietf.api.views.related_email_list", kwargs={'email': 'jòe@spain.com'})
         r = self.client.get(non_ascii_url, headers={"X-Api-Key": "valid-token"})
diff --git a/ietf/api/views.py b/ietf/api/views.py
index 06a386b2e3..e8e38b25b4 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -618,7 +618,7 @@ def _http_err(code, text):
             return JsonResponse({"addresses": []})
         return JsonResponse(
             {
-                "addresses": list(person.email_set.exclude(address=email).values_list("address", flat=True)),
+                "addresses": list(person.email_set.values_list("address", flat=True)),
             }
         )
     return HttpResponse(status=405)

From 0c8171c17b112f62e53fb496fe45f4776d20811e Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 18 Jun 2025 14:57:26 -0300
Subject: [PATCH 354/656] feat: toggle all areas in schedule editor (#9012)

---
 ietf/static/js/edit-meeting-schedule.js           | 14 +++++++++++++-
 ietf/templates/meeting/edit_meeting_schedule.html |  5 +++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/ietf/static/js/edit-meeting-schedule.js b/ietf/static/js/edit-meeting-schedule.js
index 3f0afe0841..2a73a8c29d 100644
--- a/ietf/static/js/edit-meeting-schedule.js
+++ b/ietf/static/js/edit-meeting-schedule.js
@@ -50,6 +50,7 @@ $(function () {
     let sessionPurposeInputs = schedEditor.find('.session-purpose-toggles input');
     let timeSlotGroupInputs = schedEditor.find("#timeslot-group-toggles-modal .modal-body .individual-timeslots input");
     let sessionParentInputs = schedEditor.find(".session-parent-toggles input");
+    let sessionParentToggleAll = schedEditor.find(".session-parent-toggles .session-parent-toggle-all")
     const classes_to_hide = '.hidden-timeslot-group,.hidden-timeslot-type';
 
     // hack to work around lack of position sticky support in old browsers, see https://caniuse.com/#feat=css-sticky
@@ -769,6 +770,17 @@ $(function () {
     sessionParentInputs.on("click", updateSessionParentToggling);
     updateSessionParentToggling();
 
+    // Toggle _all_ session parents
+    function toggleAllSessionParents() {
+        if (sessionParentInputs.filter(":checked").length < sessionParentInputs.length) {
+            sessionParentInputs.prop("checked", true);
+        } else {
+            sessionParentInputs.prop("checked", false);
+        }
+        updateSessionParentToggling();
+    }
+    sessionParentToggleAll.on("click", toggleAllSessionParents);
+
     // Toggling timeslot types
     function updateTimeSlotTypeToggling() {
         const checkedTypes = jQuery.map(timeSlotTypeInputs.filter(":checked"), elt => elt.value);
@@ -1020,4 +1032,4 @@ $(function () {
         .on("mouseleave", ".other-session", function () {
             sessions.filter("#session" + this.dataset.othersessionid).removeClass("highlight");
         });
-});
\ No newline at end of file
+});
diff --git a/ietf/templates/meeting/edit_meeting_schedule.html b/ietf/templates/meeting/edit_meeting_schedule.html
index 462f97715d..a975c2e61a 100644
--- a/ietf/templates/meeting/edit_meeting_schedule.html
+++ b/ietf/templates/meeting/edit_meeting_schedule.html
@@ -223,6 +223,11 @@ <h1>
                                     {{ p.acronym }}
                                 </label>
                             {% endfor %}
+                            <button type="button"
+                                    class="btn btn-outline-primary btn-sm session-parent-toggle-all">
+                                Toggle All
+                            </button>
+                                    
                         </div>
                         <div class="my-3">
                         {% if session_purposes|length > 1 %}

From d0d69650299e8332775ca3cf51347d75343e2fa8 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 18 Jun 2025 22:24:50 -0500
Subject: [PATCH 355/656] chore: tweak test setup so blob storage is ok with
 overwrites (#9025)

---
 ietf/submit/tests.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index 6b1c998e76..7e70c55965 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -2981,7 +2981,7 @@ def test_process_and_accept_uploaded_submission_invalid(self):
         xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml'
         with xml_path.open('w') as f:
             f.write(xml_data)
-        store_str("staging", "draft-somebody-test-00.xml", xml_data)
+        store_str("staging", "draft-somebody-test-00.xml", xml_data, allow_overwrite=True)
         with mock.patch(
                 'ietf.submit.utils.apply_checkers',
                 side_effect = lambda _, __: submission.checks.create(
@@ -3047,25 +3047,25 @@ def test_process_submission_xml(self):
         # Should behave on missing or partial <date> elements
         TestBlobstoreManager().emptyTestBlobstores()
         xml_path.write_text(re.sub(r"<date.+>", "", xml_contents))  # strip <date...> entirely
-        store_str("staging", "draft-somebody-test-00.xml", re.sub(r"<date.+>", "", xml_contents))
+        store_str("staging", "draft-somebody-test-00.xml", re.sub(r"<date.+>", "", xml_contents), allow_overwrite=True)
         output = process_submission_xml("draft-somebody-test", "00")
         self.assertEqual(output["document_date"], None)
 
         TestBlobstoreManager().emptyTestBlobstores()
         xml_path.write_text(re.sub(r"<date year=.+ month", "<date month", xml_contents))  # remove year
-        store_str("staging", "draft-somebody-test-00.xml", re.sub(r"<date year=.+ month", "<date month", xml_contents))
+        store_str("staging", "draft-somebody-test-00.xml", re.sub(r"<date year=.+ month", "<date month", xml_contents), allow_overwrite=True)
         output = process_submission_xml("draft-somebody-test", "00")
         self.assertEqual(output["document_date"], date_today())
 
         TestBlobstoreManager().emptyTestBlobstores()
         xml_path.write_text(re.sub(r"(<date.+) month=.+day=(.+>)", r"\1 day=\2", xml_contents))  # remove month
-        store_str("staging", "draft-somebody-test-00.xml", re.sub(r"(<date.+) month=.+day=(.+>)", r"\1 day=\2", xml_contents))
+        store_str("staging", "draft-somebody-test-00.xml", re.sub(r"(<date.+) month=.+day=(.+>)", r"\1 day=\2", xml_contents), allow_overwrite=True)
         output = process_submission_xml("draft-somebody-test", "00")
         self.assertEqual(output["document_date"], date_today())
 
         TestBlobstoreManager().emptyTestBlobstores()
         xml_path.write_text(re.sub(r"<date(.+) day=.+>", r"<date\1>", xml_contents))  # remove day
-        store_str("staging", "draft-somebody-test-00.xml", re.sub(r"<date(.+) day=.+>", r"<date\1>", xml_contents))
+        store_str("staging", "draft-somebody-test-00.xml", re.sub(r"<date(.+) day=.+>", r"<date\1>", xml_contents), allow_overwrite=True)
         output = process_submission_xml("draft-somebody-test", "00")
         self.assertEqual(output["document_date"], date_today())
 
@@ -3080,7 +3080,7 @@ def test_process_submission_xml(self):
         )
         xml_path.write_text(xml.read())
         xml.seek(0)
-        store_str("staging", "draft-somebody-test-00.xml", xml.read())
+        store_str("staging", "draft-somebody-test-00.xml", xml.read(), allow_overwrite=True)
         with self.assertRaisesMessage(SubmissionError, "disagrees with submission filename"):
             process_submission_xml("draft-somebody-test", "00")
 
@@ -3095,7 +3095,7 @@ def test_process_submission_xml(self):
         )
         xml_path.write_text(xml.read())
         xml.seek(0)
-        store_str("staging", "draft-somebody-test-00.xml", xml.read())
+        store_str("staging", "draft-somebody-test-00.xml", xml.read(), allow_overwrite=True)
         with self.assertRaisesMessage(SubmissionError, "disagrees with submission revision"):
             process_submission_xml("draft-somebody-test", "00")
 
@@ -3110,7 +3110,7 @@ def test_process_submission_xml(self):
         )
         xml_path.write_text(xml.read())
         xml.seek(0)
-        store_str("staging", "draft-somebody-test-00.xml", xml.read())
+        store_str("staging", "draft-somebody-test-00.xml", xml.read(), allow_overwrite=True)
         with self.assertRaisesMessage(SubmissionError, "Could not extract a valid title"):
             process_submission_xml("draft-somebody-test", "00")
 
@@ -3153,7 +3153,7 @@ def test_process_submission_text(self):
         with txt_path.open('w') as fd:
             fd.write(txt.read())
         txt.seek(0)
-        store_str("staging", "draft-somebody-test-00.txt", txt.read())
+        store_str("staging", "draft-somebody-test-00.txt", txt.read(), allow_overwrite=True)
         txt.close()
         with self.assertRaisesMessage(SubmissionError, 'disagrees with submission filename'):
             process_submission_text("draft-somebody-test", "00")
@@ -3170,7 +3170,7 @@ def test_process_submission_text(self):
         with txt_path.open('w') as fd:
             fd.write(txt.read())
         txt.seek(0)
-        store_str("staging", "draft-somebody-test-00.txt", txt.read())
+        store_str("staging", "draft-somebody-test-00.txt", txt.read(), allow_overwrite=True)
         txt.close()
         with self.assertRaisesMessage(SubmissionError, 'disagrees with submission revision'):
             process_submission_text("draft-somebody-test", "00")

From 8e082b698b39c3a658f04185951a1bb3be1fb854 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 19 Jun 2025 00:25:43 -0300
Subject: [PATCH 356/656] fix: charset for telechat_agenda_content_view (#8967)

* fix: charset for telechat_agenda_content_view

* refactor: remove explicit encoding

HttpResponse knows how to read content_type and complies
with the specified encoding.
---
 ietf/iesg/tests.py | 2 +-
 ietf/iesg/views.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py
index 86910bc0ce..746ea3f56f 100644
--- a/ietf/iesg/tests.py
+++ b/ietf/iesg/tests.py
@@ -759,7 +759,7 @@ def test_telechat_agenda_content_view(self):
                 urlreverse("ietf.iesg.views.telechat_agenda_content_view", kwargs={"section": section})
             )
             self.assertContains(r, content, status_code=200)
-            self.assertEqual(r.get("Content-Type", None), "text/plain")
+            self.assertEqual(r.get("Content-Type", None), "text/plain; charset=utf-8")
 
     def test_telechat_agenda_content_view_permissions(self):
         for section in TelechatAgendaSectionName.objects.filter(used=True).values_list("slug", flat=True):
diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py
index df02754f2e..f1fe77f763 100644
--- a/ietf/iesg/views.py
+++ b/ietf/iesg/views.py
@@ -610,4 +610,4 @@ def telechat_agenda_content_manage(request):
 @role_required("Secretariat", "IAB Chair", "Area Director")
 def telechat_agenda_content_view(request, section):
     content = get_object_or_404(TelechatAgendaContent, section__slug=section, section__used=True)
-    return HttpResponse(content=content.text, content_type="text/plain")
+    return HttpResponse(content=content.text, content_type="text/plain; charset=utf-8")

From 0968719f7482c1d3e23e3b60e6b1ced9294aaf77 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 19 Jun 2025 00:44:29 -0300
Subject: [PATCH 357/656] feat: secretariat can change statement state (#9026)

* feat: secretariat can change statement state

* style: ruff

* test: test statement state change

* chore: copyright statmenets

* chore: exercise new template with a GET so coverage sees it.

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/doc/forms.py                             | 13 ++++--
 ietf/doc/tests_statement.py                   | 35 ++++++++++++++-
 ietf/doc/urls.py                              |  3 +-
 ietf/doc/views_statement.py                   | 45 ++++++++++++++++++-
 ietf/templates/doc/document_statement.html    |  9 +++-
 .../doc/statement/change_statement_state.html | 22 +++++++++
 6 files changed, 118 insertions(+), 9 deletions(-)
 create mode 100644 ietf/templates/doc/statement/change_statement_state.html

diff --git a/ietf/doc/forms.py b/ietf/doc/forms.py
index 8a1e9ecb98..768d6f96af 100644
--- a/ietf/doc/forms.py
+++ b/ietf/doc/forms.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2013-2020, All Rights Reserved
+# Copyright The IETF Trust 2013-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -9,7 +9,7 @@
 from django.core.validators import validate_email
 
 from ietf.doc.fields import SearchableDocumentField, SearchableDocumentsField
-from ietf.doc.models import RelatedDocument, DocExtResource
+from ietf.doc.models import RelatedDocument, DocExtResource, State
 from ietf.iesg.models import TelechatDate
 from ietf.iesg.utils import telechat_page_count
 from ietf.person.fields import SearchablePersonField, SearchablePersonsField
@@ -61,7 +61,7 @@ class DocAuthorChangeBasisForm(forms.Form):
     basis = forms.CharField(max_length=255, 
                             label='Reason for change', 
                             help_text='What is the source or reasoning for the changes to the author list?')
-    
+
 class AdForm(forms.Form):
     ad = forms.ModelChoiceField(Person.objects.filter(role__name="ad", role__group__state="active", role__group__type='area').order_by('name'),
                                 label="Shepherding AD", empty_label="(None)", required=True)
@@ -288,3 +288,10 @@ def clean_name_fragment(self):
         if any(c in name_fragment for c in disallowed_characters):
             raise ValidationError(f"The following characters are disallowed: {', '.join(disallowed_characters)}")
         return name_fragment
+
+
+class ChangeStatementStateForm(forms.Form):
+    state = forms.ModelChoiceField(
+        State.objects.filter(used=True, type="statement"),
+        empty_label=None,
+    )
diff --git a/ietf/doc/tests_statement.py b/ietf/doc/tests_statement.py
index fea42b97d6..bb097b1828 100644
--- a/ietf/doc/tests_statement.py
+++ b/ietf/doc/tests_statement.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2023, All Rights Reserved
+# Copyright The IETF Trust 2023-2025, All Rights Reserved
 
 import debug  # pyflakes:ignore
 
@@ -372,3 +372,36 @@ def test_submit_non_markdown_formats(self):
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertTrue("Unexpected content" in q("#id_statement_file").next().text())
+
+    def test_change_statement_state(self):
+        statement = StatementFactory()  # starts in "active" state
+        active_state = State.objects.get(type_id="statement", slug="active")
+        replaced_state = State.objects.get(type_id="statement", slug="replaced")
+        url = urlreverse(
+            "ietf.doc.views_statement.change_statement_state",
+            kwargs={"name": statement.name},
+        )
+
+        events_before = statement.docevent_set.count()
+        login_testing_unauthorized(self, "secretary", url)
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+
+        r = self.client.post(url, {"state": active_state.pk}, follow=True)
+        self.assertContains(r, "State not changed", status_code=200)
+        statement = Document.objects.get(pk=statement.pk)  # bust the state cache
+        self.assertEqual(statement.get_state(), active_state)
+
+        r = self.client.post(url, {"state": replaced_state.pk}, follow=True)
+        self.assertContains(r, "State changed to", status_code=200)
+        statement = Document.objects.get(pk=statement.pk)  # bust the state cache
+        self.assertEqual(statement.get_state(), replaced_state)
+
+        events_after = statement.docevent_set.count()
+        self.assertEqual(events_after, events_before + 1)
+        event = statement.docevent_set.first()
+        self.assertEqual(event.type, "changed_state")
+        self.assertEqual(
+            event.desc, "Statement State changed to <b>Replaced</b> from Active"
+        )
diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py
index 0fa1a04b49..60255af856 100644
--- a/ietf/doc/urls.py
+++ b/ietf/doc/urls.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2009-2023, All Rights Reserved
+# Copyright The IETF Trust 2009-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 # Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
 # All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
@@ -145,6 +145,7 @@
     url(r'^%(name)s/edit/adopt/$' % settings.URL_REGEXPS, views_draft.adopt_draft),
     url(r'^%(name)s/edit/release/$' % settings.URL_REGEXPS, views_draft.release_draft),
     url(r'^%(name)s/edit/state/(?P<state_type>draft-stream-[a-z]+)/$' % settings.URL_REGEXPS, views_draft.change_stream_state),
+    url(r'^%(name)s/edit/state/statement/$' % settings.URL_REGEXPS, views_statement.change_statement_state),
 
     url(r'^%(name)s/edit/clearballot/(?P<ballot_type_slug>[\w-]+)/$' % settings.URL_REGEXPS, views_ballot.clear_ballot),
     url(r'^%(name)s/edit/deferballot/$' % settings.URL_REGEXPS, views_ballot.defer_ballot),
diff --git a/ietf/doc/views_statement.py b/ietf/doc/views_statement.py
index 9dc8c8ad69..4b85c81d83 100644
--- a/ietf/doc/views_statement.py
+++ b/ietf/doc/views_statement.py
@@ -1,4 +1,5 @@
-# Copyright The IETF Trust 2023, All Rights Reserved
+# Copyright The IETF Trust 2023-2025, All Rights Reserved
+from django.contrib import messages
 
 import debug  # pyflakes: ignore
 
@@ -6,10 +7,13 @@
 
 from django import forms
 from django.conf import settings
-from django.http import FileResponse, Http404
+from django.http import FileResponse, Http404, HttpResponseRedirect
 from django.views.decorators.cache import cache_control
 from django.shortcuts import get_object_or_404, render, redirect
 from django.template.loader import render_to_string
+
+from ietf.doc.forms import ChangeStatementStateForm
+from ietf.doc.utils import add_state_change_event
 from ietf.utils import markdown
 from django.utils.html import escape
 
@@ -278,3 +282,40 @@ def new_statement(request):
         }
         form = NewStatementForm(initial=init)
     return render(request, "doc/statement/new_statement.html", {"form": form})
+
+
+@role_required("Secretariat")
+def change_statement_state(request, name):
+    """Change state of a statement Document"""
+    statement = get_object_or_404(
+        Document.objects.filter(type_id="statement"),
+        name=name,
+    )
+    if request.method == "POST":
+        form = ChangeStatementStateForm(request.POST)
+        if form.is_valid():
+            new_state = form.cleaned_data["state"]
+            prev_state = statement.get_state()
+            if new_state == prev_state:
+                messages.info(request, f"State not changed, remains {prev_state}.")
+            else:
+                statement.set_state(new_state)
+                e = add_state_change_event(
+                    statement,
+                    request.user.person,
+                    prev_state,
+                    new_state,
+                )
+                statement.save_with_history([e])
+                messages.success(request, f"State changed to {new_state}.")
+            return HttpResponseRedirect(statement.get_absolute_url())
+    else:
+        form = ChangeStatementStateForm(initial={"state": statement.get_state()})
+    return render(
+        request,
+        "doc/statement/change_statement_state.html",
+        {
+            "form": form,
+            "statement": statement,
+        },
+    )
diff --git a/ietf/templates/doc/document_statement.html b/ietf/templates/doc/document_statement.html
index 7b9759c3e9..cc3ea5a44c 100644
--- a/ietf/templates/doc/document_statement.html
+++ b/ietf/templates/doc/document_statement.html
@@ -1,5 +1,5 @@
 {% extends "base.html" %}
-{# Copyright The IETF Trust 2023, All Rights Reserved #}
+{# Copyright The IETF Trust 2023-2025, All Rights Reserved #}
 {% load origin %}
 {% load static %}
 {% load ietf_filters %}
@@ -49,7 +49,12 @@
                 <th scope="row">
                     <a href="{% url 'ietf.doc.views_help.state_help' type='statement' %}">State</a>
                 </th>
-                <td class="edit"></td>
+                <td class="edit">{% if can_manage %}
+                    <a class="btn btn-primary btn-sm"
+                       href="{% url 'ietf.doc.views_statement.change_statement_state' name=doc.name %}">
+                        Edit
+                    </a>
+                {% endif %}</td>
                 <td id="statement-state">
                     {% if doc.get_state %}
                         <span title="{{ doc.get_state.desc }}" class="badge rounded-pill {% if doc.get_state.name|slugify == 'active' %}text-bg-success{% else %}text-bg-warning{% endif %}">{{ doc.get_state.name }}</span>
diff --git a/ietf/templates/doc/statement/change_statement_state.html b/ietf/templates/doc/statement/change_statement_state.html
new file mode 100644
index 0000000000..aa5cb934e8
--- /dev/null
+++ b/ietf/templates/doc/statement/change_statement_state.html
@@ -0,0 +1,22 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% extends "base.html" %}
+{% load origin %}
+{% load django_bootstrap5 %}
+{% block title %}Change state for {{ statement }}{% endblock %}
+{% block content %}
+    {% origin %}
+    <h1>
+        Change state
+        <br>
+        <small class="text-body-secondary">{{ statement }}</small>
+    </h1>
+    <form class="mt-3" method="post">
+        {% csrf_token %}
+        {% bootstrap_form form %}
+        <button type="submit" class="btn btn-primary">Submit</button>
+        <a class="btn btn-secondary float-end"
+           href="{{ statement.get_absolute_url }}">
+            Back
+        </a>
+    </form>
+{% endblock %}
\ No newline at end of file

From 2a16b6d5f4b2add753ef2c0158c0b5f6f7e5608c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 19 Jun 2025 00:57:46 -0400
Subject: [PATCH 358/656] chore(deps): bump
 stefanzweifel/git-auto-commit-action from 5 to 6 (#9007)

Bumps [stefanzweifel/git-auto-commit-action](https://github.com/stefanzweifel/git-auto-commit-action) from 5 to 6.
- [Release notes](https://github.com/stefanzweifel/git-auto-commit-action/releases)
- [Changelog](https://github.com/stefanzweifel/git-auto-commit-action/blob/master/CHANGELOG.md)
- [Commits](https://github.com/stefanzweifel/git-auto-commit-action/compare/v5...v6)

---
updated-dependencies:
- dependency-name: stefanzweifel/git-auto-commit-action
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build-base-app.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index 5f0a0d11b8..479cd7cadf 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -59,7 +59,7 @@ jobs:
         echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE
 
     - name: Commit CHANGELOG.md
-      uses: stefanzweifel/git-auto-commit-action@v5
+      uses: stefanzweifel/git-auto-commit-action@v6
       with:
         branch: ${{ github.ref_name }}
         commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}'

From df6266cfb3833f23a066212635960a2b86ec4b3f Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 20 Jun 2025 14:53:38 -0300
Subject: [PATCH 359/656] fix: block creation of floorplan without image
 (#9040)

* fix: block creation of floorplan without image

* chore: migration
---
 .../migrations/0014_alter_floorplan_image.py  | 25 +++++++++++++++++++
 ietf/meeting/models.py                        |  2 +-
 2 files changed, 26 insertions(+), 1 deletion(-)
 create mode 100644 ietf/meeting/migrations/0014_alter_floorplan_image.py

diff --git a/ietf/meeting/migrations/0014_alter_floorplan_image.py b/ietf/meeting/migrations/0014_alter_floorplan_image.py
new file mode 100644
index 0000000000..e125625edc
--- /dev/null
+++ b/ietf/meeting/migrations/0014_alter_floorplan_image.py
@@ -0,0 +1,25 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+import ietf.meeting.models
+import ietf.utils.storage
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("meeting", "0013_correct_reg_checkedin"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="floorplan",
+            name="image",
+            field=models.ImageField(
+                default=None,
+                storage=ietf.utils.storage.BlobShadowFileSystemStorage(
+                    kind="", location=None
+                ),
+                upload_to=ietf.meeting.models.floorplan_path,
+            ),
+        ),
+    ]
diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index cc5241efa2..b148568156 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -530,7 +530,7 @@ class FloorPlan(models.Model):
     image   = models.ImageField(
         storage=BlobShadowFileSystemStorage(kind="floorplan"),
         upload_to=floorplan_path,
-        blank=True,
+        blank=False,
         default=None,
     )
     #

From e93a56bb350d4e5d21fa3031f29d78db2887d816 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 20 Jun 2025 15:27:27 -0300
Subject: [PATCH 360/656] chore: increase timeout for rabbitmq probe (#9030)

---
 k8s/rabbitmq.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml
index 0c8f0705b5..780a399239 100644
--- a/k8s/rabbitmq.yaml
+++ b/k8s/rabbitmq.yaml
@@ -52,17 +52,17 @@ spec:
                   key: CELERY_PASSWORD
           livenessProbe:
             exec:
-              command: ["rabbitmq-diagnostics", "-q", "ping"]
+              command: ["rabbitmq-diagnostics", "-q", "ping", "-t", "30"]
             periodSeconds: 30
-            timeoutSeconds: 5
+            timeoutSeconds: 35  # slightly longer than ping "-t" option
           startupProbe:
             initialDelaySeconds: 15
             periodSeconds: 5
-            timeoutSeconds: 5
+            timeoutSeconds: 35  # slightly longer than ping "-t" option
             successThreshold: 1
             failureThreshold: 60
             exec:
-              command: ["rabbitmq-diagnostics", "-q", "ping"]
+              command: ["rabbitmq-diagnostics", "-q", "ping", "-t", "30"]
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:

From 2f1d00942c6a57184db2ac05ba182a1cab14d2f6 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 20 Jun 2025 17:45:59 -0300
Subject: [PATCH 361/656] refactor: no mutating countries/timezones choices
 (#9037)

* refactor: don't mutate countries/timezones

* refactor: capitalize/make tuples to discourage mutation
---
 ietf/meeting/forms.py  | 11 +++--------
 ietf/meeting/models.py | 25 +++++++++++++++----------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py
index e1d1e90b8d..b6b1a1591f 100644
--- a/ietf/meeting/forms.py
+++ b/ietf/meeting/forms.py
@@ -22,7 +22,7 @@
 from ietf.doc.models import Document, State, NewRevisionDocEvent
 from ietf.group.models import Group
 from ietf.group.utils import groups_managed_by
-from ietf.meeting.models import Session, Meeting, Schedule, countries, timezones, TimeSlot, Room
+from ietf.meeting.models import Session, Meeting, Schedule, COUNTRIES, TIMEZONES, TimeSlot, Room
 from ietf.meeting.helpers import get_next_interim_number, make_materials_directories
 from ietf.meeting.helpers import is_interim_meeting_approved, get_next_agenda_name
 from ietf.message.models import Message
@@ -38,11 +38,6 @@
 from ietf.utils.validators import ( validate_file_size, validate_mime_type,
     validate_file_extension, validate_no_html_frame)
 
-# need to insert empty option for use in ChoiceField
-# countries.insert(0, ('', '-'*9 ))
-countries.insert(0, ('', '-' * 9))
-timezones.insert(0, ('', '-' * 9))
-
 # -------------------------------------------------
 # Helpers
 # -------------------------------------------------
@@ -140,12 +135,12 @@ class InterimMeetingModelForm(forms.ModelForm):
     approved = forms.BooleanField(required=False)
     city = forms.CharField(max_length=255, required=False)
     city.widget.attrs['placeholder'] = "City"
-    country = forms.ChoiceField(choices=countries, required=False)
+    country = forms.ChoiceField(choices=COUNTRIES, required=False)
     country.widget.attrs['class'] = "select2-field"
     country.widget.attrs['data-max-entries'] = 1
     country.widget.attrs['data-placeholder'] = "Country"
     country.widget.attrs['data-minimum-input-length'] = 0
-    time_zone = forms.ChoiceField(choices=timezones)
+    time_zone = forms.ChoiceField(choices=TIMEZONES)
     time_zone.widget.attrs['class'] = "select2-field"
     time_zone.widget.attrs['data-max-entries'] = 1
     time_zone.widget.attrs['data-minimum-input-length'] = 0
diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index b148568156..62275082e4 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -49,15 +49,20 @@
 )
 from ietf.utils.fields import MissingOkImageField
 
-countries = list(pytz.country_names.items())
-countries.sort(key=lambda x: x[1])
+# Set up countries / timezones, including an empty choice for fields
+EMPTY_CHOICE = ("", "-" * 9)
+COUNTRIES = (EMPTY_CHOICE,) + tuple(
+    sorted(pytz.country_names.items(), key=lambda x: x[1])
+)
 
-timezones = []
-for name in pytz.common_timezones:
-    tzfn = os.path.join(settings.TZDATA_ICS_PATH, name + ".ics")
-    if not os.path.islink(tzfn):
-        timezones.append((name, name))
-timezones.sort()
+_tzdata_ics_path = Path(settings.TZDATA_ICS_PATH)
+TIMEZONES = (EMPTY_CHOICE,) + tuple(
+    sorted(
+        (name, name)
+        for name in pytz.common_timezones
+        if not (_tzdata_ics_path / f"{name}.ics").is_symlink()
+    )
+)
 
 
 class Meeting(models.Model):
@@ -72,11 +77,11 @@ class Meeting(models.Model):
     days = models.IntegerField(default=7, null=False, validators=[MinValueValidator(1)],
         help_text="The number of days the meeting lasts")
     city = models.CharField(blank=True, max_length=255)
-    country = models.CharField(blank=True, max_length=2, choices=countries)
+    country = models.CharField(blank=True, max_length=2, choices=COUNTRIES)
     # We can't derive time-zone from country, as there are some that have
     # more than one timezone, and the pytz module doesn't provide timezone
     # lookup information for all relevant city/country combinations.
-    time_zone = models.CharField(max_length=255, choices=timezones, default='UTC')
+    time_zone = models.CharField(max_length=255, choices=TIMEZONES, default='UTC')
     idsubmit_cutoff_day_offset_00 = models.IntegerField(blank=True,
         default=settings.IDSUBMIT_DEFAULT_CUTOFF_DAY_OFFSET_00,
         help_text = "The number of days before the meeting start date when the submission of -00 drafts will be closed.")

From f85c1a4913eabd59d1ed2b594218e96ab47d57d0 Mon Sep 17 00:00:00 2001
From: Benson Muite <bkmgit@users.noreply.github.com>
Date: Sat, 21 Jun 2025 00:22:20 +0300
Subject: [PATCH 362/656] fix: do not list GMT as a meeting time option (#8507)
 (#8681)

* fix: do not list GMT as a meeting time option (#8507)

* chore: copyright

* chore: migration

* fix: migration conflict

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 .../0015_alter_meeting_time_zone.py           | 451 ++++++++++++++++++
 ietf/meeting/models.py                        |   5 +-
 2 files changed, 453 insertions(+), 3 deletions(-)
 create mode 100644 ietf/meeting/migrations/0015_alter_meeting_time_zone.py

diff --git a/ietf/meeting/migrations/0015_alter_meeting_time_zone.py b/ietf/meeting/migrations/0015_alter_meeting_time_zone.py
new file mode 100644
index 0000000000..2a4b7859ee
--- /dev/null
+++ b/ietf/meeting/migrations/0015_alter_meeting_time_zone.py
@@ -0,0 +1,451 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+
+
+def forward(apps, schema_editor):
+    """Migrate 'GMT' meeting time zones to 'UTC'"""
+    Meeting = apps.get_model("meeting", "Meeting")
+    Meeting.objects.filter(time_zone="GMT").update(time_zone="UTC")
+
+
+def reverse(apps, schema_editor):
+    pass  # nothing to do
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("meeting", "0014_alter_floorplan_image"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+        migrations.AlterField(
+            model_name="meeting",
+            name="time_zone",
+            field=models.CharField(
+                choices=[
+                    ("", "---------"),
+                    ("Africa/Abidjan", "Africa/Abidjan"),
+                    ("Africa/Accra", "Africa/Accra"),
+                    ("Africa/Addis_Ababa", "Africa/Addis_Ababa"),
+                    ("Africa/Algiers", "Africa/Algiers"),
+                    ("Africa/Asmara", "Africa/Asmara"),
+                    ("Africa/Bamako", "Africa/Bamako"),
+                    ("Africa/Bangui", "Africa/Bangui"),
+                    ("Africa/Banjul", "Africa/Banjul"),
+                    ("Africa/Bissau", "Africa/Bissau"),
+                    ("Africa/Blantyre", "Africa/Blantyre"),
+                    ("Africa/Brazzaville", "Africa/Brazzaville"),
+                    ("Africa/Bujumbura", "Africa/Bujumbura"),
+                    ("Africa/Cairo", "Africa/Cairo"),
+                    ("Africa/Casablanca", "Africa/Casablanca"),
+                    ("Africa/Ceuta", "Africa/Ceuta"),
+                    ("Africa/Conakry", "Africa/Conakry"),
+                    ("Africa/Dakar", "Africa/Dakar"),
+                    ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"),
+                    ("Africa/Djibouti", "Africa/Djibouti"),
+                    ("Africa/Douala", "Africa/Douala"),
+                    ("Africa/El_Aaiun", "Africa/El_Aaiun"),
+                    ("Africa/Freetown", "Africa/Freetown"),
+                    ("Africa/Gaborone", "Africa/Gaborone"),
+                    ("Africa/Harare", "Africa/Harare"),
+                    ("Africa/Johannesburg", "Africa/Johannesburg"),
+                    ("Africa/Juba", "Africa/Juba"),
+                    ("Africa/Kampala", "Africa/Kampala"),
+                    ("Africa/Khartoum", "Africa/Khartoum"),
+                    ("Africa/Kigali", "Africa/Kigali"),
+                    ("Africa/Kinshasa", "Africa/Kinshasa"),
+                    ("Africa/Lagos", "Africa/Lagos"),
+                    ("Africa/Libreville", "Africa/Libreville"),
+                    ("Africa/Lome", "Africa/Lome"),
+                    ("Africa/Luanda", "Africa/Luanda"),
+                    ("Africa/Lubumbashi", "Africa/Lubumbashi"),
+                    ("Africa/Lusaka", "Africa/Lusaka"),
+                    ("Africa/Malabo", "Africa/Malabo"),
+                    ("Africa/Maputo", "Africa/Maputo"),
+                    ("Africa/Maseru", "Africa/Maseru"),
+                    ("Africa/Mbabane", "Africa/Mbabane"),
+                    ("Africa/Mogadishu", "Africa/Mogadishu"),
+                    ("Africa/Monrovia", "Africa/Monrovia"),
+                    ("Africa/Nairobi", "Africa/Nairobi"),
+                    ("Africa/Ndjamena", "Africa/Ndjamena"),
+                    ("Africa/Niamey", "Africa/Niamey"),
+                    ("Africa/Nouakchott", "Africa/Nouakchott"),
+                    ("Africa/Ouagadougou", "Africa/Ouagadougou"),
+                    ("Africa/Porto-Novo", "Africa/Porto-Novo"),
+                    ("Africa/Sao_Tome", "Africa/Sao_Tome"),
+                    ("Africa/Tripoli", "Africa/Tripoli"),
+                    ("Africa/Tunis", "Africa/Tunis"),
+                    ("Africa/Windhoek", "Africa/Windhoek"),
+                    ("America/Adak", "America/Adak"),
+                    ("America/Anchorage", "America/Anchorage"),
+                    ("America/Anguilla", "America/Anguilla"),
+                    ("America/Antigua", "America/Antigua"),
+                    ("America/Araguaina", "America/Araguaina"),
+                    (
+                        "America/Argentina/Buenos_Aires",
+                        "America/Argentina/Buenos_Aires",
+                    ),
+                    ("America/Argentina/Catamarca", "America/Argentina/Catamarca"),
+                    ("America/Argentina/Cordoba", "America/Argentina/Cordoba"),
+                    ("America/Argentina/Jujuy", "America/Argentina/Jujuy"),
+                    ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"),
+                    ("America/Argentina/Mendoza", "America/Argentina/Mendoza"),
+                    (
+                        "America/Argentina/Rio_Gallegos",
+                        "America/Argentina/Rio_Gallegos",
+                    ),
+                    ("America/Argentina/Salta", "America/Argentina/Salta"),
+                    ("America/Argentina/San_Juan", "America/Argentina/San_Juan"),
+                    ("America/Argentina/San_Luis", "America/Argentina/San_Luis"),
+                    ("America/Argentina/Tucuman", "America/Argentina/Tucuman"),
+                    ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"),
+                    ("America/Aruba", "America/Aruba"),
+                    ("America/Asuncion", "America/Asuncion"),
+                    ("America/Atikokan", "America/Atikokan"),
+                    ("America/Bahia", "America/Bahia"),
+                    ("America/Bahia_Banderas", "America/Bahia_Banderas"),
+                    ("America/Barbados", "America/Barbados"),
+                    ("America/Belem", "America/Belem"),
+                    ("America/Belize", "America/Belize"),
+                    ("America/Blanc-Sablon", "America/Blanc-Sablon"),
+                    ("America/Boa_Vista", "America/Boa_Vista"),
+                    ("America/Bogota", "America/Bogota"),
+                    ("America/Boise", "America/Boise"),
+                    ("America/Cambridge_Bay", "America/Cambridge_Bay"),
+                    ("America/Campo_Grande", "America/Campo_Grande"),
+                    ("America/Cancun", "America/Cancun"),
+                    ("America/Caracas", "America/Caracas"),
+                    ("America/Cayenne", "America/Cayenne"),
+                    ("America/Cayman", "America/Cayman"),
+                    ("America/Chicago", "America/Chicago"),
+                    ("America/Chihuahua", "America/Chihuahua"),
+                    ("America/Costa_Rica", "America/Costa_Rica"),
+                    ("America/Creston", "America/Creston"),
+                    ("America/Cuiaba", "America/Cuiaba"),
+                    ("America/Curacao", "America/Curacao"),
+                    ("America/Danmarkshavn", "America/Danmarkshavn"),
+                    ("America/Dawson", "America/Dawson"),
+                    ("America/Dawson_Creek", "America/Dawson_Creek"),
+                    ("America/Denver", "America/Denver"),
+                    ("America/Detroit", "America/Detroit"),
+                    ("America/Dominica", "America/Dominica"),
+                    ("America/Edmonton", "America/Edmonton"),
+                    ("America/Eirunepe", "America/Eirunepe"),
+                    ("America/El_Salvador", "America/El_Salvador"),
+                    ("America/Fort_Nelson", "America/Fort_Nelson"),
+                    ("America/Fortaleza", "America/Fortaleza"),
+                    ("America/Glace_Bay", "America/Glace_Bay"),
+                    ("America/Goose_Bay", "America/Goose_Bay"),
+                    ("America/Grand_Turk", "America/Grand_Turk"),
+                    ("America/Grenada", "America/Grenada"),
+                    ("America/Guadeloupe", "America/Guadeloupe"),
+                    ("America/Guatemala", "America/Guatemala"),
+                    ("America/Guayaquil", "America/Guayaquil"),
+                    ("America/Guyana", "America/Guyana"),
+                    ("America/Halifax", "America/Halifax"),
+                    ("America/Havana", "America/Havana"),
+                    ("America/Hermosillo", "America/Hermosillo"),
+                    ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"),
+                    ("America/Indiana/Knox", "America/Indiana/Knox"),
+                    ("America/Indiana/Marengo", "America/Indiana/Marengo"),
+                    ("America/Indiana/Petersburg", "America/Indiana/Petersburg"),
+                    ("America/Indiana/Tell_City", "America/Indiana/Tell_City"),
+                    ("America/Indiana/Vevay", "America/Indiana/Vevay"),
+                    ("America/Indiana/Vincennes", "America/Indiana/Vincennes"),
+                    ("America/Indiana/Winamac", "America/Indiana/Winamac"),
+                    ("America/Inuvik", "America/Inuvik"),
+                    ("America/Iqaluit", "America/Iqaluit"),
+                    ("America/Jamaica", "America/Jamaica"),
+                    ("America/Juneau", "America/Juneau"),
+                    ("America/Kentucky/Louisville", "America/Kentucky/Louisville"),
+                    ("America/Kentucky/Monticello", "America/Kentucky/Monticello"),
+                    ("America/La_Paz", "America/La_Paz"),
+                    ("America/Lima", "America/Lima"),
+                    ("America/Los_Angeles", "America/Los_Angeles"),
+                    ("America/Maceio", "America/Maceio"),
+                    ("America/Managua", "America/Managua"),
+                    ("America/Manaus", "America/Manaus"),
+                    ("America/Martinique", "America/Martinique"),
+                    ("America/Matamoros", "America/Matamoros"),
+                    ("America/Mazatlan", "America/Mazatlan"),
+                    ("America/Menominee", "America/Menominee"),
+                    ("America/Merida", "America/Merida"),
+                    ("America/Metlakatla", "America/Metlakatla"),
+                    ("America/Mexico_City", "America/Mexico_City"),
+                    ("America/Miquelon", "America/Miquelon"),
+                    ("America/Moncton", "America/Moncton"),
+                    ("America/Monterrey", "America/Monterrey"),
+                    ("America/Montevideo", "America/Montevideo"),
+                    ("America/Montserrat", "America/Montserrat"),
+                    ("America/Nassau", "America/Nassau"),
+                    ("America/New_York", "America/New_York"),
+                    ("America/Nipigon", "America/Nipigon"),
+                    ("America/Nome", "America/Nome"),
+                    ("America/Noronha", "America/Noronha"),
+                    ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"),
+                    ("America/North_Dakota/Center", "America/North_Dakota/Center"),
+                    (
+                        "America/North_Dakota/New_Salem",
+                        "America/North_Dakota/New_Salem",
+                    ),
+                    ("America/Nuuk", "America/Nuuk"),
+                    ("America/Ojinaga", "America/Ojinaga"),
+                    ("America/Panama", "America/Panama"),
+                    ("America/Pangnirtung", "America/Pangnirtung"),
+                    ("America/Paramaribo", "America/Paramaribo"),
+                    ("America/Phoenix", "America/Phoenix"),
+                    ("America/Port-au-Prince", "America/Port-au-Prince"),
+                    ("America/Port_of_Spain", "America/Port_of_Spain"),
+                    ("America/Porto_Velho", "America/Porto_Velho"),
+                    ("America/Puerto_Rico", "America/Puerto_Rico"),
+                    ("America/Punta_Arenas", "America/Punta_Arenas"),
+                    ("America/Rainy_River", "America/Rainy_River"),
+                    ("America/Rankin_Inlet", "America/Rankin_Inlet"),
+                    ("America/Recife", "America/Recife"),
+                    ("America/Regina", "America/Regina"),
+                    ("America/Resolute", "America/Resolute"),
+                    ("America/Rio_Branco", "America/Rio_Branco"),
+                    ("America/Santarem", "America/Santarem"),
+                    ("America/Santiago", "America/Santiago"),
+                    ("America/Santo_Domingo", "America/Santo_Domingo"),
+                    ("America/Sao_Paulo", "America/Sao_Paulo"),
+                    ("America/Scoresbysund", "America/Scoresbysund"),
+                    ("America/Sitka", "America/Sitka"),
+                    ("America/St_Johns", "America/St_Johns"),
+                    ("America/St_Kitts", "America/St_Kitts"),
+                    ("America/St_Lucia", "America/St_Lucia"),
+                    ("America/St_Thomas", "America/St_Thomas"),
+                    ("America/St_Vincent", "America/St_Vincent"),
+                    ("America/Swift_Current", "America/Swift_Current"),
+                    ("America/Tegucigalpa", "America/Tegucigalpa"),
+                    ("America/Thule", "America/Thule"),
+                    ("America/Thunder_Bay", "America/Thunder_Bay"),
+                    ("America/Tijuana", "America/Tijuana"),
+                    ("America/Toronto", "America/Toronto"),
+                    ("America/Tortola", "America/Tortola"),
+                    ("America/Vancouver", "America/Vancouver"),
+                    ("America/Whitehorse", "America/Whitehorse"),
+                    ("America/Winnipeg", "America/Winnipeg"),
+                    ("America/Yakutat", "America/Yakutat"),
+                    ("America/Yellowknife", "America/Yellowknife"),
+                    ("Antarctica/Casey", "Antarctica/Casey"),
+                    ("Antarctica/Davis", "Antarctica/Davis"),
+                    ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"),
+                    ("Antarctica/Macquarie", "Antarctica/Macquarie"),
+                    ("Antarctica/Mawson", "Antarctica/Mawson"),
+                    ("Antarctica/McMurdo", "Antarctica/McMurdo"),
+                    ("Antarctica/Palmer", "Antarctica/Palmer"),
+                    ("Antarctica/Rothera", "Antarctica/Rothera"),
+                    ("Antarctica/Syowa", "Antarctica/Syowa"),
+                    ("Antarctica/Troll", "Antarctica/Troll"),
+                    ("Antarctica/Vostok", "Antarctica/Vostok"),
+                    ("Asia/Aden", "Asia/Aden"),
+                    ("Asia/Almaty", "Asia/Almaty"),
+                    ("Asia/Amman", "Asia/Amman"),
+                    ("Asia/Anadyr", "Asia/Anadyr"),
+                    ("Asia/Aqtau", "Asia/Aqtau"),
+                    ("Asia/Aqtobe", "Asia/Aqtobe"),
+                    ("Asia/Ashgabat", "Asia/Ashgabat"),
+                    ("Asia/Atyrau", "Asia/Atyrau"),
+                    ("Asia/Baghdad", "Asia/Baghdad"),
+                    ("Asia/Bahrain", "Asia/Bahrain"),
+                    ("Asia/Baku", "Asia/Baku"),
+                    ("Asia/Bangkok", "Asia/Bangkok"),
+                    ("Asia/Barnaul", "Asia/Barnaul"),
+                    ("Asia/Beirut", "Asia/Beirut"),
+                    ("Asia/Bishkek", "Asia/Bishkek"),
+                    ("Asia/Brunei", "Asia/Brunei"),
+                    ("Asia/Chita", "Asia/Chita"),
+                    ("Asia/Choibalsan", "Asia/Choibalsan"),
+                    ("Asia/Colombo", "Asia/Colombo"),
+                    ("Asia/Damascus", "Asia/Damascus"),
+                    ("Asia/Dhaka", "Asia/Dhaka"),
+                    ("Asia/Dili", "Asia/Dili"),
+                    ("Asia/Dubai", "Asia/Dubai"),
+                    ("Asia/Dushanbe", "Asia/Dushanbe"),
+                    ("Asia/Famagusta", "Asia/Famagusta"),
+                    ("Asia/Gaza", "Asia/Gaza"),
+                    ("Asia/Hebron", "Asia/Hebron"),
+                    ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"),
+                    ("Asia/Hong_Kong", "Asia/Hong_Kong"),
+                    ("Asia/Hovd", "Asia/Hovd"),
+                    ("Asia/Irkutsk", "Asia/Irkutsk"),
+                    ("Asia/Jakarta", "Asia/Jakarta"),
+                    ("Asia/Jayapura", "Asia/Jayapura"),
+                    ("Asia/Jerusalem", "Asia/Jerusalem"),
+                    ("Asia/Kabul", "Asia/Kabul"),
+                    ("Asia/Kamchatka", "Asia/Kamchatka"),
+                    ("Asia/Karachi", "Asia/Karachi"),
+                    ("Asia/Kathmandu", "Asia/Kathmandu"),
+                    ("Asia/Khandyga", "Asia/Khandyga"),
+                    ("Asia/Kolkata", "Asia/Kolkata"),
+                    ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"),
+                    ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"),
+                    ("Asia/Kuching", "Asia/Kuching"),
+                    ("Asia/Kuwait", "Asia/Kuwait"),
+                    ("Asia/Macau", "Asia/Macau"),
+                    ("Asia/Magadan", "Asia/Magadan"),
+                    ("Asia/Makassar", "Asia/Makassar"),
+                    ("Asia/Manila", "Asia/Manila"),
+                    ("Asia/Muscat", "Asia/Muscat"),
+                    ("Asia/Nicosia", "Asia/Nicosia"),
+                    ("Asia/Novokuznetsk", "Asia/Novokuznetsk"),
+                    ("Asia/Novosibirsk", "Asia/Novosibirsk"),
+                    ("Asia/Omsk", "Asia/Omsk"),
+                    ("Asia/Oral", "Asia/Oral"),
+                    ("Asia/Phnom_Penh", "Asia/Phnom_Penh"),
+                    ("Asia/Pontianak", "Asia/Pontianak"),
+                    ("Asia/Pyongyang", "Asia/Pyongyang"),
+                    ("Asia/Qatar", "Asia/Qatar"),
+                    ("Asia/Qostanay", "Asia/Qostanay"),
+                    ("Asia/Qyzylorda", "Asia/Qyzylorda"),
+                    ("Asia/Riyadh", "Asia/Riyadh"),
+                    ("Asia/Sakhalin", "Asia/Sakhalin"),
+                    ("Asia/Samarkand", "Asia/Samarkand"),
+                    ("Asia/Seoul", "Asia/Seoul"),
+                    ("Asia/Shanghai", "Asia/Shanghai"),
+                    ("Asia/Singapore", "Asia/Singapore"),
+                    ("Asia/Srednekolymsk", "Asia/Srednekolymsk"),
+                    ("Asia/Taipei", "Asia/Taipei"),
+                    ("Asia/Tashkent", "Asia/Tashkent"),
+                    ("Asia/Tbilisi", "Asia/Tbilisi"),
+                    ("Asia/Tehran", "Asia/Tehran"),
+                    ("Asia/Thimphu", "Asia/Thimphu"),
+                    ("Asia/Tokyo", "Asia/Tokyo"),
+                    ("Asia/Tomsk", "Asia/Tomsk"),
+                    ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"),
+                    ("Asia/Urumqi", "Asia/Urumqi"),
+                    ("Asia/Ust-Nera", "Asia/Ust-Nera"),
+                    ("Asia/Vientiane", "Asia/Vientiane"),
+                    ("Asia/Vladivostok", "Asia/Vladivostok"),
+                    ("Asia/Yakutsk", "Asia/Yakutsk"),
+                    ("Asia/Yangon", "Asia/Yangon"),
+                    ("Asia/Yekaterinburg", "Asia/Yekaterinburg"),
+                    ("Asia/Yerevan", "Asia/Yerevan"),
+                    ("Atlantic/Azores", "Atlantic/Azores"),
+                    ("Atlantic/Bermuda", "Atlantic/Bermuda"),
+                    ("Atlantic/Canary", "Atlantic/Canary"),
+                    ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"),
+                    ("Atlantic/Faroe", "Atlantic/Faroe"),
+                    ("Atlantic/Madeira", "Atlantic/Madeira"),
+                    ("Atlantic/Reykjavik", "Atlantic/Reykjavik"),
+                    ("Atlantic/South_Georgia", "Atlantic/South_Georgia"),
+                    ("Atlantic/St_Helena", "Atlantic/St_Helena"),
+                    ("Atlantic/Stanley", "Atlantic/Stanley"),
+                    ("Australia/Adelaide", "Australia/Adelaide"),
+                    ("Australia/Brisbane", "Australia/Brisbane"),
+                    ("Australia/Broken_Hill", "Australia/Broken_Hill"),
+                    ("Australia/Darwin", "Australia/Darwin"),
+                    ("Australia/Eucla", "Australia/Eucla"),
+                    ("Australia/Hobart", "Australia/Hobart"),
+                    ("Australia/Lindeman", "Australia/Lindeman"),
+                    ("Australia/Lord_Howe", "Australia/Lord_Howe"),
+                    ("Australia/Melbourne", "Australia/Melbourne"),
+                    ("Australia/Perth", "Australia/Perth"),
+                    ("Australia/Sydney", "Australia/Sydney"),
+                    ("Europe/Amsterdam", "Europe/Amsterdam"),
+                    ("Europe/Andorra", "Europe/Andorra"),
+                    ("Europe/Astrakhan", "Europe/Astrakhan"),
+                    ("Europe/Athens", "Europe/Athens"),
+                    ("Europe/Belgrade", "Europe/Belgrade"),
+                    ("Europe/Berlin", "Europe/Berlin"),
+                    ("Europe/Brussels", "Europe/Brussels"),
+                    ("Europe/Bucharest", "Europe/Bucharest"),
+                    ("Europe/Budapest", "Europe/Budapest"),
+                    ("Europe/Chisinau", "Europe/Chisinau"),
+                    ("Europe/Copenhagen", "Europe/Copenhagen"),
+                    ("Europe/Dublin", "Europe/Dublin"),
+                    ("Europe/Gibraltar", "Europe/Gibraltar"),
+                    ("Europe/Helsinki", "Europe/Helsinki"),
+                    ("Europe/Istanbul", "Europe/Istanbul"),
+                    ("Europe/Kaliningrad", "Europe/Kaliningrad"),
+                    ("Europe/Kirov", "Europe/Kirov"),
+                    ("Europe/Kyiv", "Europe/Kyiv"),
+                    ("Europe/Lisbon", "Europe/Lisbon"),
+                    ("Europe/London", "Europe/London"),
+                    ("Europe/Luxembourg", "Europe/Luxembourg"),
+                    ("Europe/Madrid", "Europe/Madrid"),
+                    ("Europe/Malta", "Europe/Malta"),
+                    ("Europe/Minsk", "Europe/Minsk"),
+                    ("Europe/Monaco", "Europe/Monaco"),
+                    ("Europe/Moscow", "Europe/Moscow"),
+                    ("Europe/Oslo", "Europe/Oslo"),
+                    ("Europe/Paris", "Europe/Paris"),
+                    ("Europe/Prague", "Europe/Prague"),
+                    ("Europe/Riga", "Europe/Riga"),
+                    ("Europe/Rome", "Europe/Rome"),
+                    ("Europe/Samara", "Europe/Samara"),
+                    ("Europe/Saratov", "Europe/Saratov"),
+                    ("Europe/Simferopol", "Europe/Simferopol"),
+                    ("Europe/Sofia", "Europe/Sofia"),
+                    ("Europe/Stockholm", "Europe/Stockholm"),
+                    ("Europe/Tallinn", "Europe/Tallinn"),
+                    ("Europe/Tirane", "Europe/Tirane"),
+                    ("Europe/Ulyanovsk", "Europe/Ulyanovsk"),
+                    ("Europe/Uzhgorod", "Europe/Uzhgorod"),
+                    ("Europe/Vaduz", "Europe/Vaduz"),
+                    ("Europe/Vienna", "Europe/Vienna"),
+                    ("Europe/Vilnius", "Europe/Vilnius"),
+                    ("Europe/Volgograd", "Europe/Volgograd"),
+                    ("Europe/Warsaw", "Europe/Warsaw"),
+                    ("Europe/Zaporozhye", "Europe/Zaporozhye"),
+                    ("Europe/Zurich", "Europe/Zurich"),
+                    ("Indian/Antananarivo", "Indian/Antananarivo"),
+                    ("Indian/Chagos", "Indian/Chagos"),
+                    ("Indian/Christmas", "Indian/Christmas"),
+                    ("Indian/Cocos", "Indian/Cocos"),
+                    ("Indian/Comoro", "Indian/Comoro"),
+                    ("Indian/Kerguelen", "Indian/Kerguelen"),
+                    ("Indian/Mahe", "Indian/Mahe"),
+                    ("Indian/Maldives", "Indian/Maldives"),
+                    ("Indian/Mauritius", "Indian/Mauritius"),
+                    ("Indian/Mayotte", "Indian/Mayotte"),
+                    ("Indian/Reunion", "Indian/Reunion"),
+                    ("Pacific/Apia", "Pacific/Apia"),
+                    ("Pacific/Auckland", "Pacific/Auckland"),
+                    ("Pacific/Bougainville", "Pacific/Bougainville"),
+                    ("Pacific/Chatham", "Pacific/Chatham"),
+                    ("Pacific/Chuuk", "Pacific/Chuuk"),
+                    ("Pacific/Easter", "Pacific/Easter"),
+                    ("Pacific/Efate", "Pacific/Efate"),
+                    ("Pacific/Fakaofo", "Pacific/Fakaofo"),
+                    ("Pacific/Fiji", "Pacific/Fiji"),
+                    ("Pacific/Funafuti", "Pacific/Funafuti"),
+                    ("Pacific/Galapagos", "Pacific/Galapagos"),
+                    ("Pacific/Gambier", "Pacific/Gambier"),
+                    ("Pacific/Guadalcanal", "Pacific/Guadalcanal"),
+                    ("Pacific/Guam", "Pacific/Guam"),
+                    ("Pacific/Honolulu", "Pacific/Honolulu"),
+                    ("Pacific/Kanton", "Pacific/Kanton"),
+                    ("Pacific/Kiritimati", "Pacific/Kiritimati"),
+                    ("Pacific/Kosrae", "Pacific/Kosrae"),
+                    ("Pacific/Kwajalein", "Pacific/Kwajalein"),
+                    ("Pacific/Majuro", "Pacific/Majuro"),
+                    ("Pacific/Marquesas", "Pacific/Marquesas"),
+                    ("Pacific/Midway", "Pacific/Midway"),
+                    ("Pacific/Nauru", "Pacific/Nauru"),
+                    ("Pacific/Niue", "Pacific/Niue"),
+                    ("Pacific/Norfolk", "Pacific/Norfolk"),
+                    ("Pacific/Noumea", "Pacific/Noumea"),
+                    ("Pacific/Pago_Pago", "Pacific/Pago_Pago"),
+                    ("Pacific/Palau", "Pacific/Palau"),
+                    ("Pacific/Pitcairn", "Pacific/Pitcairn"),
+                    ("Pacific/Pohnpei", "Pacific/Pohnpei"),
+                    ("Pacific/Port_Moresby", "Pacific/Port_Moresby"),
+                    ("Pacific/Rarotonga", "Pacific/Rarotonga"),
+                    ("Pacific/Saipan", "Pacific/Saipan"),
+                    ("Pacific/Tahiti", "Pacific/Tahiti"),
+                    ("Pacific/Tarawa", "Pacific/Tarawa"),
+                    ("Pacific/Tongatapu", "Pacific/Tongatapu"),
+                    ("Pacific/Wake", "Pacific/Wake"),
+                    ("Pacific/Wallis", "Pacific/Wallis"),
+                    ("UTC", "UTC"),
+                ],
+                default="UTC",
+                max_length=255,
+            ),
+        ),
+    ]
diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index 62275082e4..de0192769e 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright The IETF Trust 2007-2024, All Rights Reserved
+# Copyright The IETF Trust 2007-2025, All Rights Reserved
 
 
 # old meeting models can be found in ../proceedings/models.py
@@ -60,7 +59,7 @@
     sorted(
         (name, name)
         for name in pytz.common_timezones
-        if not (_tzdata_ics_path / f"{name}.ics").is_symlink()
+        if name != "GMT" and not (_tzdata_ics_path / f"{name}.ics").is_symlink()
     )
 )
 

From ecc478653ad6ff09122f07c7663884eee74b80ab Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 23 Jun 2025 13:26:20 -0300
Subject: [PATCH 363/656] fix: handle absent <title> in XML submission (#9044)

* fix: handle absent front/title in get_title()

* test: test get_title()

* chore: copyrights

* chore: fix copyright + remove stray newline
---
 ietf/utils/tests.py    | 19 ++++++++++++++++++-
 ietf/utils/xmldraft.py |  5 +++--
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py
index 2dd861cd11..872aa366b9 100644
--- a/ietf/utils/tests.py
+++ b/ietf/utils/tests.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2014-2020, All Rights Reserved
+# Copyright The IETF Trust 2014-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -741,6 +741,23 @@ def test_render_author_name(self):
             "J. Q.",
         )
 
+    @patch("ietf.utils.xmldraft.XMLDraft.__init__", return_value=None)
+    def test_get_title(self, mock_init):
+        xmldraft = XMLDraft("fake")
+        self.assertTrue(mock_init.called)
+        # Stub XML that does not have a front/title element
+        xmldraft.xmlroot = lxml.etree.XML(
+            "<rfc><front></front></rfc>"  # no title
+        )
+        self.assertEqual(xmldraft.get_title(), "")
+
+        # Stub XML that has a front/title element
+        xmldraft.xmlroot = lxml.etree.XML(
+            "<rfc><front><title>This Is the Title</title></front></rfc>"
+        )
+        self.assertEqual(xmldraft.get_title(), "This Is the Title")
+
+        
     def test_capture_xml2rfc_output(self):
         """capture_xml2rfc_output reroutes and captures xml2rfc logs"""
         orig_write_out = xml2rfc_log.write_out
diff --git a/ietf/utils/xmldraft.py b/ietf/utils/xmldraft.py
index 3ac9a269c7..7ef6605c78 100644
--- a/ietf/utils/xmldraft.py
+++ b/ietf/utils/xmldraft.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2022, All Rights Reserved
+# Copyright The IETF Trust 2022-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 import datetime
 import io
@@ -147,7 +147,8 @@ def parse_docname(xmlroot):
         return revmatch.group('filename'), revmatch.group('rev')
 
     def get_title(self):
-        return self.xmlroot.findtext('front/title').strip()
+        title_text = self.xmlroot.findtext('front/title')
+        return "" if title_text is None else title_text.strip()
 
     @staticmethod
     def parse_creation_date(date_elt):

From 9e4660e89008c1acb286f2001ba00096f2342119 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 24 Jun 2025 12:43:15 -0300
Subject: [PATCH 364/656] feat: add ability to require strong passwords (#8828)

* feat: enforce pw strength at login

* chore(deps): add zxcvbn to requirements.txt

* feat: use zxcvbn for password strength check

* feat: validate password strength for setting pw

* feat: feedback on password change page

* refactor: avoid field validator munging

* feat: give more info about how to choose a pw

* refactor: use password_validation module

Only for changing password so; may need to update StrongPasswordAuthenticationForm to match

* feat: password min_length validdation

* refactor: use password_validation for login form

* fix: UI feedback consistent with validation

* chore: update chpw page to state length req

* chore(dev): disable password validators in dev

* fix: drop JS validation when password val disabled

* style: ruff on ChangePasswordForm

* chore: lint

* chore(dev): preserve pw validator cfg for tests

* test: fix test_change_password

* test: fix test_change_username

* test: fix test_reset_password

* style: ruff refactored tests

* chore: type lint

* feat: require pw reset for very stale accounts

* test: test stale account login

* test: rejection of short/simple PWs

* Revert "test: test stale account login"

This reverts commit ae0d90bb29fb379e547dd33f1a6c9aa04c03477b.

* Revert "feat: require pw reset for very stale accounts"

This reverts commit 1a98ef4c63500e4cbebbf6150d9bb5e15815fb4a.

* test: disable pw validators for playwright legacy tests

* feat: make pw enforcement at login optional
---
 ietf/ietfauth/forms.py                        |  54 +++--
 ietf/ietfauth/password_validation.py          |  23 ++
 ietf/ietfauth/tests.py                        | 205 +++++++++++++-----
 ietf/ietfauth/views.py                        |  34 ++-
 ietf/ietfauth/widgets.py                      |   9 +-
 ietf/settings.py                              |  20 ++
 ietf/settings_test.py                         |  15 +-
 ietf/static/js/password_strength.js           |  48 +++-
 .../registration/change_password.html         |  11 +-
 k8s/settings_local.py                         |   5 +
 requirements.txt                              |   2 +
 11 files changed, 321 insertions(+), 105 deletions(-)
 create mode 100644 ietf/ietfauth/password_validation.py

diff --git a/ietf/ietfauth/forms.py b/ietf/ietfauth/forms.py
index a70f7b6ca1..53bf7b5888 100644
--- a/ietf/ietfauth/forms.py
+++ b/ietf/ietfauth/forms.py
@@ -3,16 +3,19 @@
 
 
 import re
+
 from unidecode import unidecode
 
 from django import forms
+from django.contrib.auth.models import User
+from django.contrib.auth import password_validation
 from django.core.exceptions import ValidationError
 from django.db import models
-from django.contrib.auth.models import User
 
 from ietf.person.models import Person, Email
 from ietf.mailinglists.models import Allowlisted
 from ietf.utils.text import isascii
+from .password_validation import StrongPasswordValidator
 
 from .validators import prevent_at_symbol, prevent_system_name, prevent_anonymous_name, is_allowed_address
 from .widgets import PasswordStrengthInput, PasswordConfirmationInput
@@ -170,33 +173,52 @@ class Meta:
         model = Allowlisted
         exclude = ['by', 'time' ]
 
-    
-from django import forms
-
 
 class ChangePasswordForm(forms.Form):
     current_password = forms.CharField(widget=forms.PasswordInput)
 
-    new_password = forms.CharField(widget=PasswordStrengthInput(attrs={'class':'password_strength'}))
-    new_password_confirmation = forms.CharField(widget=PasswordConfirmationInput(
-                                                    confirm_with='new_password',
-                                                    attrs={'class':'password_confirmation'}))
+    new_password = forms.CharField(
+        widget=PasswordStrengthInput(
+            attrs={
+                "class": "password_strength",
+                "data-disable-strength-enforcement": "",  # usually removed in init
+            }
+        ),
+    )
+    new_password_confirmation = forms.CharField(
+        widget=PasswordConfirmationInput(
+            confirm_with="new_password", attrs={"class": "password_confirmation"}
+        )
+    )
 
     def __init__(self, user, data=None):
         self.user = user
-        super(ChangePasswordForm, self).__init__(data)
+        super().__init__(data)
+        # Check whether we have validators to enforce
+        new_password_field = self.fields["new_password"]
+        for pwval in password_validation.get_default_password_validators():
+            if isinstance(pwval, password_validation.MinimumLengthValidator):
+                new_password_field.widget.attrs["minlength"] = pwval.min_length
+            elif isinstance(pwval, StrongPasswordValidator):
+                new_password_field.widget.attrs.pop(
+                    "data-disable-strength-enforcement", None
+                )
 
     def clean_current_password(self):
-        password = self.cleaned_data.get('current_password', None)
+        # n.b., password = None is handled by check_password and results in a failed check
+        password = self.cleaned_data.get("current_password", None)
         if not self.user.check_password(password):
-            raise ValidationError('Invalid password')
+            raise ValidationError("Invalid password")
         return password
-            
+
     def clean(self):
-        new_password = self.cleaned_data.get('new_password', None)
-        conf_password = self.cleaned_data.get('new_password_confirmation', None)
-        if not new_password == conf_password:
-            raise ValidationError("The password confirmation is different than the new password")
+        new_password = self.cleaned_data.get("new_password", "")
+        conf_password = self.cleaned_data.get("new_password_confirmation", "")
+        if new_password != conf_password:
+            raise ValidationError(
+                "The password confirmation is different than the new password"
+            )
+        password_validation.validate_password(conf_password, self.user)
 
 
 class ChangeUsernameForm(forms.Form):
diff --git a/ietf/ietfauth/password_validation.py b/ietf/ietfauth/password_validation.py
new file mode 100644
index 0000000000..bfed4a784e
--- /dev/null
+++ b/ietf/ietfauth/password_validation.py
@@ -0,0 +1,23 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from django.core.exceptions import ValidationError
+from zxcvbn import zxcvbn
+
+
+class StrongPasswordValidator:
+    message = "This password does not meet complexity requirements and is easily guessable."
+    code = "weak"
+    min_zxcvbn_score = 3
+
+    def __init__(self, message=None, code=None, min_zxcvbn_score=None):
+        if message is not None:
+            self.message = message
+        if code is not None:
+            self.code = code
+        if min_zxcvbn_score is not None:
+            self.min_zxcvbn_score = min_zxcvbn_score
+
+    def validate(self, password, user=None):
+        """Validate that a password is strong enough"""
+        strength_report = zxcvbn(password[:72], max_length=72)
+        if strength_report["score"] < self.min_zxcvbn_score:
+            raise ValidationError(message=self.message, code=self.code)
diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py
index f6d7671bc9..dd23277b63 100644
--- a/ietf/ietfauth/tests.py
+++ b/ietf/ietfauth/tests.py
@@ -392,14 +392,14 @@ def test_nomcom_dressing_on_profile(self):
         self.assertFalse(q('#volunteer-button'))
         self.assertTrue(q('#volunteered'))
 
-
     def test_reset_password(self):
+        VALID_PASSWORD = "complex-and-long-valid-password"
+        ANOTHER_VALID_PASSWORD = "very-complicated-and-lengthy-password"
         url = urlreverse("ietf.ietfauth.views.password_reset")
-        email = 'someone@example.com'
-        password = 'foobar'
+        email = "someone@example.com"
 
         user = PersonFactory(user__email=email).user
-        user.set_password(password)
+        user.set_password(VALID_PASSWORD)
         user.save()
 
         # get
@@ -407,21 +407,23 @@ def test_reset_password(self):
         self.assertEqual(r.status_code, 200)
 
         # ask for reset, wrong username (form should not fail)
-        r = self.client.post(url, { 'username': "nobody@example.com" })
+        r = self.client.post(url, {"username": "nobody@example.com"})
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertTrue(len(q("form .is-invalid")) == 0)
 
         # ask for reset
         empty_outbox()
-        r = self.client.post(url, { 'username': user.username })
+        r = self.client.post(url, {"username": user.username})
         self.assertEqual(r.status_code, 200)
         self.assertEqual(len(outbox), 1)
 
         # goto change password page, logged in as someone else
         confirm_url = self.extract_confirm_url(outbox[-1])
         other_user = UserFactory()
-        self.client.login(username=other_user.username, password=other_user.username + '+password')
+        self.client.login(
+            username=other_user.username, password=other_user.username + "+password"
+        )
         r = self.client.get(confirm_url)
         self.assertEqual(r.status_code, 403)
 
@@ -430,17 +432,32 @@ def test_reset_password(self):
         r = self.client.get(confirm_url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertNotIn(user.username, q('.nav').text(),
-                         'user should not appear signed in while resetting password')
+        self.assertNotIn(
+            user.username,
+            q(".nav").text(),
+            "user should not appear signed in while resetting password",
+        )
 
         # password mismatch
-        r = self.client.post(confirm_url, { 'password': 'secret', 'password_confirmation': 'nosecret' })
+        r = self.client.post(
+            confirm_url,
+            {
+                "password": ANOTHER_VALID_PASSWORD,
+                "password_confirmation": ANOTHER_VALID_PASSWORD[::-1],
+            },
+        )
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertTrue(len(q("form .is-invalid")) > 0)
 
         # confirm
-        r = self.client.post(confirm_url, { 'password': 'secret', 'password_confirmation': 'secret' })
+        r = self.client.post(
+            confirm_url,
+            {
+                "password": ANOTHER_VALID_PASSWORD,
+                "password_confirmation": ANOTHER_VALID_PASSWORD,
+            },
+        )
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertEqual(len(q("form .is-invalid")), 0)
@@ -451,15 +468,18 @@ def test_reset_password(self):
 
         # login after reset request
         empty_outbox()
-        user.set_password(password)
+        user.set_password(VALID_PASSWORD)
         user.save()
 
-        r = self.client.post(url, { 'username': user.username })
+        r = self.client.post(url, {"username": user.username})
         self.assertEqual(r.status_code, 200)
         self.assertEqual(len(outbox), 1)
         confirm_url = self.extract_confirm_url(outbox[-1])
 
-        r = self.client.post(urlreverse("ietf.ietfauth.views.login"), {'username': email, 'password': password})
+        r = self.client.post(
+            urlreverse("ietf.ietfauth.views.login"),
+            {"username": email, "password": VALID_PASSWORD},
+        )
 
         r = self.client.get(confirm_url)
         self.assertEqual(r.status_code, 404)
@@ -467,12 +487,12 @@ def test_reset_password(self):
         # change password after reset request
         empty_outbox()
 
-        r = self.client.post(url, { 'username': user.username })
+        r = self.client.post(url, {"username": user.username})
         self.assertEqual(r.status_code, 200)
         self.assertEqual(len(outbox), 1)
         confirm_url = self.extract_confirm_url(outbox[-1])
 
-        user.set_password('newpassword')
+        user.set_password(ANOTHER_VALID_PASSWORD)
         user.save()
 
         r = self.client.get(confirm_url)
@@ -586,98 +606,175 @@ def test_review_overview(self):
         self.assertEqual(ReviewWish.objects.filter(doc=doc, team=review_req.team).count(), 0)
 
     def test_change_password(self):
+        VALID_PASSWORD = "complex-and-long-valid-password"
+        ANOTHER_VALID_PASSWORD = "very-complicated-and-lengthy-password"
         chpw_url = urlreverse("ietf.ietfauth.views.change_password")
         prof_url = urlreverse("ietf.ietfauth.views.profile")
         login_url = urlreverse("ietf.ietfauth.views.login")
-        redir_url = '%s?next=%s' % (login_url, chpw_url)
+        redir_url = "%s?next=%s" % (login_url, chpw_url)
 
         # get without logging in
         r = self.client.get(chpw_url)
         self.assertRedirects(r, redir_url)
 
-        user = User.objects.create(username="someone@example.com", email="someone@example.com")
-        user.set_password("password")
+        user = User.objects.create(
+            username="someone@example.com", email="someone@example.com"
+        )
+        user.set_password(VALID_PASSWORD)
         user.save()
         p = Person.objects.create(name="Some One", ascii="Some One", user=user)
         Email.objects.create(address=user.username, person=p, origin=user.username)
 
         # log in
-        r = self.client.post(redir_url, {"username":user.username, "password":"password"})
+        r = self.client.post(
+            redir_url, {"username": user.username, "password": VALID_PASSWORD}
+        )
         self.assertRedirects(r, chpw_url)
 
         # wrong current password
-        r = self.client.post(chpw_url, {"current_password": "fiddlesticks",
-                                        "new_password": "foobar",
-                                        "new_password_confirmation": "foobar",
-                                       })
+        r = self.client.post(
+            chpw_url,
+            {
+                "current_password": "fiddlesticks",
+                "new_password": ANOTHER_VALID_PASSWORD,
+                "new_password_confirmation": ANOTHER_VALID_PASSWORD,
+            },
+        )
         self.assertEqual(r.status_code, 200)
-        self.assertFormError(r.context["form"], 'current_password', 'Invalid password')
+        self.assertFormError(r.context["form"], "current_password", "Invalid password")
 
         # mismatching new passwords
-        r = self.client.post(chpw_url, {"current_password": "password",
-                                        "new_password": "foobar",
-                                        "new_password_confirmation": "barfoo",
-                                       })
+        r = self.client.post(
+            chpw_url,
+            {
+                "current_password": VALID_PASSWORD,
+                "new_password": ANOTHER_VALID_PASSWORD,
+                "new_password_confirmation": ANOTHER_VALID_PASSWORD[::-1],
+            },
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertFormError(
+            r.context["form"],
+            None,
+            "The password confirmation is different than the new password",
+        )
+
+        # password too short
+        r = self.client.post(
+            chpw_url,
+            {
+                "current_password": VALID_PASSWORD,
+                "new_password": "sh0rtpw0rd",
+                "new_password_confirmation": "sh0rtpw0rd",
+            }
+        )
         self.assertEqual(r.status_code, 200)
-        self.assertFormError(r.context["form"], None, "The password confirmation is different than the new password")
+        self.assertFormError(
+            r.context["form"],
+            None,
+            "This password is too short. It must contain at least "
+            f"{settings.PASSWORD_POLICY_MIN_LENGTH} characters."
+        )
+
+        # password too simple
+        r = self.client.post(
+            chpw_url,
+            {
+                "current_password": VALID_PASSWORD,
+                "new_password": "passwordpassword",
+                "new_password_confirmation": "passwordpassword",
+            }
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertFormError(
+            r.context["form"],
+            None,
+            "This password does not meet complexity requirements "
+            "and is easily guessable."
+        )
 
         # correct password change
-        r = self.client.post(chpw_url, {"current_password": "password",
-                                        "new_password": "foobar",
-                                        "new_password_confirmation": "foobar",
-                                       })
+        r = self.client.post(
+            chpw_url,
+            {
+                "current_password": VALID_PASSWORD,
+                "new_password": ANOTHER_VALID_PASSWORD,
+                "new_password_confirmation": ANOTHER_VALID_PASSWORD,
+            },
+        )
         self.assertRedirects(r, prof_url)
         # refresh user object
         user = User.objects.get(username="someone@example.com")
-        self.assertTrue(user.check_password('foobar'))
+        self.assertTrue(user.check_password(ANOTHER_VALID_PASSWORD))
 
     def test_change_username(self):
-
+        VALID_PASSWORD = "complex-and-long-valid-password"
         chun_url = urlreverse("ietf.ietfauth.views.change_username")
         prof_url = urlreverse("ietf.ietfauth.views.profile")
         login_url = urlreverse("ietf.ietfauth.views.login")
-        redir_url = '%s?next=%s' % (login_url, chun_url)
+        redir_url = "%s?next=%s" % (login_url, chun_url)
 
         # get without logging in
         r = self.client.get(chun_url)
         self.assertRedirects(r, redir_url)
 
-        user = User.objects.create(username="someone@example.com", email="someone@example.com")
-        user.set_password("password")
+        user = User.objects.create(
+            username="someone@example.com", email="someone@example.com"
+        )
+        user.set_password(VALID_PASSWORD)
         user.save()
         p = Person.objects.create(name="Some One", ascii="Some One", user=user)
         Email.objects.create(address=user.username, person=p, origin=user.username)
-        Email.objects.create(address="othername@example.org", person=p, origin=user.username)
+        Email.objects.create(
+            address="othername@example.org", person=p, origin=user.username
+        )
 
         # log in
-        r = self.client.post(redir_url, {"username":user.username, "password":"password"})
+        r = self.client.post(
+            redir_url, {"username": user.username, "password": VALID_PASSWORD}
+        )
         self.assertRedirects(r, chun_url)
 
         # wrong username
-        r = self.client.post(chun_url, {"username": "fiddlesticks",
-                                        "password": "password",
-                                       })
+        r = self.client.post(
+            chun_url,
+            {
+                "username": "fiddlesticks",
+                "password": VALID_PASSWORD,
+            },
+        )
         self.assertEqual(r.status_code, 200)
-        self.assertFormError(r.context["form"], 'username',
-            "Select a valid choice. fiddlesticks is not one of the available choices.")
+        self.assertFormError(
+            r.context["form"],
+            "username",
+            "Select a valid choice. fiddlesticks is not one of the available choices.",
+        )
 
         # wrong password
-        r = self.client.post(chun_url, {"username": "othername@example.org",
-                                        "password": "foobar",
-                                       })
+        r = self.client.post(
+            chun_url,
+            {
+                "username": "othername@example.org",
+                "password": "foobar",
+            },
+        )
         self.assertEqual(r.status_code, 200)
-        self.assertFormError(r.context["form"], 'password', 'Invalid password')
+        self.assertFormError(r.context["form"], "password", "Invalid password")
 
         # correct username change
-        r = self.client.post(chun_url, {"username": "othername@example.org",
-                                        "password": "password",
-                                       })
+        r = self.client.post(
+            chun_url,
+            {
+                "username": "othername@example.org",
+                "password": VALID_PASSWORD,
+            },
+        )
         self.assertRedirects(r, prof_url)
         # refresh user object
         prev = user
         user = User.objects.get(username="othername@example.org")
         self.assertEqual(prev, user)
-        self.assertTrue(user.check_password('password'))
+        self.assertTrue(user.check_password(VALID_PASSWORD))
 
     def test_apikey_management(self):
         # Create a person with a role that will give at least one valid apikey
diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py
index 23f66ce824..84d5490873 100644
--- a/ietf/ietfauth/views.py
+++ b/ietf/ietfauth/views.py
@@ -38,14 +38,14 @@
 import importlib
 
 # needed if we revert to higher barrier for account creation
-#from datetime import datetime as DateTime, timedelta as TimeDelta, date as Date
+# from datetime import datetime as DateTime, timedelta as TimeDelta, date as Date
 from collections import defaultdict
 
 import django.core.signing
 from django import forms
 from django.contrib import messages
 from django.conf import settings
-from django.contrib.auth import logout, update_session_auth_hash
+from django.contrib.auth import logout, update_session_auth_hash, password_validation
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.forms import AuthenticationForm
 from django.contrib.auth.hashers import identify_hasher
@@ -80,7 +80,6 @@
 # These are needed if we revert to the higher bar for account creation
 
 
-
 def index(request):
     return render(request, 'registration/index.html')
 
@@ -97,7 +96,7 @@ def index(request):
 # def ietf_login(request):
 #     if not request.user.is_authenticated:
 #         return HttpResponse("Not authenticated?", status=500)
-# 
+#
 #     redirect_to = request.REQUEST.get(REDIRECT_FIELD_NAME, '')
 #     request.session.set_test_cookie()
 #     return HttpResponseRedirect('/accounts/loggedin/?%s=%s' % (REDIRECT_FIELD_NAME, urlquote(redirect_to)))
@@ -582,7 +581,6 @@ def test_email(request):
     return r
 
 
-
 class AddReviewWishForm(forms.Form):
     doc = SearchableDocumentField(label="Document", doc_type="draft")
     team = forms.ModelChoiceField(queryset=Group.objects.all(), empty_label="(Choose review team)")
@@ -696,7 +694,7 @@ def change_password(request):
         'hasher': hasher,
     })
 
-    
+
 @login_required
 @person_required
 def change_username(request):
@@ -764,6 +762,28 @@ def clean(self):
                 )
         return super().clean()
 
+    def confirm_login_allowed(self, user):
+        """Check whether a successfully authenticated user is permitted to log in"""
+        super().confirm_login_allowed(user)
+        # Optionally enforce password validation
+        if getattr(settings, "PASSWORD_POLICY_ENFORCE_AT_LOGIN", False):
+            try:
+                password_validation.validate_password(
+                    self.cleaned_data["password"], user
+                )
+            except ValidationError:
+                raise ValidationError(
+                    # dict mapping field to error / error list
+                    {
+                        "__all__": ValidationError(
+                            'You entered your password correctly, but it  does not '
+                            'meet our current length and complexity requirements. '
+                            'Please use the "Forgot your password?" button below to '
+                            'set a new password for your account.'
+                        ),
+                    }
+                )
+
 
 class AnyEmailLoginView(LoginView):
     """LoginView that allows any email address as the username
@@ -779,7 +799,7 @@ def form_valid(self, form):
             logout(self.request)  # should not be logged in yet, but just in case...
             return render(self.request, "registration/missing_person.html")
         return super().form_valid(form)
-        
+
 
 @login_required
 @person_required
diff --git a/ietf/ietfauth/widgets.py b/ietf/ietfauth/widgets.py
index c9a0523402..fd7fa16726 100644
--- a/ietf/ietfauth/widgets.py
+++ b/ietf/ietfauth/widgets.py
@@ -39,18 +39,19 @@ def render(self, name, value, attrs=None, renderer=None):
         strength_markup = """
         <div style="margin-top: 10px;">
             <div class="progress" style="margin-bottom: 10px;">
-                <div class="progress-bar progress-bar-warning password_strength_bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="5" style="width: 0%%"></div>
+                <div class="progress-bar progress-bar-warning password_strength_bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="5" style="width: 0%"></div>
             </div>
             <p class="text-body-secondary password_strength_info hidden">
                 <span class="label label-danger">
-                    %s
+                    {}
                 </span>
                 <span style="margin-left:5px;">
-                    %s
+                    {}
                 </span>
+                <span class="password_improvement_hint d-none"></span>
             </p>
         </div>
-        """ % (
+        """.format(
             _("Warning"),
             _(
                 'This password would take <em class="password_strength_time"></em> to crack.'
diff --git a/ietf/settings.py b/ietf/settings.py
index 0b6b3745d2..5e33673611 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -61,6 +61,26 @@
     'django.contrib.auth.hashers.CryptPasswordHasher',
 ]
 
+
+PASSWORD_POLICY_MIN_LENGTH = 12
+PASSWORD_POLICY_ENFORCE_AT_LOGIN = False  # should turn this on for prod
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
+        "OPTIONS": {
+            "min_length": PASSWORD_POLICY_MIN_LENGTH,
+        }
+    },
+    {
+        "NAME": "ietf.ietfauth.password_validation.StrongPasswordValidator",
+    },
+]
+# In dev environments, settings_local overrides the password validators. Save
+# a handle to the original value so settings_test can restore it so tests match
+# production.
+ORIG_AUTH_PASSWORD_VALIDATORS = AUTH_PASSWORD_VALIDATORS
+
 ALLOWED_HOSTS = [".ietf.org", ".ietf.org.", "209.208.19.216", "4.31.198.44", "127.0.0.1", "localhost", ]
 
 # Server name of the tools server
diff --git a/ietf/settings_test.py b/ietf/settings_test.py
index b5da9b833b..9a42e8b99d 100755
--- a/ietf/settings_test.py
+++ b/ietf/settings_test.py
@@ -14,7 +14,7 @@
 import shutil
 import tempfile
 from ietf.settings import *                                          # pyflakes:ignore
-from ietf.settings import TEST_CODE_COVERAGE_CHECKER
+from ietf.settings import TEST_CODE_COVERAGE_CHECKER, ORIG_AUTH_PASSWORD_VALIDATORS
 import debug                            # pyflakes:ignore
 debug.debug = True
 
@@ -110,11 +110,8 @@ def tempdir_with_cleanup(**kwargs):
     },
 }
 
-# Configure storages for the blob store - use env settings if present. See the --no-manage-blobstore test option.
-_blob_store_endpoint_url = os.environ.get("DATATRACKER_BLOB_STORE_ENDPOINT_URL", "http://blobstore:9000")
-_blob_store_access_key = os.environ.get("DATATRACKER_BLOB_STORE_ACCESS_KEY", "minio_root")
-_blob_store_secret_key = os.environ.get("DATATRACKER_BLOB_STORE_SECRET_KEY", "minio_pass")
-_blob_store_bucket_prefix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "test-")
-_blob_store_enable_profiling = (
-    os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true"
-)
+# Restore AUTH_PASSWORD_VALIDATORS if they were reset in settings_local
+try:
+    AUTH_PASSWORD_VALIDATORS = ORIG_AUTH_PASSWORD_VALIDATORS
+except NameError:
+    pass
diff --git a/ietf/static/js/password_strength.js b/ietf/static/js/password_strength.js
index 4df5c14439..74a245fb5c 100644
--- a/ietf/static/js/password_strength.js
+++ b/ietf/static/js/password_strength.js
@@ -57,22 +57,40 @@
                         .parent()
                         .parent()
                         .find('.password_strength_offline_info');
+                    let password_improvement_hint = $(password_strength_info)
+                        .find('.password_improvement_hint');
 
                     if ($(this)
                         .val()) {
                         var result = zxcvbn($(this)
                             .val());
-
-                        if (result.score < 3) {
-                            password_strength_bar.removeClass('text-bg-success')
-                                .addClass('text-bg-warning');
-                            password_strength_info.find('.badge')
-                                .removeClass('d-none');
+                        const enforceStrength = !('disableStrengthEnforcement' in this.dataset);
+                        const strongEnough = !enforceStrength || (result.score >= 3);
+                        if (strongEnough) {
+                            // Mark input as valid
+                            this.setCustomValidity('');
                         } else {
+                            // Mark input as invalid
+                            this.setCustomValidity('This password does not meet complexity requirements');
+                        }
+                        
+                        if (this.checkValidity()) {
                             password_strength_bar.removeClass('text-bg-warning')
                                 .addClass('text-bg-success');
                             password_strength_info.find('.badge')
                                 .addClass('d-none');
+                            this.classList.remove('is-invalid')
+                            password_improvement_hint.addClass('d-none');
+                            password_improvement_hint.text('')
+                        } else {
+                            this.classList.add('is-invalid')
+                            password_improvement_hint.text(this.validationMessage)
+                            password_improvement_hint.removeClass('d-none');
+
+                            password_strength_bar.removeClass('text-bg-success')
+                                .addClass('text-bg-warning');
+                            password_strength_info.find('.badge')
+                                .removeClass('d-none');
                         }
 
                         password_strength_bar.width(((result.score + 1) / 5) * 100 + '%')
@@ -152,23 +170,31 @@
                     .data('confirm-with');
 
                 if (confirm_with && confirm_with == password_field.attr('id')) {
-                    if (confirm_value && password) {
+                    if (password) {
                         if (confirm_value === password) {
                             $(confirm_field)
                                 .parent()
                                 .find('.password_strength_info')
                                 .addClass('d-none');
+                            confirm_field.setCustomValidity('')
+                            confirm_field.classList.remove('is-invalid')
                         } else {
-                            $(confirm_field)
-                                .parent()
-                                .find('.password_strength_info')
-                                .removeClass('d-none');
+                            if (confirm_value !== '') {
+                                $(confirm_field)
+                                    .parent()
+                                    .find('.password_strength_info')
+                                    .removeClass('d-none');
+                            }
+                            confirm_field.setCustomValidity('Does not match new password')
+                            confirm_field.classList.add('is-invalid')
                         }
                     } else {
                         $(confirm_field)
                             .parent()
                             .find('.password_strength_info')
                             .addClass('d-none');
+                        confirm_field.setCustomValidity('')
+                        confirm_field.classList.remove('is-invalid')
                     }
                 }
             });
diff --git a/ietf/templates/registration/change_password.html b/ietf/templates/registration/change_password.html
index 21c102bd0a..58bc2d2587 100644
--- a/ietf/templates/registration/change_password.html
+++ b/ietf/templates/registration/change_password.html
@@ -34,11 +34,14 @@ <h1>Change password</h1>
             <button type="submit" class="btn btn-primary">Change password</button>
         </form>
         <div class="alert alert-info mt-5">
-            <b>Online attack: </b>This password form uses the
+            <b>Password strength requirements: </b>
+            You must choose a password at least 12 characters long that scores at least a 3 according to the
             <a href="https://blogs.dropbox.com/tech/2012/04/zxcvbn-realistic-password-strength-estimation/">zxcvbn</a>
-            password strength estimator to give an indication of password strength.
-            The crack time estimate given above assume online attack without rate
-            limiting, at a rate of 10 attempts per second.
+            password strength estimator. A warning will appear if your password does not meet this standard.
+            <hr>
+            <b>Online attack: </b>
+            The crack time estimate given above assumes an online attack at a rate of 10 attempts per second.
+            It is only a very rough guideline.
             <hr>
             <b>Offline cracking: </b>
             The datatracker currently uses the <code>{{ hasher.algorithm }}</code>
diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 074888728f..7a3c369750 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -391,3 +391,8 @@ def _multiline_to_list(s):
     "EXCLUDE_BUCKETS": ["staging"],
     "VERBOSE_LOGGING": _blobdb_replication_verbose_logging,
 }
+
+# Optionally disable password strength enforcement at login (on by default)
+PASSWORD_POLICY_ENFORCE_AT_LOGIN = (
+    os.environ.get("DATATRACKER_ENFORCE_PW_POLICY", "true").lower() != "false"
+)
diff --git a/requirements.txt b/requirements.txt
index eb72600fe3..4eb573ce36 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -79,8 +79,10 @@ selenium>=4.0
 tblib>=1.7.0    # So that the django test runner provides tracebacks
 tlds>=2022042700    # Used to teach bleach about which TLDs currently exist
 tqdm>=4.64.0
+types-zxcvbn~=4.5.0.20250223  # match zxcvbn version
 Unidecode>=1.3.4
 urllib3>=1.26,<2
 weasyprint>=64.1
 xml2rfc>=3.23.0
 xym>=0.6,<1.0
+zxcvbn>=4.5.0

From e8252aa84ef66058e23716e46145ae129152cdac Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Tue, 24 Jun 2025 15:58:14 +0000
Subject: [PATCH 365/656] ci: update base image target version to 20250624T1543

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 3ad11e4d9e..8317195446 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250514T1627
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250624T1543
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 4bd4db561e..1f2e39a0a2 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250514T1627
+20250624T1543

From ec3aec189afb872b1ffe1c00952bac20d6712897 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 24 Jun 2025 14:34:46 -0300
Subject: [PATCH 366/656] fix: remove stale "contains" docs on rfc-index sync
 (#9049)

* fix: clean out stale "contains" docs correctly

* style: ruff ruff + copyright

* refactor: safer "contains" relation removal

* test: replacement of subseries doc in sync
---
 ietf/sync/rfceditor.py |  47 ++++++++++------
 ietf/sync/tests.py     | 122 ++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 152 insertions(+), 17 deletions(-)

diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 889317cdcf..a3c6580452 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2012-2020, All Rights Reserved
+# Copyright The IETF Trust 2012-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -12,6 +12,7 @@
 from xml.dom import pulldom, Node
 
 from django.conf import settings
+from django.db import transaction
 from django.db.models import Subquery, OuterRef, F, Q
 from django.utils import timezone
 from django.utils.encoding import smart_bytes, force_str
@@ -30,9 +31,9 @@
 from ietf.utils.mail import send_mail_text
 from ietf.utils.timezone import datetime_from_date, RPC_TZINFO
 
-#QUEUE_URL = "https://www.rfc-editor.org/queue2.xml"
-#INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml"
-#POST_APPROVED_DRAFT_URL = "https://www.rfc-editor.org/sdev/jsonexp/jsonparser.php"
+# QUEUE_URL = "https://www.rfc-editor.org/queue2.xml"
+# INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml"
+# POST_APPROVED_DRAFT_URL = "https://www.rfc-editor.org/sdev/jsonexp/jsonparser.php"
 
 MIN_ERRATA_RESULTS = 5000
 MIN_INDEX_RESULTS = 8000
@@ -427,7 +428,7 @@ def update_docs_from_rfc_index(
                 pass
                 # Logging below warning turns out to be unhelpful - there are many references
                 # to such things in the index:
-                # * all april-1 RFCs have an internal name that looks like a draft name, but there 
+                # * all april-1 RFCs have an internal name that looks like a draft name, but there
                 # was never such a draft. More of these will exist in the future
                 # * Several documents were created with out-of-band input to the RFC-editor, for a
                 # variety of reasons.
@@ -436,7 +437,7 @@ def update_docs_from_rfc_index(
                 # If there is no draft to point to, don't point to one, even if there was an RPC
                 # internal name in use (and in the RPC database). This will be a requirement on the
                 # reimplementation of the creation of the rfc-index.
-                # 
+                #
                 # log(f"Warning: RFC index for {rfc_number} referred to unknown draft {draft_name}")
 
         # Find or create the RFC document
@@ -466,7 +467,7 @@ def update_docs_from_rfc_index(
             if draft:
                 doc.formal_languages.set(draft.formal_languages.all())
                 for author in draft.documentauthor_set.all():
-                    # Copy the author but point at the new doc. 
+                    # Copy the author but point at the new doc.
                     # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances
                     author.pk = None
                     author.id = None
@@ -707,12 +708,27 @@ def parse_relation_list(l):
                         subseries_doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Added {doc.name} to {subseries_doc.name}")
                         rfc_events.append(doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Added {doc.name} to {subseries_doc.name}"))
 
-        for subdoc in doc.related_that("contains"):
-            if subdoc.name not in also:
-                assert(not first_sync_creating_subseries)
-                subseries_doc.relateddocument_set.filter(target=subdoc).delete()
-                rfc_events.append(doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Removed {doc.name} from {subseries_doc.name}"))
-                subseries_doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Removed {doc.name} from {subseries_doc.name}")
+        # Delete subseries relations that are no longer current. Use a transaction
+        # so we are sure we iterate over the same relations that we delete!
+        with transaction.atomic():
+            stale_subseries_relations = doc.relations_that("contains").exclude(
+                source__name__in=also
+            )
+            for stale_relation in stale_subseries_relations:
+                stale_subseries_doc = stale_relation.source
+                rfc_events.append(
+                    doc.docevent_set.create(
+                        type="sync_from_rfc_editor",
+                        by=system,
+                        desc=f"Removed {doc.name} from {stale_subseries_doc.name}",
+                    )
+                )
+                stale_subseries_doc.docevent_set.create(
+                    type="sync_from_rfc_editor",
+                    by=system,
+                    desc=f"Removed {doc.name} from {stale_subseries_doc.name}",
+                )
+            stale_subseries_relations.delete()
 
         doc_errata = errata.get(f"RFC{rfc_number}", [])
         all_rejected = doc_errata and all(
@@ -754,9 +770,9 @@ def parse_relation_list(l):
             )
             doc.save_with_history(rfc_events)
             yield rfc_number, rfc_changes, doc, rfc_published  # yield changes to the RFC
-    
+
     if first_sync_creating_subseries:
-        # First - create the known subseries documents that have ghosted. 
+        # First - create the known subseries documents that have ghosted.
         # The RFC editor (as of 31 Oct 2023) claims these subseries docs do not exist.
         # The datatracker, on the other hand, will say that the series doc currently contains no RFCs.
         for name in ["fyi17", "std1", "bcp12", "bcp113", "bcp66"]:
@@ -769,7 +785,6 @@ def parse_relation_list(l):
                 subseries_slug = name[:3]
                 subseries_doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of this {subseries_slug.upper()} document is currently available in the datatracker before this point")
 
-
         RelatedDocument.objects.filter(
             Q(originaltargetaliasname__startswith="bcp") |
             Q(originaltargetaliasname__startswith="std") |
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index b0cdf863f0..14d65de0b2 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -20,7 +20,13 @@
 import debug                            # pyflakes:ignore
 
 from ietf.api.views import EmailIngestionError
-from ietf.doc.factories import WgDraftFactory, RfcFactory, DocumentAuthorFactory, DocEventFactory
+from ietf.doc.factories import (
+    WgDraftFactory,
+    RfcFactory,
+    DocumentAuthorFactory,
+    DocEventFactory,
+    BcpFactory,
+)
 from ietf.doc.models import Document, DocEvent, DeletedEvent, DocTagName, RelatedDocument, State, StateDocEvent
 from ietf.doc.utils import add_state_change_event
 from ietf.group.factories import GroupFactory
@@ -508,6 +514,120 @@ def test_rfc_index(self):
         changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)))
         self.assertEqual(len(changed), 0)
 
+    def test_rfc_index_subseries_replacement(self):
+        today = date_today()
+        author = PersonFactory(name="Some Bozo")
+
+        # Start with two BCPs, each containing an rfc
+        rfc1, rfc2, rfc3 = RfcFactory.create_batch(3, authors=[author])
+        bcp1 = BcpFactory(contains=[rfc1])
+        bcp2 = BcpFactory(contains=[rfc2])
+        
+        def _nameify(doc):
+            """Convert a name like 'rfc1' to 'RFC0001"""
+            return f"{doc.name[:3].upper()}{int(doc.name[3:]):04d}"
+
+        # RFC index that replaces rfc2 with rfc3 in bcp2
+        index_xml = f"""<?xml version="1.0" encoding="UTF-8"?>
+<rfc-index xmlns="http://www.rfc-editor.org/rfc-index"
+           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
+           xsi:schemaLocation="http://www.rfc-editor.org/rfc-index 
+                               http://www.rfc-editor.org/rfc-index.xsd">
+    <bcp-entry>
+        <doc-id>{_nameify(bcp1)}</doc-id>
+        <is-also>
+            <doc-id>{_nameify(rfc1)}</doc-id>
+        </is-also>
+    </bcp-entry>
+    <bcp-entry>
+        <doc-id>{_nameify(bcp2)}</doc-id>
+        <is-also>
+            <doc-id>{_nameify(rfc3)}</doc-id>
+        </is-also>
+    </bcp-entry>
+    <rfc-entry>
+        <doc-id>{_nameify(rfc1)}</doc-id>
+        <title>{rfc1.title}</title>
+        <author>
+            <name>Some Bozo</name>
+        </author>
+        <date>
+            <month>{today.strftime('%B')}</month>
+            <year>{today.strftime('%Y')}</year>
+        </date>
+        <format>
+            <file-format>ASCII</file-format>
+        </format>
+        <page-count>42</page-count>
+        <keywords>
+            <kw>test</kw>
+        </keywords>
+        <abstract><p>This is some interesting text.</p></abstract>
+        <is-also>
+            <doc-id>{_nameify(bcp1)}</doc-id>
+        </is-also>
+        <current-status>PROPOSED STANDARD</current-status>
+        <publication-status>PROPOSED STANDARD</publication-status>
+        <stream>IETF</stream>
+    </rfc-entry>
+    <rfc-entry>
+        <doc-id>{_nameify(rfc2)}</doc-id>
+        <title>{rfc2.title}</title>
+        <author>
+            <name>Some Bozo</name>
+        </author>
+        <date>
+            <month>{today.strftime('%B')}</month>
+            <year>{today.strftime('%Y')}</year>
+        </date>
+        <format>
+            <file-format>ASCII</file-format>
+        </format>
+        <page-count>42</page-count>
+        <keywords>
+            <kw>test</kw>
+        </keywords>
+        <abstract><p>This is some interesting text.</p></abstract>
+        <current-status>PROPOSED STANDARD</current-status>
+        <publication-status>PROPOSED STANDARD</publication-status>
+        <stream>IETF</stream>
+    </rfc-entry>
+    <rfc-entry>
+        <doc-id>{_nameify(rfc3)}</doc-id>
+        <title>{rfc3.title}</title>
+        <author>
+            <name>Some Bozo</name>
+        </author>
+        <date>
+            <month>{today.strftime('%B')}</month>
+            <year>{today.strftime('%Y')}</year>
+        </date>
+        <format>
+            <file-format>ASCII</file-format>
+        </format>
+        <page-count>42</page-count>
+        <keywords>
+            <kw>test</kw>
+        </keywords>
+        <abstract><p>This is some interesting text.</p></abstract>
+        <is-also>
+            <doc-id>{_nameify(bcp2)}</doc-id>
+        </is-also>
+        <current-status>PROPOSED STANDARD</current-status>
+        <publication-status>PROPOSED STANDARD</publication-status>
+        <stream>IETF</stream>
+    </rfc-entry>
+</rfc-index>"""
+        data = rfceditor.parse_index(io.StringIO(index_xml))  # parse index
+        self.assertEqual(len(data), 3)  # check that we parsed 3 RFCs
+        # Process the data by consuming the generator
+        for _ in rfceditor.update_docs_from_rfc_index(data, []):
+            pass
+        # Confirm that the expected changes were made
+        self.assertCountEqual(rfc1.related_that("contains"), [bcp1])
+        self.assertCountEqual(rfc2.related_that("contains"), [])
+        self.assertCountEqual(rfc3.related_that("contains"), [bcp2])
+
     def _generate_rfc_queue_xml(self, draft, state, auth48_url=None):
         """Generate an RFC queue xml string for a draft"""
         t = '''<rfc-editor-queue xmlns="http://www.rfc-editor.org/rfc-editor-queue">

From 5ede134ec5710309565cd8260204bad4b0804603 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 24 Jun 2025 13:02:57 -0500
Subject: [PATCH 367/656] fix: task that repairs bad docevents related to
 subseries removals (#9053)

---
 ietf/sync/tasks.py | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py
index 53e23d7913..18ab4fe66e 100644
--- a/ietf/sync/tasks.py
+++ b/ietf/sync/tasks.py
@@ -11,6 +11,7 @@
 from django.conf import settings
 from django.utils import timezone
 
+from ietf.doc.models import DocEvent, RelatedDocument
 from ietf.sync import iana
 from ietf.sync import rfceditor
 from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue
@@ -180,3 +181,44 @@ def batched(l, n):
 
         for d in updated:
             log.log("Added history entry for %s" % d.display_name())
+
+@shared_task
+def fix_subseries_docevents_task():
+    """Repairs DocEvents related to bugs around removing docs from subseries
+
+    Removes bogus and repairs the date of non-bogus DocEvents
+    about removing RFCs from subseries
+
+    This is designed to be a one-shot task that should be removed
+    after running it. It is intended to be safe if it runs more than once.
+    """
+    log.log("Repairing DocEvents related to bugs around removing docs from subseries")
+    bogus_event_descs = [
+        "Removed rfc8499 from bcp218",
+        "Removed rfc7042 from bcp184",
+        "Removed rfc9499 from bcp238",
+        "Removed rfc5033 from std74",
+        "Removed rfc3228 from bcp55",
+        "Removed rfc8109 from std85",
+    ]
+    DocEvent.objects.filter(
+        type="sync_from_rfc_editor", desc__in=bogus_event_descs
+    ).delete()
+    needs_moment_fix = [
+        "Removed rfc8499 from bcp219",
+        "Removed rfc7042 from bcp141",
+        "Removed rfc5033 from bcp133",
+        "Removed rfc3228 from bcp57",
+    ]
+    # Assumptions (which have been manually verified):
+    # 1) each of the above RFCs is obsoleted by exactly one other RFC
+    # 2) each of the obsoleting RFCs has exactly one published_rfc docevent
+    for desc in needs_moment_fix:
+        obsoleted_rfc_name = desc.split(" ")[1]
+        obsoleting_rfc = RelatedDocument.objects.get(
+            relationship_id="obs", target__name=obsoleted_rfc_name
+        ).source
+        obsoleting_time = obsoleting_rfc.docevent_set.get(type="published_rfc").time
+        DocEvent.objects.filter(type="sync_from_rfc_editor", desc=desc).update(
+            time=obsoleting_time
+        )

From 32ff440ca7f6ceb34f54460bd09e6315ec80d530 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 24 Jun 2025 15:15:25 -0300
Subject: [PATCH 368/656] chore(k8s): config reg participants API key/url
 (#9054)

* chore(k8s): config reg participants API key/url

* fix: typo
---
 k8s/settings_local.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 7a3c369750..482a4b110a 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -165,6 +165,21 @@ def _multiline_to_list(s):
     raise RuntimeError("DATATRACKER_REGISTRATION_API_KEY must be set")
 STATS_REGISTRATION_ATTENDEES_JSON_URL = f"https://registration.ietf.org/{{number}}/attendees/?apikey={_registration_api_key}"
 
+# Registration Participants API config - key must be set, but the URL can be left
+# to the default in settings.py
+_registration_participants_api_key = os.environ.get(
+    "DATATRACKER_REGISTRATION_PARTICIPANTS_API_KEY", None
+)
+if _registration_participants_api_key is None:
+    raise RuntimeError("DATATRACKER_REGISTRATION_PARTICIPANTS_API_KEY must be set")
+REGISTRATION_PARTICIPANTS_API_KEY = _registration_participants_api_key
+
+_registration_participants_api_url = os.environ.get(
+    "DATATRACKER_REGISTRATION_PARTICIPANTS_API_URL", None
+)
+if _registration_participants_api_url is not None:
+    REGISTRATION_PARTICIPANTS_API_URL = _registration_participants_api_url
+
 # FIRST_CUTOFF_DAYS = 12
 # SECOND_CUTOFF_DAYS = 12
 # SUBMISSION_CUTOFF_DAYS = 26

From 0197e44d9a89cbaf8e5f68f42576518ba3f93df2 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 24 Jun 2025 17:05:48 -0500
Subject: [PATCH 369/656] feat: add liaison coordinators, remove from
 restrictions (#9055)

* refactor: clean up outgoing LS from field init

* feat: additional LS "from" groups for IETF/IAB Chair+ADs

* refactor: reduce queries in get_internal_choices

* refactor: break down / rename get_groups_for_person

* refactor: inline / remove unneeded methods

* refactor: colocate similar field config

* refactor: unify role logic for LS To fields

* fix: typo

* refactor: update EditLiaisonForm to match changes

* refactor: update IncomingLiaisonForm to match

* fix: typo / add docstring

* test: framing for new tests; test_flatten_choices()

* test: test_choices_from_group_queryset()

* test: test_all_internal_groups()

* fix: no person = no internal groups for LS

* test: test_all_internal_groups()

* test: test_external_groups_for_person()

* chore: adjust iab execdir and sdo auth'd individs

Need to confirm what is actually needed, but without
these users may be able to create/edit LS's they can't
actually fill in.

* fix: f-string flakes

* fix: remove authorized persons from sdo groups

* fix: remove liaison_contact and liaison_cc_contact Role objects

* feat: liaison coordinators (#8944)

* feat: liaison coordinators

* fix: give liaison coordinator the ability to send incoming

* feat: more flexible liaison stmt "From Contact" (#8958)

* refactor: CharField for LS from_contacts

* refactor: replace from_contact field

* style: copyrights + ruff

* fix: listify in the right place

* refactor: better validator

* refactor: use new from_contact field; fix lint

* fix: cleanup

* feat: default addresses for roles

* chore: update liaisons.resources

from_name and to_name were removed in 2015!

* fix: improve managing liaison coordinators (#8960)

* fix: align liaison coordinator perms with secretariat perms

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 .../0005_remove_sdo_authorized_individuals.py | 192 +++++++++++
 .../migrations/0006_remove_liason_contacts.py | 270 +++++++++++++++
 ietf/group/migrations/0007_used_roles.py      |  49 +++
 ietf/ietfauth/utils.py                        |   4 +
 ietf/liaisons/admin.py                        |   4 +-
 ietf/liaisons/factories.py                    |   2 +-
 ietf/liaisons/forms.py                        | 320 ++++++++++--------
 .../0003_liaisonstatement_from_contact_tmp.py |  22 ++
 ...ulate_liaisonstatement_from_contact_tmp.py |  60 ++++
 ...5_replace_liaisonstatement_from_contact.py |  20 ++
 ietf/liaisons/models.py                       |  12 +-
 ietf/liaisons/resources.py                    |   6 +-
 ietf/liaisons/tests.py                        |  28 +-
 ietf/liaisons/tests_forms.py                  | 229 +++++++++++++
 ietf/liaisons/utils.py                        |  25 +-
 ietf/liaisons/views.py                        |  10 +-
 ietf/name/fixtures/names.json                 |  46 ++-
 ietf/name/migrations/0018_alter_rolenames.py  |  36 ++
 ietf/templates/liaisons/detail.html           |   2 +-
 ietf/templates/liaisons/liaison_mail.txt      |   2 +-
 ietf/utils/validators.py                      |  23 +-
 21 files changed, 1194 insertions(+), 168 deletions(-)
 create mode 100644 ietf/group/migrations/0005_remove_sdo_authorized_individuals.py
 create mode 100644 ietf/group/migrations/0006_remove_liason_contacts.py
 create mode 100644 ietf/group/migrations/0007_used_roles.py
 create mode 100644 ietf/liaisons/migrations/0003_liaisonstatement_from_contact_tmp.py
 create mode 100644 ietf/liaisons/migrations/0004_populate_liaisonstatement_from_contact_tmp.py
 create mode 100644 ietf/liaisons/migrations/0005_replace_liaisonstatement_from_contact.py
 create mode 100644 ietf/liaisons/tests_forms.py
 create mode 100644 ietf/name/migrations/0018_alter_rolenames.py

diff --git a/ietf/group/migrations/0005_remove_sdo_authorized_individuals.py b/ietf/group/migrations/0005_remove_sdo_authorized_individuals.py
new file mode 100644
index 0000000000..77fe25b467
--- /dev/null
+++ b/ietf/group/migrations/0005_remove_sdo_authorized_individuals.py
@@ -0,0 +1,192 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from collections import defaultdict
+
+from django.db import migrations
+
+from ietf.person.name import plain_name
+
+
+def get_plain_name(person):
+    return person.plain or plain_name(person.name)
+
+
+def forward(apps, schema_editor):
+    """Remove any 'auth' Role objects for groups of type 'sdo'
+
+    The IAB has decided that the Authorized Individual concept for
+    authorizing entry or management of liaison statments hasn't worked
+    well - the roles for the groups are not being maintained, Instead,
+    the concept will be removed and the liaison managers or secretariat
+    (and soon the liaison coordinators) will operate the liaison tool
+    on their behalf.
+    """
+    Role = apps.get_model("group", "Role")
+    GroupEvent = apps.get_model("group", "GroupEvent")
+    groups = defaultdict(list)
+    role_qs = Role.objects.filter(name_id="auth", group__type_id="sdo")
+    for role in role_qs:
+        groups[role.group].append(role)
+    for group in groups:
+        desc = f"Removed Authorized Persons: {', '.join([get_plain_name(role.person) for role in groups[group]])}"
+        GroupEvent.objects.create(
+            group=group,
+            by_id=1,  # (System)
+            desc=desc,
+        )
+    role_qs.delete()
+
+
+def reverse(apps, schema_editor):
+    """Intentionally does nothing"""
+    pass
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("group", "0004_modern_list_archive"),
+    ]
+
+    operations = [migrations.RunPython(forward, reverse)]
+
+
+# At the time this migration was created, it would have removed these Role objects:
+# { "authorized_individuals" : [
+# {"person_id": 107937, "group_id": 56, "email": "hannu.hietalahti@nokia.com" }, # Hannu Hietalahti is Authorized Individual in 3gpp
+# {"person_id": 107943, "group_id": 56, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Authorized Individual in 3gpp
+# {"person_id": 112807, "group_id": 56, "email": "Paolo.Usai@etsi.org" }, # Paolo Usai is Authorized Individual in 3gpp
+# {"person_id": 105859, "group_id": 56, "email": "atle.monrad@ericsson.com" }, # Atle Monrad is Authorized Individual in 3gpp
+# {"person_id": 116149, "group_id": 1907, "email": "tsgsx_chair@3GPP2.org" }, # Xiaowu Zhao is Authorized Individual in 3gpp2-tsg-sx
+# {"person_id": 120914, "group_id": 1902, "email": "ozgur.oyman@intel.com" }, # Ozgur Oyman is Authorized Individual in 3gpp-tsgsa-sa4
+# {"person_id": 107943, "group_id": 1902, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Authorized Individual in 3gpp-tsgsa-sa4
+# {"person_id": 119203, "group_id": 1902, "email": "fanyanping@huawei.com" }, # Yanping Fan is Authorized Individual in 3gpp-tsgsa-sa4
+# {"person_id": 112977, "group_id": 1902, "email": "tomas.frankkila@ericsson.com" }, # Tomas Frankkila is Authorized Individual in 3gpp-tsgsa-sa4
+# {"person_id": 120240, "group_id": 2019, "email": "CM8655@att.com" }, # Peter Musgrove is Authorized Individual in atis-eloc-tf
+# {"person_id": 120241, "group_id": 2019, "email": "Christian.Militeau@intrado.com" }, # Christian Militeau is Authorized Individual in atis-eloc-tf
+# {"person_id": 120243, "group_id": 2019, "email": "ablasgen@atis.org" }, # Alexandra Blasgen is Authorized Individual in atis-eloc-tf
+# {"person_id": 114696, "group_id": 67, "email": "KEN.KO@adtran.com" }, # Ken Ko is Authorized Individual in broadband-forum
+# {"person_id": 119494, "group_id": 67, "email": "michael.fargano@centurylink.com" }, # Michael Fargano is Authorized Individual in broadband-forum
+# {"person_id": 124318, "group_id": 67, "email": "joey.boyd@adtran.com" }, # Joey Boyd is Authorized Individual in broadband-forum
+# {"person_id": 114762, "group_id": 67, "email": "bwelch@juniper.net" }, # Bill Welch is Authorized Individual in broadband-forum
+# {"person_id": 112837, "group_id": 67, "email": "christophe.alter@orange.com" }, # Christophe Alter is Authorized Individual in broadband-forum
+# {"person_id": 141083, "group_id": 2407, "email": "dan.middleton@intel.com" }, # Dan Middleton is Authorized Individual in confidential-computing-consortium
+# {"person_id": 117421, "group_id": 1933, "email": "chairman@dmtf.org" }, # Winston Bumpus is Authorized Individual in dmtf
+# {"person_id": 116529, "group_id": 1919, "email": "istvan@ecma-international.org" }, # Istvan Sebestyen is Authorized Individual in ecma-tc39
+# {"person_id": 116363, "group_id": 1915, "email": "e2nasupport@etsi.org" }, # Sonia Compans is Authorized Individual in etsi-e2na
+# {"person_id": 116862, "group_id": 2003, "email": "latif@ladid.lu" }, # Latif Ladid is Authorized Individual in etsi-isg-ip6
+# {"person_id": 116283, "group_id": 2198, "email": "adrian.neal@vodafone.com" }, # Adrian Neal is Authorized Individual in etsi-isg-mec
+# {"person_id": 119412, "group_id": 2004, "email": "jkfernic@uwaterloo.ca" }, # Jennifer Fernick is Authorized Individual in etsi-isg-qsc
+# {"person_id": 122406, "group_id": 2165, "email": "d.lake@surrey.ac.uk" }, # David Lake is Authorized Individual in etsi-ngp
+# {"person_id": 122407, "group_id": 2165, "email": "andy.sutton@ee.co.uk" }, # Andy Sutton is Authorized Individual in etsi-ngp
+# {"person_id": 112609, "group_id": 2165, "email": "richard.li@futurewei.com" }, # Richard Li is Authorized Individual in etsi-ngp
+# {"person_id": 122406, "group_id": 2177, "email": "d.lake@surrey.ac.uk" }, # David Lake is Authorized Individual in etsi-ngp-isp
+# {"person_id": 112609, "group_id": 2177, "email": "richard.li@futurewei.com" }, # Richard Li is Authorized Individual in etsi-ngp-isp
+# {"person_id": 122407, "group_id": 2177, "email": "andy.sutton@ee.co.uk" }, # Andy Sutton is Authorized Individual in etsi-ngp-isp
+# {"person_id": 118527, "group_id": 1986, "email": "luca.pesando@telecomitalia.it" }, # Luca Pesando is Authorized Individual in etsi-ntech
+# {"person_id": 118526, "group_id": 1986, "email": "NTECHsupport@etsi.org" }, # Sylwia Korycinska is Authorized Individual in etsi-ntech
+# {"person_id": 116052, "group_id": 1904, "email": "Beniamino.gorini@alcatel-lucent.com" }, # Gorini Beniamino is Authorized Individual in etsi-tc-ee
+# {"person_id": 19651, "group_id": 63, "email": "glenn.parsons@ericsson.com" }, # Glenn Parsons is Authorized Individual in ieee-802-1
+# {"person_id": 107599, "group_id": 63, "email": "tony@jeffree.co.uk" }, # Tony Jeffree is Authorized Individual in ieee-802-1
+# {"person_id": 117415, "group_id": 1862, "email": "Adrian.P.Stephens@intel.com" }, # Adrian Stephens is Authorized Individual in ieee-802-11
+# {"person_id": 106284, "group_id": 1862, "email": "dstanley@arubanetworks.com" }, # Dorothy Stanley is Authorized Individual in ieee-802-11
+# {"person_id": 114106, "group_id": 1871, "email": "r.b.marks@ieee.org" }, # Roger Marks is Authorized Individual in ieee-802-16
+# {"person_id": 101753, "group_id": 1885, "email": "max.riegel@ieee.org" }, # Max Riegel is Authorized Individual in ieee-802-ec-omniran
+# {"person_id": 113810, "group_id": 1859, "email": "jehrig@inventures.com" }, # John Ehrig is Authorized Individual in imtc
+# {"person_id": 123010, "group_id": 48, "email": "Emil.Kowalczyk@orange.com" }, # Emil Kowalczyk is Authorized Individual in iso-iec-jtc1-sc2
+# {"person_id": 11182, "group_id": 48, "email": "paf@netnod.se" }, # Patrik Fältström is Authorized Individual in iso-iec-jtc1-sc2
+# {"person_id": 117429, "group_id": 1939, "email": "krystyna.passia@din.de" }, # Krystyna Passia is Authorized Individual in iso-iec-jtc1-sc27
+# {"person_id": 117428, "group_id": 1939, "email": "walter.fumy@bdr.de" }, # Walter Fumy is Authorized Individual in iso-iec-jtc1-sc27
+# {"person_id": 114435, "group_id": 74, "email": "watanabe@itscj.ipsj.or.jp" }, # Shinji Watanabe is Authorized Individual in iso-iec-jtc1-sc29-wg11
+# {"person_id": 112106, "group_id": 49, "email": "jooran@kisi.or.kr" }, # Jooran Lee is Authorized Individual in iso-iec-jtc1-sc6
+# {"person_id": 17037, "group_id": 49, "email": "dykim@comsun.chungnnam.ac.kr" }, # Dae Kim is Authorized Individual in iso-iec-jtc1-sc6
+# {"person_id": 117426, "group_id": 1938, "email": "chair@jtc1-sc7.org" }, # Francois Coallier is Authorized Individual in iso-iec-jtc1-sc7
+# {"person_id": 117427, "group_id": 1938, "email": "secretariat@jtc1-sc7.org" }, # Witold Suryn is Authorized Individual in iso-iec-jtc1-sc7
+# {"person_id": 118769, "group_id": 2144, "email": "alexandre.petrescu@gmail.com" }, # Alexandre Petrescu is Authorized Individual in isotc204
+# {"person_id": 115544, "group_id": 1890, "email": "sergio.buonomo@itu.int" }, # Sergio Buonomo is Authorized Individual in itu-r
+# {"person_id": 122111, "group_id": 2157, "email": "h.mazar@atdi.com" }, # Haim Mazar is Authorized Individual in itu-r-wp-5c
+# {"person_id": 115544, "group_id": 2157, "email": "sergio.buonomo@itu.int" }, # Sergio Buonomo is Authorized Individual in itu-r-wp-5c
+# {"person_id": 112105, "group_id": 51, "email": "Malcolm.Johnson@itu.int" }, # Malcom Johnson is Authorized Individual in itu-t
+# {"person_id": 113911, "group_id": 1860, "email": "martin.adolph@itu.int" }, # Martin Adolph is Authorized Individual in itu-t-fg-dist
+# {"person_id": 122779, "group_id": 2180, "email": "Leo.Lehmann@bakom.admin.ch" }, # Leo Lehmann is Authorized Individual in itu-t-fg-imt-2020
+# {"person_id": 103383, "group_id": 2180, "email": "peter.ashwoodsmith@huawei.com" }, # Peter Ashwood-Smith is Authorized Individual in itu-t-fg-imt-2020
+# {"person_id": 107300, "group_id": 1872, "email": "tatiana.kurakova@itu.int" }, # Tatiana Kurakova is Authorized Individual in itu-t-jca-cloud
+# {"person_id": 106224, "group_id": 1872, "email": "mmorrow@cisco.com" }, # Monique Morrow is Authorized Individual in itu-t-jca-cloud
+# {"person_id": 105714, "group_id": 1874, "email": "martin.euchner@itu.int" }, # Martin Euchner is Authorized Individual in itu-t-jca-cop
+# {"person_id": 106475, "group_id": 2170, "email": "khj@etri.re.kr" }, # Hyoung-Jun Kim is Authorized Individual in itu-t-jca-iot-scc
+# {"person_id": 122491, "group_id": 2170, "email": "tsbjcaiot@itu.int" }, # ITU Tsb is Authorized Individual in itu-t-jca-iot-scc
+# {"person_id": 122490, "group_id": 2170, "email": "fabio.bigi@virgilio.it" }, # Fabio Bigi is Authorized Individual in itu-t-jca-iot-scc
+# {"person_id": 116952, "group_id": 1927, "email": "chengying10@chinaunicom.cn" }, # Ying Cheng is Authorized Individual in itu-t-jca-sdn
+# {"person_id": 111205, "group_id": 1927, "email": "t-egawa@ct.jp.nec.com" }, # Takashi Egawa is Authorized Individual in itu-t-jca-sdn
+# {"person_id": 107298, "group_id": 2178, "email": "tsbsg11@itu.int" }, # Arshey Odedra is Authorized Individual in itu-tsbsg-11
+# {"person_id": 107300, "group_id": 77, "email": "tatiana.kurakova@itu.int" }, # Tatiana Kurakova is Authorized Individual in itu-t-sg-11
+# {"person_id": 112573, "group_id": 77, "email": "stefano.polidori@itu.int" }, # Stefano Polidori is Authorized Individual in itu-t-sg-11
+# {"person_id": 115401, "group_id": 84, "email": "spennock@rim.com" }, # Scott Pennock is Authorized Individual in itu-t-sg-12
+# {"person_id": 114255, "group_id": 84, "email": "hiroshi.ota@itu.int" }, # Hiroshi Ota is Authorized Individual in itu-t-sg-12
+# {"person_id": 113032, "group_id": 84, "email": "catherine.quinquis@orange.com" }, # Catherine Quinquis is Authorized Individual in itu-t-sg-12
+# {"person_id": 113031, "group_id": 84, "email": "gunilla.berndtsson@ericsson.com" }, # Gunilla Berndtsson is Authorized Individual in itu-t-sg-12
+# {"person_id": 113672, "group_id": 84, "email": "sarah.scott@itu.int" }, # Sarah Scott is Authorized Individual in itu-t-sg-12
+# {"person_id": 122459, "group_id": 81, "email": "chan@etri.re.kr" }, # Kangchan Lee is Authorized Individual in itu-t-sg-13
+# {"person_id": 107300, "group_id": 81, "email": "tatiana.kurakova@itu.int" }, # Tatiana Kurakova is Authorized Individual in itu-t-sg-13
+# {"person_id": 109145, "group_id": 62, "email": "lihan@chinamobile.com" }, # Han Li is Authorized Individual in itu-t-sg-15
+# {"person_id": 115875, "group_id": 62, "email": "mark.jones@xtera.com" }, # Mark Jones is Authorized Individual in itu-t-sg-15
+# {"person_id": 115846, "group_id": 62, "email": "peter.stassar@huawei.com" }, # Peter Stassar is Authorized Individual in itu-t-sg-15
+# {"person_id": 123452, "group_id": 62, "email": "sshew@ciena.com" }, # Stephen Shew is Authorized Individual in itu-t-sg-15
+# {"person_id": 109312, "group_id": 62, "email": "huubatwork@gmail.com" }, # Huub van Helvoort is Authorized Individual in itu-t-sg-15
+# {"person_id": 115874, "group_id": 62, "email": "tom.huber@tellabs.com" }, # Tom Huber is Authorized Individual in itu-t-sg-15
+# {"person_id": 110799, "group_id": 62, "email": "koike.yoshinori@lab.ntt.co.jp" }, # Yoshinori Koike is Authorized Individual in itu-t-sg-15
+# {"person_id": 110831, "group_id": 62, "email": "kam.lam@nokia.com" }, # Hing-Kam Lam is Authorized Individual in itu-t-sg-15
+# {"person_id": 114255, "group_id": 62, "email": "hiroshi.ota@itu.int" }, # Hiroshi Ota is Authorized Individual in itu-t-sg-15
+# {"person_id": 115874, "group_id": 62, "email": "tom.huber@coriant.com" }, # Tom Huber is Authorized Individual in itu-t-sg-15
+# {"person_id": 123014, "group_id": 62, "email": "jessy.rouyer@nokia.com" }, # Jessy Rouyer is Authorized Individual in itu-t-sg-15
+# {"person_id": 111160, "group_id": 62, "email": "ryoo@etri.re.kr" }, # Jeong-dong Ryoo is Authorized Individual in itu-t-sg-15
+# {"person_id": 107296, "group_id": 62, "email": "greg.jones@itu.int" }, # Greg Jones is Authorized Individual in itu-t-sg-15
+# {"person_id": 118539, "group_id": 72, "email": "rosa.angelesleondev@itu.int" }, # Rosa De Vivero is Authorized Individual in itu-t-sg-16
+# {"person_id": 123169, "group_id": 72, "email": "garysull@microsoft.com" }, # Gary Sullivan is Authorized Individual in itu-t-sg-16
+# {"person_id": 107746, "group_id": 72, "email": "hiwasaki.yusuke@lab.ntt.co.jp" }, # Yusuke Hiwasaki is Authorized Individual in itu-t-sg-16
+# {"person_id": 108160, "group_id": 1987, "email": "Christian.Groves@nteczone.com" }, # Christian Groves is Authorized Individual in itu-t-sg-16-q3
+# {"person_id": 118539, "group_id": 1987, "email": "rosa.angelesleondev@itu.int" }, # Rosa De Vivero is Authorized Individual in itu-t-sg-16-q3
+# {"person_id": 124354, "group_id": 76, "email": "jhbaek@kisa.or.kr" }, # Jonghyun Baek is Authorized Individual in itu-t-sg-17
+# {"person_id": 12898, "group_id": 1937, "email": "youki-k@is.aist-nara.ac.jp" }, # Youki Kadobayashi is Authorized Individual in itu-t-sg-17-q4
+# {"person_id": 113593, "group_id": 79, "email": "maite.comasbarnes@itu.int" }, # Maite Barnes is Authorized Individual in itu-t-sg-3
+# {"person_id": 122983, "group_id": 2000, "email": "cristina.bueti@itu.int" }, # Cristina Bueti is Authorized Individual in itu-t-sg-5
+# {"person_id": 112573, "group_id": 2072, "email": "stefano.polidori@itu.int" }, # Stefano Polidori is Authorized Individual in itu-t-sg-9
+# {"person_id": 113101, "group_id": 82, "email": "steve.trowbridge@alcatel-lucent.com" }, # Stephen Trowbridge is Authorized Individual in itu-t-tsag
+# {"person_id": 20783, "group_id": 82, "email": "reinhard.scholl@itu.int" }, # Reinhard Scholl is Authorized Individual in itu-t-tsag
+# {"person_id": 107300, "group_id": 1846, "email": "tatiana.kurakova@itu.int" }, # Tatiana Kurakova is Authorized Individual in itu-t-wp-5-13
+# {"person_id": 112107, "group_id": 69, "email": "michael.oreirdan@maawg.org" }, # Michael O'Reirdan is Authorized Individual in maawg
+# {"person_id": 121870, "group_id": 75, "email": "liaisons@mef.net" }, # Liaison Mef is Authorized Individual in mef
+# {"person_id": 112510, "group_id": 75, "email": "nan@mef.net" }, # Nan Chen is Authorized Individual in mef
+# {"person_id": 124306, "group_id": 75, "email": "jason.wolfe@bell.ca" }, # WOLFE Jason is Authorized Individual in mef
+# {"person_id": 114454, "group_id": 75, "email": "mike.bencheck@siamasystems.com" }, # Mike Bencheck is Authorized Individual in mef
+# {"person_id": 115327, "group_id": 1888, "email": "klaus.moschner@ngmn.org" }, # Klaus Moschner is Authorized Individual in ngmn
+# {"person_id": 123305, "group_id": 1888, "email": "office@ngmn.org" }, # Office Ngmn is Authorized Individual in ngmn
+# {"person_id": 115160, "group_id": 1888, "email": "jminlee@sk.com" }, # Jongmin Lee is Authorized Individual in ngmn
+# {"person_id": 117424, "group_id": 1936, "email": "patrick.gallagher@nist.gov" }, # Patrick Gallagher is Authorized Individual in nist
+# {"person_id": 117431, "group_id": 1941, "email": "chet.ensign@xn--oasis-open-vt6e.org" }, # Chet Ensign is Authorized Individual in oasis
+# {"person_id": 120913, "group_id": 2142, "email": "james.walker@tatacommunications.com" }, # James Walker is Authorized Individual in occ
+# {"person_id": 6699, "group_id": 2142, "email": "dromasca@gmail.com" }, # Dan Romascanu is Authorized Individual in occ
+# {"person_id": 118403, "group_id": 2142, "email": "richard.schell@verizon.com" }, # Rick Schell is Authorized Individual in occ
+# {"person_id": 109676, "group_id": 83, "email": "Jonathan.Sadler@tellabs.com" }, # Jonathan Sadler is Authorized Individual in oif
+# {"person_id": 122843, "group_id": 2122, "email": "tzhang@omaorg.org" }, # Tiffany Zhang is Authorized Individual in oma
+# {"person_id": 116967, "group_id": 1947, "email": "JMudge@omaorg.org" }, # John Mudge is Authorized Individual in oma-architecture-wg
+# {"person_id": 117423, "group_id": 1935, "email": "soley@omg.org" }, # Richard Soley is Authorized Individual in omg
+# {"person_id": 110831, "group_id": 1858, "email": "kam.lam@nokia.com" }, # Hing-Kam Lam is Authorized Individual in onf
+# {"person_id": 113674, "group_id": 1858, "email": "dan.pitt@opennetworking.org" }, # Dan Pitt is Authorized Individual in onf
+# {"person_id": 118348, "group_id": 1984, "email": "dave.hood@ericsson.com" }, # Dave Hood is Authorized Individual in onf-arch-wg
+# {"person_id": 116967, "group_id": 60, "email": "JMudge@omaorg.org" }, # John Mudge is Authorized Individual in open-mobile-alliance
+# {"person_id": 112613, "group_id": 60, "email": "jerry.shih@att.com" }, # Jerry Shih is Authorized Individual in open-mobile-alliance
+# {"person_id": 113067, "group_id": 60, "email": "laurent.goix@econocom.com" }, # Laurent Goix is Authorized Individual in open-mobile-alliance
+# {"person_id": 112772, "group_id": 60, "email": "zhiyuan.hu@alcatel-sbell.com.cn" }, # Hu Zhiyuan is Authorized Individual in open-mobile-alliance
+# {"person_id": 113064, "group_id": 60, "email": "thierry.berisot@telekom.de" }, # Thierry Berisot is Authorized Individual in open-mobile-alliance
+# {"person_id": 124276, "group_id": 2212, "email": "jmisener@qti.qualcomm.com" }, # Jim Misener is Authorized Individual in sae-cell-v2x
+# {"person_id": 124278, "group_id": 2212, "email": "Keith.Wilson@sae.org" }, # Keith Wilson is Authorized Individual in sae-cell-v2x
+# {"person_id": 124277, "group_id": 2212, "email": "Elizabeth.Perry@sae.org" }, # Elizabeth Perry is Authorized Individual in sae-cell-v2x
+# {"person_id": 117430, "group_id": 1940, "email": "admin@trustedcomputinggroup.org" }, # Lindsay Adamson is Authorized Individual in tcg
+# {"person_id": 117422, "group_id": 1934, "email": "j.hietala@opengroup.org" }, # Jim Hietala is Authorized Individual in the-open-group
+# {"person_id": 112104, "group_id": 53, "email": "rick@unicode.org" }, # Rick McGowan is Authorized Individual in unicode
+# {"person_id": 112103, "group_id": 54, "email": "plh@w3.org" }, # Philippe Le Hégaret is Authorized Individual in w3c
+# {"person_id": 120261, "group_id": 54, "email": "wendy@seltzer.org" }, # Wendy Seltzer is Authorized Individual in w3c
+# {"person_id": 118020, "group_id": 1955, "email": "tiago@wballiance.com" }, # Tiago Rodrigues is Authorized Individual in wba
+# {"person_id": 125489, "group_id": 1955, "email": "bruno@wballiance.com" }, # Bruno Tomas is Authorized Individual in wba
+# {"person_id": 109129, "group_id": 70, "email": "smccammon@amsl.com" }, # Stephanie McCammon is Authorized Individual in zigbee-alliance
+# ]}
diff --git a/ietf/group/migrations/0006_remove_liason_contacts.py b/ietf/group/migrations/0006_remove_liason_contacts.py
new file mode 100644
index 0000000000..13afd1a53e
--- /dev/null
+++ b/ietf/group/migrations/0006_remove_liason_contacts.py
@@ -0,0 +1,270 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from collections import defaultdict
+
+from django.db import migrations
+
+from ietf.person.name import plain_name
+
+
+def get_plain_name(person):
+    return person.plain or plain_name(person.name)
+
+
+def forward(apps, schema_editor):
+    """Removes liaison_contact and liaison_cc_contact roles from all groups
+
+    The IAB has decided to remove the liaison_contact and liaison_cc_contact
+    role concept from the datatracker as the roles are not well understood
+    and have not been being maintained.
+    """
+    Role = apps.get_model("group", "Role")
+    GroupEvent = apps.get_model("group", "GroupEvent")
+    for role_name in ["liaison_contact", "liaison_cc_contact"]:
+        groups = defaultdict(list)
+        role_qs = Role.objects.filter(name_id=role_name)
+        for role in role_qs:
+            groups[role.group].append(role)
+        for group in groups:
+            desc = f"Removed {role_name}: {', '.join([get_plain_name(role.person) for role in groups[group]])}"
+            GroupEvent.objects.create(
+                group=group,
+                by_id=1,  # (System)
+                desc=desc,
+            )
+        role_qs.delete()
+
+
+def reverse(apps, schema_editor):
+    """Intentionally does nothing"""
+    pass
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("group", "0005_remove_sdo_authorized_individuals"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]
+
+
+# At the time this migration was created, it would remove these objects
+# {"liaison_contacts":[
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 56, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp
+#   { "role_name": "liaison_contact", "person_id": 107737, "group_id": 56, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp
+#   { "role_name": "liaison_contact", "person_id": 127959, "group_id": 57, "email": "mahendra@qualcomm.com" }, # Mahendran Ac is Liaison Contact in 3gpp2
+#   { "role_name": "liaison_contact", "person_id": 111440, "group_id": 2026, "email": "georg.mayer.huawei@gmx.com" }, # Georg Mayer is Liaison Contact in 3gpp-tsgct
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2026, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgct
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2027, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgct-ct1
+#   { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2027, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgct-ct1
+#   { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2410, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgct-ct3
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2410, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgct-ct3
+#   { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2028, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgct-ct4
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2028, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgct-ct4
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2029, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgran
+#   { "role_name": "liaison_contact", "person_id": 111440, "group_id": 2029, "email": "georg.mayer.huawei@gmx.com" }, # Georg Mayer is Liaison Contact in 3gpp-tsgran
+#   { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2030, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgran-ran2
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2030, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgran-ran2
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2023, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgsa
+#   { "role_name": "liaison_contact", "person_id": 111440, "group_id": 2023, "email": "georg.mayer.huawei@gmx.com" }, # Georg Mayer is Liaison Contact in 3gpp-tsgsa
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2024, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgsa-sa2
+#   { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2024, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgsa-sa2
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2025, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgsa-sa3
+#   { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2025, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgsa-sa3
+#   { "role_name": "liaison_contact", "person_id": 107737, "group_id": 1902, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgsa-sa4
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 1902, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgsa-sa4
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2031, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgt-wg2
+#   { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2031, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgt-wg2
+#   { "role_name": "liaison_contact", "person_id": 106345, "group_id": 1396, "email": "Menachem.Dodge@ecitele.com" }, # Menachem Dodge is Liaison Contact in adslmib
+#   { "role_name": "liaison_contact", "person_id": 108054, "group_id": 1956, "email": "shengjiang@bupt.edu.cn" }, # Sheng Jiang is Liaison Contact in anima
+#   { "role_name": "liaison_contact", "person_id": 11834, "group_id": 1956, "email": "tte@cs.fau.de" }, # Toerless Eckert is Liaison Contact in anima
+#   { "role_name": "liaison_contact", "person_id": 21684, "group_id": 1805, "email": "barryleiba@computer.org" }, # Barry Leiba is Liaison Contact in appsawg
+#   { "role_name": "liaison_contact", "person_id": 102154, "group_id": 1805, "email": "alexey.melnikov@isode.com" }, # Alexey Melnikov is Liaison Contact in appsawg
+#   { "role_name": "liaison_contact", "person_id": 107279, "group_id": 1805, "email": "yaojk@cnnic.cn" }, # Jiankang Yao is Liaison Contact in appsawg
+#   { "role_name": "liaison_contact", "person_id": 100754, "group_id": 941, "email": "tom.taylor@rogers.com" }, # Tom Taylor is Liaison Contact in avt
+#   { "role_name": "liaison_contact", "person_id": 105873, "group_id": 941, "email": "ron.even.tlv@gmail.com" }, # Roni Even is Liaison Contact in avt
+#   { "role_name": "liaison_contact", "person_id": 105097, "group_id": 1813, "email": "keith.drage@alcatel-lucent.com" }, # Keith Drage is Liaison Contact in avtext
+#   { "role_name": "liaison_contact", "person_id": 101923, "group_id": 1813, "email": "jonathan@vidyo.com" }, # Jonathan Lennox is Liaison Contact in avtext
+#   { "role_name": "liaison_contact", "person_id": 108279, "group_id": 1960, "email": "martin.vigoureux@alcatel-lucent.com" }, # Martin Vigoureux is Liaison Contact in bess
+#   { "role_name": "liaison_contact", "person_id": 109666, "group_id": 66, "email": "g.white@cablelabs.com" }, # Greg White is Liaison Contact in cablelabs
+#   { "role_name": "liaison_contact", "person_id": 117421, "group_id": 1933, "email": "chairman@dmtf.org" }, # Winston Bumpus is Liaison Contact in dmtf
+#   { "role_name": "liaison_contact", "person_id": 127961, "group_id": 1739, "email": "statements@ietf.org" }, # statements@ietf.org is Liaison Contact in drinks
+#   { "role_name": "liaison_contact", "person_id": 109505, "group_id": 1787, "email": "bernie@ietf.hoeneisen.ch" }, # Bernie Hoeneisen is Liaison Contact in e2md
+#   { "role_name": "liaison_contact", "person_id": 109059, "group_id": 1787, "email": "ray.bellis@nominet.org.uk" }, # Ray Bellis is Liaison Contact in e2md
+#   { "role_name": "liaison_contact", "person_id": 116529, "group_id": 1919, "email": "istvan@ecma-interational.org" }, # Istvan Sebestyen is Liaison Contact in ecma-tc39
+#   { "role_name": "liaison_contact", "person_id": 127964, "group_id": 1919, "email": "johnneumann.openstrat@gmail.com" }, # John Neuman is Liaison Contact in ecma-tc39
+#   { "role_name": "liaison_contact", "person_id": 106012, "group_id": 1643, "email": "marc.linsner@cisco.com" }, # Marc Linsner is Liaison Contact in ecrit
+#   { "role_name": "liaison_contact", "person_id": 107084, "group_id": 1643, "email": "rmarshall@telecomsys.com" }, # Roger Marshall is Liaison Contact in ecrit
+#   { "role_name": "liaison_contact", "person_id": 116363, "group_id": 1915, "email": "e2nasupport@etsi.org" }, # Sonia Compans is Liaison Contact in etsi-e2na
+#   { "role_name": "liaison_contact", "person_id": 126473, "group_id": 2261, "email": "isgsupport@etsi.org" }, # Sonia Compan is Liaison Contact in etsi-isg-sai
+#   { "role_name": "liaison_contact", "person_id": 128316, "group_id": 2301, "email": "GSMALiaisons@gsma.com" }, # David Pollington is Liaison Contact in gsma-ztc
+#   { "role_name": "liaison_contact", "person_id": 3056, "group_id": 1875, "email": "shares@ndzh.com" }, # Susan Hares is Liaison Contact in i2rs
+#   { "role_name": "liaison_contact", "person_id": 105046, "group_id": 1875, "email": "jhaas@pfrc.org" }, # Jeffrey Haas is Liaison Contact in i2rs
+#   { "role_name": "liaison_contact", "person_id": 120845, "group_id": 61, "email": "tale@dd.org" }, # David Lawrence is Liaison Contact in icann-board-of-directors
+#   { "role_name": "liaison_contact", "person_id": 112851, "group_id": 2105, "email": "pthaler@broadcom.com" }, # Patricia Thaler is Liaison Contact in ieee-802
+#   { "role_name": "liaison_contact", "person_id": 127968, "group_id": 2105, "email": "p.nikolich@ieee.org" }, # Paul Nikolich is Liaison Contact in ieee-802
+#   { "role_name": "liaison_contact", "person_id": 19651, "group_id": 63, "email": "glenn.parsons@ericsson.com" }, # Glenn Parsons is Liaison Contact in ieee-802-1
+#   { "role_name": "liaison_contact", "person_id": 123875, "group_id": 63, "email": "JMessenger@advaoptical.com" }, # John Messenger is Liaison Contact in ieee-802-1
+#   { "role_name": "liaison_contact", "person_id": 127968, "group_id": 63, "email": "p.nikolich@ieee.org" }, # Paul Nikolich is Liaison Contact in ieee-802-1
+#   { "role_name": "liaison_contact", "person_id": 117415, "group_id": 1862, "email": "Adrian.P.Stephens@intel.com" }, # Adrian Stephens is Liaison Contact in ieee-802-11
+#   { "role_name": "liaison_contact", "person_id": 106284, "group_id": 1862, "email": "dstanley@agere.com" }, # Dorothy Stanley is Liaison Contact in ieee-802-11
+#   { "role_name": "liaison_contact", "person_id": 128345, "group_id": 2302, "email": "liaison@iowngf.org" }, # Forum Iown is Liaison Contact in iown-global-forum
+#   { "role_name": "liaison_contact", "person_id": 117428, "group_id": 1939, "email": "walter.fumy@bdr.de" }, # Walter Fumy is Liaison Contact in iso-iec-jtc1-sc27
+#   { "role_name": "liaison_contact", "person_id": 117429, "group_id": 1939, "email": "krystyna.passia@din.de" }, # Krystyna Passia is Liaison Contact in iso-iec-jtc1-sc27
+#   { "role_name": "liaison_contact", "person_id": 151289, "group_id": 50, "email": "koike@itscj.ipsj.or.jp" }, # Mayumi Koike is Liaison Contact in iso-iec-jtc1-sc29
+#   { "role_name": "liaison_contact", "person_id": 151289, "group_id": 2110, "email": "koike@itscj.ipsj.or.jp" }, # Mayumi Koike is Liaison Contact in iso-iec-jtc1-sc29-wg1
+#   { "role_name": "liaison_contact", "person_id": 114435, "group_id": 74, "email": "watanabe@itscj.ipsj.or.jp" }, # Shinji Watanabe is Liaison Contact in iso-iec-jtc1-sc29-wg11
+#   { "role_name": "liaison_contact", "person_id": 112106, "group_id": 49, "email": "jooran@kisi.or.kr" }, # Jooran Lee is Liaison Contact in iso-iec-jtc1-sc6
+#   { "role_name": "liaison_contact", "person_id": 113587, "group_id": 49, "email": "dykim@cnu.kr" }, # Chungnam University is Liaison Contact in iso-iec-jtc1-sc6
+#   { "role_name": "liaison_contact", "person_id": 117427, "group_id": 1938, "email": "secretariat@jtc1-sc7.org" }, # Witold Suryn is Liaison Contact in iso-iec-jtc1-sc7
+#   { "role_name": "liaison_contact", "person_id": 117426, "group_id": 1938, "email": "chair@jtc1-sc7.org" }, # Francois Coallier is Liaison Contact in iso-iec-jtc1-sc7
+#   { "role_name": "liaison_contact", "person_id": 127971, "group_id": 68, "email": "sabine.donnardcusse@afnor.org" }, # sabine.donnardcusse@afnor.org is Liaison Contact in isotc46
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2057, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1890, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-r
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2058, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-r-wp5a
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2059, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-r-wp5d
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2060, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-r-wp8a
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2061, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-r-wp8f
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 51, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2063, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-fg-cloud
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1860, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-fg-dist
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2064, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-fg-iptv
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2065, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-fg-ngnm
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2062, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-ipv6-group
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1872, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-jca-cloud
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1874, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-jca-cop
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2066, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-jca-idm
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1927, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-jca-sdn
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 65, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-mpls
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 52, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-ngn
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2067, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-ngnmfg
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 77, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-11
+#   { "role_name": "liaison_contact", "person_id": 128236, "group_id": 77, "email": "denis.andreev@itu.int" }, # Denis ANDREEV is Liaison Contact in itu-t-sg-11
+#   { "role_name": "liaison_contact", "person_id": 107300, "group_id": 77, "email": "tatiana.kurakova@itu.int" }, # Tatiana Kurakova is Liaison Contact in itu-t-sg-11
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2074, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-11-q5
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2075, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-11-wp2
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 84, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-12
+#   { "role_name": "liaison_contact", "person_id": 102900, "group_id": 84, "email": "acmorton@att.com" }, # Al Morton is Liaison Contact in itu-t-sg-12
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2076, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-12-q12
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2077, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-12-q17
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2082, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-q11
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2078, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-q3
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2079, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-q5
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2080, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-q7
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2081, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-q9
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2083, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-wp3
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2084, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-wp4
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2085, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-wp5
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2086, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-14
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 62, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2087, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q1
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2092, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q10
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2093, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q11
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2094, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q12
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2095, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q14
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2096, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q15
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2088, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q3
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2089, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q4
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2090, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q6
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2091, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q9
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2097, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-wp1
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2098, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-wp3
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 72, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-16
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2101, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-16-q10
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1987, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-16-q3
+#   { "role_name": "liaison_contact", "person_id": 118539, "group_id": 1987, "email": "rosa.angelesleondev@itu.int" }, # Rosa De Vivero is Liaison Contact in itu-t-sg-16-q3
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2099, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-16-q8
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2100, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-16-q9
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 76, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-17
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2102, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-17-q2
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1937, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-17-q4
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1954, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-17-tsb
+#   { "role_name": "liaison_contact", "person_id": 12898, "group_id": 1954, "email": "youki-k@is.aist-nara.ac.jp" }, # Youki Kadobayashi is Liaison Contact in itu-t-sg-17-tsb
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 78, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-2
+#   { "role_name": "liaison_contact", "person_id": 127962, "group_id": 78, "email": "dr.guinena@ntra.gov.eg" }, # dr.guinena@ntra.gov.eg is Liaison Contact in itu-t-sg-2
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2103, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-20
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2073, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-2-q1
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 79, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-3
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2068, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-4
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2000, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-5
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2069, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-6
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2070, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-7
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2071, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-8
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2072, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-9
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 82, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-tsag
+#   { "role_name": "liaison_contact", "person_id": 127957, "group_id": 82, "email": "tsbtsag@itu.int" }, # Bilel Jamoussi is Liaison Contact in itu-t-tsag
+#   { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1846, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-wp-5-13
+#   { "role_name": "liaison_contact", "person_id": 10083, "group_id": 1882, "email": "paul.hoffman@vpnc.org" }, # Paul Hoffman is Liaison Contact in json
+#   { "role_name": "liaison_contact", "person_id": 111178, "group_id": 1882, "email": "mamille2@cisco.com" }, # Matthew Miller is Liaison Contact in json
+#   { "role_name": "liaison_contact", "person_id": 106881, "group_id": 1593, "email": "vach.kompella@alcatel.com" }, # Vach Kompella is Liaison Contact in l2vpn
+#   { "role_name": "liaison_contact", "person_id": 19987, "group_id": 1593, "email": "danny@arbor.net" }, # Danny McPherson is Liaison Contact in l2vpn
+#   { "role_name": "liaison_contact", "person_id": 2329, "group_id": 1593, "email": "stbryant@cisco.com" }, # Stewart Bryant is Liaison Contact in l2vpn
+#   { "role_name": "liaison_contact", "person_id": 101552, "group_id": 1593, "email": "Shane.Amante@Level3.com" }, # Shane Amante is Liaison Contact in l2vpn
+#   { "role_name": "liaison_contact", "person_id": 110305, "group_id": 1877, "email": "jason.weil@twcable.com" }, # Jason Weil is Liaison Contact in lmap
+#   { "role_name": "liaison_contact", "person_id": 6699, "group_id": 1877, "email": "dromasca@avaya.com" }, # Dan Romascanu is Liaison Contact in lmap
+#   { "role_name": "liaison_contact", "person_id": 127969, "group_id": 69, "email": "madkins@fb.com" }, # Mike Adkins is Liaison Contact in maawg
+#   { "role_name": "liaison_contact", "person_id": 127970, "group_id": 69, "email": "technical-chair@mailman.m3aawg.org" }, # technical-chair@mailman.m3aawg.org is Liaison Contact in maawg
+#   { "role_name": "liaison_contact", "person_id": 112512, "group_id": 75, "email": "rraghu@ciena.com" }, # Raghu Ranganathan is Liaison Contact in mef
+#   { "role_name": "liaison_contact", "person_id": 119947, "group_id": 1755, "email": "mrw@lilacglade.org" }, # Margaret Cullen is Liaison Contact in mif
+#   { "role_name": "liaison_contact", "person_id": 109884, "group_id": 1755, "email": "denghui02@hotmail.com" }, # Hui Deng is Liaison Contact in mif
+#   { "role_name": "liaison_contact", "person_id": 128292, "group_id": 1936, "email": "james.olthoff@nist.gov" }, # James Olthoff is Liaison Contact in nist
+#   { "role_name": "liaison_contact", "person_id": 104183, "group_id": 1537, "email": "john.loughney@nokia.com" }, # John Loughney is Liaison Contact in nsis
+#   { "role_name": "liaison_contact", "person_id": 105786, "group_id": 1840, "email": "matthew.bocci@nokia.com" }, # Matthew Bocci is Liaison Contact in nvo3
+#   { "role_name": "liaison_contact", "person_id": 112438, "group_id": 1840, "email": "bensons@queuefull.net" }, # Benson Schliesser is Liaison Contact in nvo3
+#   { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2296, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in o3gpptsgran3
+#   { "role_name": "liaison_contact", "person_id": 127966, "group_id": 1941, "email": "chet.ensign@oasis-open.org" }, # chet.ensign@oasis-open.org is Liaison Contact in oasis
+#   { "role_name": "liaison_contact", "person_id": 117423, "group_id": 1935, "email": "soley@omg.org" }, # Richard Soley is Liaison Contact in omg
+#   { "role_name": "liaison_contact", "person_id": 127963, "group_id": 1858, "email": "dan.pitt@opennetworkingfoundation.org" }, # dan.pitt@opennetworkingfoundation.org is Liaison Contact in onf
+#   { "role_name": "liaison_contact", "person_id": 108304, "group_id": 1599, "email": "gunter.van_de_velde@nokia.com" }, # Gunter Van de Velde is Liaison Contact in opsec
+#   { "role_name": "liaison_contact", "person_id": 111647, "group_id": 1599, "email": "kk@google.com" }, # Chittimaneni Kk is Liaison Contact in opsec
+#   { "role_name": "liaison_contact", "person_id": 111656, "group_id": 1599, "email": "warren@kumari.net" }, # Warren Kumari is Liaison Contact in opsec
+#   { "role_name": "liaison_contact", "person_id": 106471, "group_id": 1188, "email": "dbrungard@att.com" }, # Deborah Brungard is Liaison Contact in ospf
+#   { "role_name": "liaison_contact", "person_id": 104198, "group_id": 1188, "email": "adrian@olddog.co.uk" }, # Adrian Farrel is Liaison Contact in ospf
+#   { "role_name": "liaison_contact", "person_id": 104816, "group_id": 1188, "email": "akr@cisco.com" }, # Abhay Roy is Liaison Contact in ospf
+#   { "role_name": "liaison_contact", "person_id": 10784, "group_id": 1188, "email": "acee@redback.com" }, # Acee Lindem is Liaison Contact in ospf
+#   { "role_name": "liaison_contact", "person_id": 108123, "group_id": 1819, "email": "Gabor.Bajko@nokia.com" }, # Gabor Bajko is Liaison Contact in paws
+#   { "role_name": "liaison_contact", "person_id": 106987, "group_id": 1819, "email": "br@brianrosen.net" }, # Brian Rosen is Liaison Contact in paws
+#   { "role_name": "liaison_cc_contact", "person_id": 122823, "group_id": 1630, "email": "ketant.ietf@gmail.com" }, # Ketan Talaulikar is Liaison CC Contact in pce
+#   { "role_name": "liaison_contact", "person_id": 125031, "group_id": 1630, "email": "andrew.stone@nokia.com" }, # Andrew Stone is Liaison Contact in pce
+#   { "role_name": "liaison_contact", "person_id": 108213, "group_id": 1630, "email": "julien.meuric@orange.com" }, # Julien Meuric is Liaison Contact in pce
+#   { "role_name": "liaison_contact", "person_id": 111477, "group_id": 1630, "email": "dd@dhruvdhody.com" }, # Dhruv Dhody is Liaison Contact in pce
+#   { "role_name": "liaison_contact", "person_id": 112773, "group_id": 1701, "email": "lars.eggert@nokia.com" }, # Lars Eggert is Liaison Contact in pcn
+#   { "role_name": "liaison_contact", "person_id": 12671, "group_id": 1437, "email": "adamson@itd.nrl.navy.mil" }, # Brian Adamson is Liaison Contact in rmt
+#   { "role_name": "liaison_contact", "person_id": 100609, "group_id": 1437, "email": "lorenzo@vicisano.net" }, # Lorenzo Vicisano is Liaison Contact in rmt
+#   { "role_name": "liaison_contact", "person_id": 115213, "group_id": 1730, "email": "maria.ines.robles@ericsson.com" }, # Ines Robles is Liaison Contact in roll
+#   { "role_name": "liaison_contact", "person_id": 110721, "group_id": 1820, "email": "ted.ietf@gmail.com" }, # Ted Hardie is Liaison Contact in rtcweb
+#   { "role_name": "liaison_contact", "person_id": 104294, "group_id": 1820, "email": "magnus.westerlund@ericsson.com" }, # Magnus Westerlund is Liaison Contact in rtcweb
+#   { "role_name": "liaison_contact", "person_id": 105791, "group_id": 1820, "email": "fluffy@iii.ca" }, # Cullen Jennings is Liaison Contact in rtcweb
+#   { "role_name": "liaison_contact", "person_id": 105906, "group_id": 1910, "email": "james.n.guichard@futurewei.com" }, # Jim Guichard is Liaison Contact in sfc
+#   { "role_name": "liaison_contact", "person_id": 3862, "group_id": 1910, "email": "jmh@joelhalpern.com" }, # Joel Halpern is Liaison Contact in sfc
+#   { "role_name": "liaison_contact", "person_id": 127960, "group_id": 1462, "email": "sipcore@ietf.org" }, # sipcore@ietf.org is Liaison Contact in sip
+#   { "role_name": "liaison_contact", "person_id": 103769, "group_id": 1762, "email": "adam@nostrum.com" }, # Adam Roach is Liaison Contact in sipcore
+#   { "role_name": "liaison_contact", "person_id": 108554, "group_id": 1762, "email": "pkyzivat@alum.mit.edu" }, # Paul Kyzivat is Liaison Contact in sipcore
+#   { "role_name": "liaison_contact", "person_id": 103539, "group_id": 1542, "email": "gonzalo.camarillo@ericsson.com" }, # Gonzalo Camarillo is Liaison Contact in sipping
+#   { "role_name": "liaison_contact", "person_id": 103612, "group_id": 1542, "email": "jf.mule@cablelabs.com" }, # Jean-Francois Mule is Liaison Contact in sipping
+#   { "role_name": "liaison_contact", "person_id": 3862, "group_id": 1905, "email": "jmh@joelhalpern.com" }, # Joel Halpern is Liaison Contact in spring
+#   { "role_name": "liaison_contact", "person_id": 109802, "group_id": 1905, "email": "aretana.ietf@gmail.com" }, # Alvaro Retana is Liaison Contact in spring
+#   { "role_name": "liaison_contact", "person_id": 107172, "group_id": 1905, "email": "bruno.decraene@orange.com" }, # Bruno Decraene is Liaison Contact in spring
+#   { "role_name": "liaison_contact", "person_id": 5376, "group_id": 1899, "email": "housley@vigilsec.com" }, # Russ Housley is Liaison Contact in stir
+#   { "role_name": "liaison_contact", "person_id": 103961, "group_id": 1899, "email": "rjsparks@nostrum.com" }, # Robert Sparks is Liaison Contact in stir
+#   { "role_name": "liaison_contact", "person_id": 117430, "group_id": 1940, "email": "admin@trustedcomputinggroup.org" }, # Lindsay Adamson is Liaison Contact in tcg
+#   { "role_name": "liaison_contact", "person_id": 110932, "group_id": 1985, "email": "oscar.gonzalezdedios@telefonica.com" }, # Oscar de Dios is Liaison Contact in teas
+#   { "role_name": "liaison_contact", "person_id": 10064, "group_id": 1985, "email": "lberger@labn.net" }, # Lou Berger is Liaison Contact in teas
+#   { "role_name": "liaison_contact", "person_id": 114351, "group_id": 1985, "email": "vbeeram@juniper.net" }, # Vishnu Beeram is Liaison Contact in teas
+#   { "role_name": "liaison_contact", "person_id": 117422, "group_id": 1934, "email": "j.hietala@opengroup.org" }, # Jim Hietala is Liaison Contact in the-open-group
+#   { "role_name": "liaison_contact", "person_id": 106414, "group_id": 1709, "email": "yaakovjstein@gmail.com" }, # Yaakov Stein is Liaison Contact in tictoc
+#   { "role_name": "liaison_contact", "person_id": 4857, "group_id": 1709, "email": "kodonog@pobox.com" }, # Karen O'Donoghue is Liaison Contact in tictoc
+#   { "role_name": "liaison_contact", "person_id": 144713, "group_id": 2420, "email": "liaisons@tmforum.org" }, # liaisons@tmforum.org is Liaison Contact in tmforum
+#   { "role_name": "liaison_contact", "person_id": 112773, "group_id": 1324, "email": "lars@eggert.org" }, # Lars Eggert is Liaison Contact in tsv
+#   { "role_name": "liaison_contact", "person_id": 112104, "group_id": 53, "email": "rick@unicode.org" }, # Rick McGowan is Liaison Contact in unicode
+#   { "role_name": "liaison_contact", "person_id": 105907, "group_id": 1864, "email": "stpeter@stpeter.im" }, # Peter Saint-Andre is Liaison Contact in videocodec
+#   { "role_name": "liaison_contact", "person_id": 120261, "group_id": 54, "email": "wseltzer@w3.org" }, # Wendy Seltzer is Liaison Contact in w3c
+#   { "role_name": "liaison_contact", "person_id": 112103, "group_id": 54, "email": "plh@w3.org" }, # Philippe Le Hégaret is Liaison Contact in w3c
+#   { "role_name": "liaison_contact", "person_id": 107520, "group_id": 1957, "email": "shida@ntt-at.com" }, # Shida Schubert is Liaison Contact in webpush
+#   { "role_name": "liaison_contact", "person_id": 110049, "group_id": 1957, "email": "jhildebr@cisco.com" }, # Joe Hildebrand is Liaison Contact in webpush
+#   { "role_name": "liaison_contact", "person_id": 103769, "group_id": 1601, "email": "adam@nostrum.com" }, # Adam Roach is Liaison Contact in xcon
+#   { "role_name": "liaison_contact", "person_id": 107520, "group_id": 1815, "email": "shida@ntt-at.com" }, # Shida Schubert is Liaison Contact in xrblock
+#   { "role_name": "liaison_contact", "person_id": 6699, "group_id": 1815, "email": "dromasca@avaya.com" }, # Dan Romascanu is Liaison Contact in xrblock
+# ]}
diff --git a/ietf/group/migrations/0007_used_roles.py b/ietf/group/migrations/0007_used_roles.py
new file mode 100644
index 0000000000..0dfa79fa03
--- /dev/null
+++ b/ietf/group/migrations/0007_used_roles.py
@@ -0,0 +1,49 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    Group = apps.get_model("group", "Group")
+    GroupFeatures = apps.get_model("group", "GroupFeatures")
+    iab = Group.objects.get(acronym="iab")
+    iab.used_roles = [
+        "chair",
+        "delegate",
+        "exofficio",
+        "liaison",
+        "liaison_coordinator",
+        "member",
+    ]
+    iab.save()
+    GroupFeatures.objects.filter(type_id="ietf").update(
+        default_used_roles=[
+            "ad",
+            "member",
+            "comdir",
+            "delegate",
+            "execdir",
+            "recman",
+            "secr",
+            "chair",
+        ]
+    )
+
+
+def reverse(apps, schema_editor):
+    Group = apps.get_model("group", "Group")
+    iab = Group.objects.get(acronym="iab")
+    iab.used_roles = []
+    iab.save()
+    # Intentionally not putting trac-* back into grouptype ietf default_used_roles
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("group", "0006_remove_liason_contacts"),
+        ("name", "0018_alter_rolenames"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]
diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py
index efdd6f3ea6..e2893a90f7 100644
--- a/ietf/ietfauth/utils.py
+++ b/ietf/ietfauth/utils.py
@@ -137,6 +137,10 @@ def has_role(user, role_names, *args, **kwargs):
                 group__type="sdo",
                 group__state="active",
             ),
+            "Liaison Coordinator": Q(
+                name="liaison_coordinator",
+                group__acronym="iab",
+            ),
             "Authorized Individual": Q(
                 name="auth",
                 group__type="sdo",
diff --git a/ietf/liaisons/admin.py b/ietf/liaisons/admin.py
index c7cb7a4dae..21515ed1a3 100644
--- a/ietf/liaisons/admin.py
+++ b/ietf/liaisons/admin.py
@@ -24,7 +24,7 @@ class LiaisonStatementAdmin(admin.ModelAdmin):
     list_display = ['id', 'title', 'submitted', 'from_groups_short_display', 'purpose', 'related_to']
     list_display_links = ['id', 'title']
     ordering = ('title', )
-    raw_id_fields = ('from_contact', 'attachments', 'from_groups', 'to_groups')
+    raw_id_fields = ('attachments', 'from_groups', 'to_groups')
     #filter_horizontal = ('from_groups', 'to_groups')
     inlines = [ RelatedLiaisonStatementInline, LiaisonStatementAttachmentInline ]
 
@@ -50,4 +50,4 @@ class LiaisonStatementEventAdmin(admin.ModelAdmin):
     raw_id_fields = ["statement", "by"]
 
 admin.site.register(LiaisonStatement, LiaisonStatementAdmin)
-admin.site.register(LiaisonStatementEvent, LiaisonStatementEventAdmin)
\ No newline at end of file
+admin.site.register(LiaisonStatementEvent, LiaisonStatementEventAdmin)
diff --git a/ietf/liaisons/factories.py b/ietf/liaisons/factories.py
index 6d93cf8cd2..ca588236e3 100644
--- a/ietf/liaisons/factories.py
+++ b/ietf/liaisons/factories.py
@@ -9,7 +9,7 @@ class Meta:
         skip_postgeneration_save = True
 
     title = factory.Faker('sentence')
-    from_contact = factory.SubFactory('ietf.person.factories.EmailFactory')
+    from_contact = factory.Faker('email')
     purpose_id = 'comment'
     body = factory.Faker('paragraph')
     state_id = 'posted'
diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py
index 1af29044b3..7483981595 100644
--- a/ietf/liaisons/forms.py
+++ b/ietf/liaisons/forms.py
@@ -3,38 +3,33 @@
 
 
 import io
-import os
 import operator
-
-from typing import Union            # pyflakes:ignore
-
+import os
 from email.utils import parseaddr
+from functools import reduce
+from typing import Union, Optional  # pyflakes:ignore
 
 from django import forms
 from django.conf import settings
-from django.core.exceptions import ObjectDoesNotExist, ValidationError
-from django.forms.utils import ErrorList
-from django.db.models import Q
-#from django.forms.widgets import RadioFieldRenderer
+from django.core.exceptions import ValidationError
 from django.core.validators import validate_email
+from django.db.models import Q, QuerySet
+from django.forms.utils import ErrorList
 from django_stubs_ext import QuerySetAny
 
-import debug                            # pyflakes:ignore
-
+from ietf.doc.models import Document
+from ietf.group.models import Group
 from ietf.ietfauth.utils import has_role
-from ietf.name.models import DocRelationshipName
-from ietf.liaisons.utils import get_person_for_user,is_authorized_individual
-from ietf.liaisons.widgets import ButtonWidget,ShowAttachmentsWidget
-from ietf.liaisons.models import (LiaisonStatement,
-    LiaisonStatementEvent,LiaisonStatementAttachment,LiaisonStatementPurposeName)
 from ietf.liaisons.fields import SearchableLiaisonStatementsField
-from ietf.group.models import Group
-from ietf.person.models import Email
-from ietf.person.fields import SearchableEmailField
-from ietf.doc.models import Document
+from ietf.liaisons.models import (LiaisonStatement,
+                                  LiaisonStatementEvent, LiaisonStatementAttachment, LiaisonStatementPurposeName)
+from ietf.liaisons.utils import get_person_for_user, is_authorized_individual, OUTGOING_LIAISON_ROLES, \
+    INCOMING_LIAISON_ROLES
+from ietf.liaisons.widgets import ButtonWidget, ShowAttachmentsWidget
+from ietf.name.models import DocRelationshipName
+from ietf.person.models import Person
 from ietf.utils.fields import DatepickerDateField, ModelMultipleChoiceField
 from ietf.utils.timezone import date_today, datetime_from_date, DEADLINE_TZINFO
-from functools import reduce
 
 '''
 NOTES:
@@ -51,45 +46,106 @@
 def liaison_manager_sdos(person):
     return Group.objects.filter(type="sdo", state="active", role__person=person, role__name="liaiman").distinct()
 
+
 def flatten_choices(choices):
-    '''Returns a flat choice list given one with option groups defined'''
+    """Returns a flat choice list given one with option groups defined
+    
+    n.b., Django allows mixing grouped options and top-level options. This helper only supports
+    the non-mixed case where every option is in an option group.
+    """
     flat = []
-    for optgroup,options in choices:
+    for optgroup, options in choices:
         flat.extend(options)
     return flat
+
+
+def choices_from_group_queryset(groups: QuerySet[Group]):
+    """Get choices list for internal IETF groups user is authorized to select
     
-def get_internal_choices(user):
-    '''Returns the set of internal IETF groups the user has permissions for, as a list
-    of choices suitable for use in a select widget.  If user == None, all active internal
-    groups are included.'''
+    Returns a grouped list of choices suitable for use with a ChoiceField. If user is None,
+    includes all groups.
+    """
+    main = []
+    areas = []
+    wgs = []
+    for g in groups.distinct().order_by("acronym"):
+        if g.acronym in ("ietf", "iesg", "iab"):
+            main.append((g.pk, f"The {g.acronym.upper()}"))
+        elif g.type_id == "area":
+            areas.append((g.pk, f"{g.acronym} - {g.name}"))
+        elif g.type_id == "wg":
+            wgs.append((g.pk, f"{g.acronym} - {g.name}"))
     choices = []
-    groups = get_groups_for_person(user.person if user else None)
-    main = [ (g.pk, 'The {}'.format(g.acronym.upper())) for g in groups.filter(acronym__in=('ietf','iesg','iab')) ]
-    areas = [ (g.pk, '{} - {}'.format(g.acronym,g.name)) for g in groups.filter(type='area') ]
-    wgs = [ (g.pk, '{} - {}'.format(g.acronym,g.name)) for g in groups.filter(type='wg') ]
-    choices.append(('Main IETF Entities', main))
-    choices.append(('IETF Areas', areas))
-    choices.append(('IETF Working Groups', wgs ))
+    if len(main) > 0:
+        choices.append(("Main IETF Entities", main))
+    if len(areas) > 0:
+        choices.append(("IETF Areas", areas))
+    if len(wgs) > 0:
+        choices.append(("IETF Working Groups", wgs))
     return choices
 
-def get_groups_for_person(person):
-    '''Returns queryset of internal Groups the person has interesting roles in.
-    This is a refactor of IETFHierarchyManager.get_entities_for_person().  If Person
-    is None or Secretariat or Liaison Manager all internal IETF groups are returned.
-    '''
-    if person == None or has_role(person.user, "Secretariat") or has_role(person.user, "Liaison Manager"):
-        # collect all internal IETF groups
-        queries = [Q(acronym__in=('ietf','iesg','iab')),
-                   Q(type='area',state='active'),
-                   Q(type='wg',state='active')]
+
+def all_internal_groups():
+    """Get a queryset of all IETF groups suitable for LS To/From assignment"""
+    return Group.objects.filter(
+        Q(acronym__in=("ietf", "iesg", "iab"))
+        | Q(type="area", state="active")
+        | Q(type="wg", state="active")
+    ).distinct()
+
+
+def internal_groups_for_person(person: Optional[Person]):
+    """Get a queryset of IETF groups suitable for LS To/From assignment by person"""
+    if person is None:
+        return Group.objects.none()  # no person = no roles
+
+    if has_role(
+        person.user,
+        (
+            "Secretariat",
+            "IETF Chair",
+            "IAB Chair",
+            "IAB Executive Director",
+            "Liaison Manager",
+            "Liaison Coordinator",
+            "Authorized Individual",
+        ),
+    ):
+        return all_internal_groups()
+    # Interesting roles, as Group queries
+    queries = [
+        Q(role__person=person, role__name="chair", acronym="ietf"),
+        Q(role__person=person, role__name__in=("chair", "execdir"), acronym="iab"),
+        Q(role__person=person, role__name="ad", type="area", state="active"),
+        Q(
+            role__person=person,
+            role__name__in=("chair", "secretary"),
+            type="wg",
+            state="active",
+        ),
+        Q(
+            parent__role__person=person,
+            parent__role__name="ad",
+            type="wg",
+            state="active",
+        ),
+    ]
+    if has_role(person.user, "Area Director"):
+        queries.append(Q(acronym__in=("ietf", "iesg")))  # AD can also choose these
+    return Group.objects.filter(reduce(operator.or_, queries)).distinct()
+
+
+def external_groups_for_person(person):
+    """Get a queryset of external groups suitable for LS To/From assignment by person"""
+    filter_expr = Q(pk__in=[])  # start with no groups
+    # These roles can add all external sdo groups
+    if has_role(person.user, set(INCOMING_LIAISON_ROLES + OUTGOING_LIAISON_ROLES) - {"Liaison Manager", "Authorized Individual"}):
+        filter_expr |= Q(type="sdo")
     else:
-        # Interesting roles, as Group queries
-        queries = [Q(role__person=person,role__name='chair',acronym='ietf'),
-                   Q(role__person=person,role__name__in=('chair','execdir'),acronym='iab'),
-                   Q(role__person=person,role__name='ad',type='area',state='active'),
-                   Q(role__person=person,role__name__in=('chair','secretary'),type='wg',state='active'),
-                   Q(parent__role__person=person,parent__role__name='ad',type='wg',state='active')]
-    return Group.objects.filter(reduce(operator.or_,queries)).order_by('acronym').distinct()
+        # The person cannot add all external sdo groups; add any for which they are Liaison Manager
+        filter_expr |= Q(type="sdo", role__person=person, role__name__in=["auth", "liaiman"])
+    return Group.objects.filter(state="active").filter(filter_expr).distinct().order_by("name")
+
 
 def liaison_form_factory(request, type=None, **kwargs):
     """Returns appropriate Liaison entry form"""
@@ -154,7 +210,7 @@ def get_results(self):
             query = self.cleaned_data.get('text')
             if query:
                 q = (Q(title__icontains=query) |
-                    Q(from_contact__address__icontains=query) |
+                    Q(from_contact__icontains=query) |
                     Q(to_contacts__icontains=query) |
                     Q(other_identifiers__icontains=query) |
                     Q(body__icontains=query) |
@@ -216,13 +272,8 @@ class LiaisonModelForm(forms.ModelForm):
     '''Specify fields which require a custom widget or that are not part of the model.
     '''
     from_groups = ModelMultipleChoiceField(queryset=Group.objects.all(),label='Groups',required=False)
-    from_groups.widget.attrs["class"] = "select2-field"
-    from_groups.widget.attrs['data-minimum-input-length'] = 0
-    from_contact = forms.EmailField()   # type: Union[forms.EmailField, SearchableEmailField]
     to_contacts = forms.CharField(label="Contacts", widget=forms.Textarea(attrs={'rows':'3', }), strip=False)
     to_groups = ModelMultipleChoiceField(queryset=Group.objects,label='Groups',required=False)
-    to_groups.widget.attrs["class"] = "select2-field"
-    to_groups.widget.attrs['data-minimum-input-length'] = 0
     deadline = DatepickerDateField(date_format="yyyy-mm-dd", picker_settings={"autoclose": "1" }, label='Deadline', required=True)
     related_to = SearchableLiaisonStatementsField(label='Related Liaison Statement', required=False)
     submitted_date = DatepickerDateField(date_format="yyyy-mm-dd", picker_settings={"autoclose": "1" }, label='Submission date', required=True, initial=lambda: date_today(DEADLINE_TZINFO))
@@ -245,13 +296,17 @@ def __init__(self, user, *args, **kwargs):
         self.person = get_person_for_user(user)
         self.is_new = not self.instance.pk
 
+        self.fields["from_groups"].widget.attrs["class"] = "select2-field"
+        self.fields["from_groups"].widget.attrs["data-minimum-input-length"] = 0
         self.fields["from_groups"].widget.attrs["data-placeholder"] = "Type in name to search for group"
+        self.fields["to_groups"].widget.attrs["class"] = "select2-field"
+        self.fields["to_groups"].widget.attrs["data-minimum-input-length"] = 0
         self.fields["to_groups"].widget.attrs["data-placeholder"] = "Type in name to search for group"
         self.fields["to_contacts"].label = 'Contacts'
         self.fields["other_identifiers"].widget.attrs["rows"] = 2
-        
+
         # add email validators
-        for field in ['from_contact','to_contacts','technical_contacts','action_holder_contacts','cc_contacts']:
+        for field in ['to_contacts','technical_contacts','action_holder_contacts','cc_contacts']:
             if field in self.fields:
                 self.fields[field].validators.append(validate_emails)
 
@@ -270,18 +325,6 @@ def clean_to_groups(self):
             raise forms.ValidationError('You must specify a To Group')
         return to_groups
         
-    def clean_from_contact(self):
-        contact = self.cleaned_data.get('from_contact')
-        from_groups = self.cleaned_data.get('from_groups')
-        try:
-            email = Email.objects.get(address=contact)
-            if not email.origin:
-                email.origin = "liaison: %s" % (','.join([ g.acronym for g in from_groups.all() ]))
-                email.save()
-        except ObjectDoesNotExist:
-            raise forms.ValidationError('Email address does not exist')
-        return email
-
 # Note to future person: This is the wrong place to fix the new lines
 # in cc_contacts and to_contacts. Those belong in the save function.
 # Or at least somewhere other than here.
@@ -434,32 +477,39 @@ def is_approved(self):
         return True
 
     def get_post_only(self):
-        from_groups = self.cleaned_data.get('from_groups')
-        if has_role(self.user, "Secretariat") or is_authorized_individual(self.user,from_groups):
+        from_groups = self.cleaned_data.get("from_groups")
+        if (
+            has_role(self.user, "Secretariat")
+            or has_role(self.user, "Liaison Coordinator")
+            or is_authorized_individual(self.user, from_groups)
+        ):
             return False
         return True
 
     def set_from_fields(self):
-        '''Set from_groups and from_contact options and initial value based on user
-        accessing the form.'''
-        if has_role(self.user, "Secretariat"):
-            queryset = Group.objects.filter(type="sdo", state="active").order_by('name')
-        else:
-            queryset = Group.objects.filter(type="sdo", state="active", role__person=self.person, role__name__in=("liaiman", "auth")).distinct().order_by('name')
-            self.fields['from_contact'].initial = self.person.role_set.filter(group=queryset[0]).first().email.address
-            self.fields['from_contact'].widget.attrs['disabled'] = True
-        self.fields['from_groups'].queryset = queryset
-        self.fields['from_groups'].widget.submitter = str(self.person)
-
+        """Configure from "From" fields based on user roles"""
+        qs = external_groups_for_person(self.person)
+        self.fields["from_groups"].queryset = qs
+        self.fields["from_groups"].widget.submitter = str(self.person)
         # if there's only one possibility make it the default
-        if len(queryset) == 1:
-            self.fields['from_groups'].initial = queryset
+        if len(qs) == 1:
+            self.fields['from_groups'].initial = qs
+
+        # Note that the IAB chair currently doesn't get to work with incoming liaison statements
+        if not (
+            has_role(self.user, "Secretariat")
+            or has_role(self.user, "Liaison Coordinator")
+        ):
+            self.fields["from_contact"].initial = (
+                self.person.role_set.filter(group=qs[0]).first().email.formatted_email()
+            )
+            self.fields["from_contact"].widget.attrs["disabled"] = True
 
     def set_to_fields(self):
         '''Set to_groups and to_contacts options and initial value based on user
         accessing the form.  For incoming Liaisons, to_groups choices is the full set.
         '''
-        self.fields['to_groups'].choices = get_internal_choices(None)
+        self.fields['to_groups'].choices = choices_from_group_queryset(all_internal_groups())
 
 
 class OutgoingLiaisonForm(LiaisonModelForm):
@@ -473,46 +523,56 @@ def is_approved(self):
         return self.cleaned_data['approved']
 
     def set_from_fields(self):
-        '''Set from_groups and from_contact options and initial value based on user
-        accessing the form'''
-        choices = get_internal_choices(self.user)
-        self.fields['from_groups'].choices = choices
-        
-        # set initial value if only one entry 
-        flat_choices = flatten_choices(choices)
+        """Configure from "From" fields based on user roles"""
+        self.set_from_groups_field()
+        self.set_from_contact_field()
+
+    def set_from_groups_field(self):
+        """Configure the from_groups field based on roles"""
+        grouped_choices = choices_from_group_queryset(internal_groups_for_person(self.person))
+        flat_choices = flatten_choices(grouped_choices)
         if len(flat_choices) == 1:
-            self.fields['from_groups'].initial = [flat_choices[0][0]]
-        
-        if has_role(self.user, "Secretariat"):
-            self.fields['from_contact'] = SearchableEmailField(only_users=True)  # secretariat can edit this field!
-            return
-
-        if self.person.role_set.filter(name='liaiman',group__state='active'):
-            email = self.person.role_set.filter(name='liaiman',group__state='active').first().email.address
-        elif self.person.role_set.filter(name__in=('ad','chair'),group__state='active'):
-            email = self.person.role_set.filter(name__in=('ad','chair'),group__state='active').first().email.address
+            self.fields["from_groups"].choices = flat_choices
+            self.fields["from_groups"].initial = [flat_choices[0][0]]
         else:
-            email = self.person.email_address()
+            self.fields["from_groups"].choices = grouped_choices
 
-        # Non-secretariat user cannot change the from_contact field. Fill in its value.
+    def set_from_contact_field(self):
+        """Configure the from_contact field based on user roles"""
+        # Secretariat can set this to any valid address but gets no default
+        if has_role(self.user, "Secretariat"):
+            return
+        elif has_role(self.user, ["IAB Chair", "Liaison Coordinator"]):
+            self.fields["from_contact"].initial = "IAB Chair <iab-chair@iab.org>"
+            return
+        elif has_role(self.user, "IETF Chair"):
+            self.fields["from_contact"].initial = "IETF Chair <chair@ietf.org>"
+            return
+        # ... others have it set to the correct value and cannot change it
         self.fields['from_contact'].disabled = True
-        self.fields['from_contact'].initial = email
-
-    def set_to_fields(self):
-        '''Set to_groups and to_contacts options and initial value based on user
-        accessing the form'''
-        # set options. if the user is a Liaison Manager and nothing more, reduce set to his SDOs
-        if has_role(self.user, "Liaison Manager") and not self.person.role_set.filter(name__in=('ad','chair'),group__state='active'):
-            queryset = Group.objects.filter(type="sdo", state="active", role__person=self.person, role__name="liaiman").distinct().order_by('name')
+        # Set up the querysets we might use - only evaluated as needed
+        liaison_manager_role = self.person.role_set.filter(name="liaiman", group__state="active")
+        chair_or_ad_role = self.person.role_set.filter(
+                    name__in=("ad", "chair"), group__state="active"
+                )  
+        if liaison_manager_role.exists():
+            from_contact_email = liaison_manager_role.first().email
+        elif chair_or_ad_role.exists():
+            from_contact_email = chair_or_ad_role.first().email
         else:
-            # get all outgoing entities
-            queryset = Group.objects.filter(type="sdo", state="active").order_by('name')
+            from_contact_email = self.person.email()
+        self.fields['from_contact'].initial = from_contact_email.formatted_email()
 
-        self.fields['to_groups'].queryset = queryset
+    def set_to_fields(self):
+        """Configure the "To" fields based on user roles"""
+        qs = external_groups_for_person(self.person)
+        self.fields['to_groups'].queryset = qs 
 
         # set initial
         if has_role(self.user, "Liaison Manager"):
-            self.fields['to_groups'].initial = [queryset.first()]
+            self.fields['to_groups'].initial = [
+                qs.filter(role__person=self.person, role__name="liaiman").first()
+            ]
 
 
 class EditLiaisonForm(LiaisonModelForm):
@@ -533,32 +593,20 @@ def save(self, *args, **kwargs):
         return self.instance
 
     def set_from_fields(self):
-        '''Set from_groups and from_contact options and initial value based on user
-        accessing the form.'''
+        """Configure from "From" fields based on user roles"""
         if self.instance.is_outgoing():
-            self.fields['from_groups'].choices = get_internal_choices(self.user)
+            self.fields['from_groups'].choices = choices_from_group_queryset(internal_groups_for_person(self.person))
         else:
-            if has_role(self.user, "Secretariat"):
-                queryset = Group.objects.filter(type="sdo").order_by('name')
-            else:
-                queryset = Group.objects.filter(type="sdo", role__person=self.person, role__name__in=("liaiman", "auth")).distinct().order_by('name')
+            self.fields["from_groups"].queryset = external_groups_for_person(self.person)
+            if not has_role(self.user, "Secretariat"):
                 self.fields['from_contact'].widget.attrs['disabled'] = True
-            self.fields['from_groups'].queryset = queryset
 
     def set_to_fields(self):
-        '''Set to_groups and to_contacts options and initial value based on user
-        accessing the form.  For incoming Liaisons, to_groups choices is the full set.
-        '''
+        """Configure the "To" fields based on user roles"""
         if self.instance.is_outgoing():
-            # if the user is a Liaison Manager and nothing more, reduce to set to his SDOs
-            if has_role(self.user, "Liaison Manager") and not self.person.role_set.filter(name__in=('ad','chair'),group__state='active'):
-                queryset = Group.objects.filter(type="sdo", role__person=self.person, role__name="liaiman").distinct().order_by('name')
-            else:
-                # get all outgoing entities
-                queryset = Group.objects.filter(type="sdo").order_by('name')
-            self.fields['to_groups'].queryset = queryset
+            self.fields['to_groups'].queryset = external_groups_for_person(self.person)
         else:
-            self.fields['to_groups'].choices = get_internal_choices(None)
+            self.fields['to_groups'].choices = choices_from_group_queryset(all_internal_groups())
 
 
 class EditAttachmentForm(forms.Form):
diff --git a/ietf/liaisons/migrations/0003_liaisonstatement_from_contact_tmp.py b/ietf/liaisons/migrations/0003_liaisonstatement_from_contact_tmp.py
new file mode 100644
index 0000000000..de2ce7ff59
--- /dev/null
+++ b/ietf/liaisons/migrations/0003_liaisonstatement_from_contact_tmp.py
@@ -0,0 +1,22 @@
+# Copyright The IETF Trust 2025 All Rights Reserved
+from django.db import migrations, models
+import ietf.utils.validators
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("liaisons", "0002_alter_liaisonstatement_response_contacts"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="liaisonstatement",
+            name="from_contact_tmp",
+            field=models.CharField(
+                blank=True,
+                help_text="Address of the formal sender of the statement",
+                max_length=512,
+                validators=[ietf.utils.validators.validate_mailbox_address],
+            ),
+        ),
+    ]
diff --git a/ietf/liaisons/migrations/0004_populate_liaisonstatement_from_contact_tmp.py b/ietf/liaisons/migrations/0004_populate_liaisonstatement_from_contact_tmp.py
new file mode 100644
index 0000000000..dbab326b0c
--- /dev/null
+++ b/ietf/liaisons/migrations/0004_populate_liaisonstatement_from_contact_tmp.py
@@ -0,0 +1,60 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from itertools import islice
+
+from django.db import migrations
+
+from ietf.person.name import plain_name
+from ietf.utils.mail import formataddr
+from ietf.utils.validators import validate_mailbox_address
+
+
+def forward(apps, schema_editor):
+    def _formatted_email(email):
+        """Format an email address to match Email.formatted_email()"""
+        person = email.person
+        if person:
+            return formataddr(
+                (
+                    # inlined Person.plain_name(), minus the caching
+                    person.plain if person.plain else plain_name(person.name),
+                    email.address,
+                )
+            )
+        return email.address
+
+    def _batched(iterable, n):
+        """Split an iterable into lists of length <= n
+
+        (based on itertools example code for batched(), which is added in py312)
+        """
+        iterator = iter(iterable)
+        batch = list(islice(iterator, n))  # consumes first n iterations
+        while batch:
+            yield batch
+            batch = list(islice(iterator, n))  # consumes next n iterations
+
+    LiaisonStatement = apps.get_model("liaisons", "LiaisonStatement")
+    LiaisonStatement.objects.update(from_contact_tmp="")  # ensure they're all blank
+    for batch in _batched(
+        LiaisonStatement.objects.exclude(from_contact=None).select_related(
+            "from_contact"
+        ),
+        100,
+    ):
+        for ls in batch:
+            ls.from_contact_tmp = _formatted_email(ls.from_contact)
+            validate_mailbox_address(
+                ls.from_contact_tmp
+            )  # be sure it's permitted before we accept it
+
+        LiaisonStatement.objects.bulk_update(batch, fields=["from_contact_tmp"])
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("liaisons", "0003_liaisonstatement_from_contact_tmp"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward),
+    ]
diff --git a/ietf/liaisons/migrations/0005_replace_liaisonstatement_from_contact.py b/ietf/liaisons/migrations/0005_replace_liaisonstatement_from_contact.py
new file mode 100644
index 0000000000..e1702ae3bc
--- /dev/null
+++ b/ietf/liaisons/migrations/0005_replace_liaisonstatement_from_contact.py
@@ -0,0 +1,20 @@
+# Copyright The IETF Trust 2025 All Rights Reserved
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("liaisons", "0004_populate_liaisonstatement_from_contact_tmp"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="liaisonstatement",
+            name="from_contact",
+        ),
+        migrations.RenameField(
+            model_name="liaisonstatement",
+            old_name="from_contact_tmp",
+            new_name="from_contact",
+        ),
+    ]
diff --git a/ietf/liaisons/models.py b/ietf/liaisons/models.py
index 2ad502102c..a2d79ea476 100644
--- a/ietf/liaisons/models.py
+++ b/ietf/liaisons/models.py
@@ -7,13 +7,14 @@
 from django.db import models
 from django.utils.text import slugify
 
-from ietf.person.models import Email, Person
+from ietf.person.models import Person
 from ietf.name.models import (LiaisonStatementPurposeName, LiaisonStatementState,
                               LiaisonStatementEventTypeName, LiaisonStatementTagName,
                               DocRelationshipName)
 from ietf.doc.models import Document
 from ietf.group.models import Group
 from ietf.utils.models import ForeignKey
+from ietf.utils.validators import validate_mailbox_address
 
 # maps (previous state id, new state id) to event type id
 STATE_EVENT_MAPPING = {
@@ -29,7 +30,12 @@
 class LiaisonStatement(models.Model):
     title = models.CharField(max_length=255)
     from_groups = models.ManyToManyField(Group, blank=True, related_name='liaisonstatement_from_set')
-    from_contact = ForeignKey(Email, blank=True, null=True)
+    from_contact = models.CharField(
+        blank=True,
+        max_length=512,
+        help_text="Address of the formal sender of the statement",
+        validators=(validate_mailbox_address,)
+    )
     to_groups = models.ManyToManyField(Group, blank=True, related_name='liaisonstatement_to_set')
     to_contacts = models.CharField(max_length=2000, help_text="Contacts at recipient group")
 
@@ -85,7 +91,7 @@ def name(self):
         if self.from_groups.count():
             frm = ', '.join([i.acronym or i.name for i in self.from_groups.all()])
         else:
-            frm = self.from_contact.person.name
+            frm = self.from_contact
         if self.to_groups.count():
             to = ', '.join([i.acronym or i.name for i in self.to_groups.all()])
         else:
diff --git a/ietf/liaisons/resources.py b/ietf/liaisons/resources.py
index 8f31ea3a64..02cd159a11 100644
--- a/ietf/liaisons/resources.py
+++ b/ietf/liaisons/resources.py
@@ -15,12 +15,10 @@
                                   RelatedLiaisonStatement)
 
 
-from ietf.person.resources import EmailResource
 from ietf.group.resources import GroupResource
 from ietf.name.resources import LiaisonStatementPurposeNameResource, LiaisonStatementTagNameResource, LiaisonStatementStateResource
 from ietf.doc.resources import DocumentResource
 class LiaisonStatementResource(ModelResource):
-    from_contact     = ToOneField(EmailResource, 'from_contact', null=True)
     purpose          = ToOneField(LiaisonStatementPurposeNameResource, 'purpose')
     state            = ToOneField(LiaisonStatementStateResource, 'state')
     from_groups      = ToManyField(GroupResource, 'from_groups', null=True)
@@ -36,6 +34,7 @@ class Meta:
         filtering = { 
             "id": ALL,
             "title": ALL,
+            "from_contact": ALL,
             "to_contacts": ALL,
             "response_contacts": ALL,
             "technical_contacts": ALL,
@@ -44,9 +43,6 @@ class Meta:
             "deadline": ALL,
             "other_identifiers": ALL,
             "body": ALL,
-            "from_name": ALL,
-            "to_name": ALL,
-            "from_contact": ALL_WITH_RELATIONS,
             "purpose": ALL_WITH_RELATIONS,
             "state": ALL_WITH_RELATIONS,
             "from_groups": ALL_WITH_RELATIONS,
diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py
index 1742687f14..1d6cfe0c14 100644
--- a/ietf/liaisons/tests.py
+++ b/ietf/liaisons/tests.py
@@ -462,11 +462,12 @@ def test_edit_liaison(self):
 
 
     def test_incoming_access(self):
-        '''Ensure only Secretariat, Liaison Managers, and Authorized Individuals
+        '''Ensure only Secretariat, Liaison Managers, Liaison Coordinators, and Authorized Individuals
         have access to incoming liaisons.
         '''
         sdo = RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman').group
         RoleFactory(name_id='auth',group=sdo,person__user__username='ulm-auth')
+        RoleFactory(name_id='liaison_coordinator', group__acronym='iab', person__user__username='liaison-coordinator')
         stmt = LiaisonStatementFactory(from_groups=[sdo,])
         LiaisonStatementEventFactory(statement=stmt,type_id='posted')
         RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars')
@@ -499,6 +500,15 @@ def test_incoming_access(self):
         r = self.client.get(addurl)
         self.assertEqual(r.status_code, 200)
 
+        # Liaison Coordinator has access
+        self.client.login(username="liaison-coordinator", password="liaison-coordinator+password")
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q('a.btn:contains("New incoming liaison")')), 1)
+        r = self.client.get(addurl)
+        self.assertEqual(r.status_code, 200)
+
         # Authorized Individual has access
         self.client.login(username="ulm-auth", password="ulm-auth+password")
         r = self.client.get(url)
@@ -521,6 +531,7 @@ def test_outgoing_access(self):
 
         sdo = RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman').group
         RoleFactory(name_id='auth',group=sdo,person__user__username='ulm-auth')
+        RoleFactory(name_id='liaison_coordinator', group__acronym='iab', person__user__username='liaison-coordinator')
         mars = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group
         RoleFactory(name_id='secr',group=mars,person__user__username='mars-secr')
         RoleFactory(name_id='execdir',group=Group.objects.get(acronym='iab'),person__user__username='iab-execdir')
@@ -599,6 +610,15 @@ def test_outgoing_access(self):
         r = self.client.get(addurl)
         self.assertEqual(r.status_code, 200)
 
+        # Liaison Coordinator has access
+        self.assertTrue(self.client.login(username="liaison-coordinator", password="liaison-coordinator+password"))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q('a.btn:contains("New outgoing liaison")')), 1)
+        r = self.client.get(addurl)
+        self.assertEqual(r.status_code, 200)
+
         # Authorized Individual has no access
         self.assertTrue(self.client.login(username="ulm-auth", password="ulm-auth+password"))
         r = self.client.get(url)
@@ -740,7 +760,7 @@ def test_add_incoming_liaison(self):
 
         l = LiaisonStatement.objects.all().order_by("-id")[0]
         self.assertEqual(l.from_groups.count(),2)
-        self.assertEqual(l.from_contact.address, submitter.email_address())
+        self.assertEqual(l.from_contact, submitter.email_address())
         self.assertSequenceEqual(l.to_groups.all(),[to_group])
         self.assertEqual(l.technical_contacts, "technical_contact@example.com")
         self.assertEqual(l.action_holder_contacts, "action_holder_contacts@example.com")
@@ -825,7 +845,7 @@ def test_add_outgoing_liaison(self):
 
         l = LiaisonStatement.objects.all().order_by("-id")[0]
         self.assertSequenceEqual(l.from_groups.all(), [from_group])
-        self.assertEqual(l.from_contact.address, submitter.email_address())
+        self.assertEqual(l.from_contact, submitter.email_address())
         self.assertSequenceEqual(l.to_groups.all(), [to_group])
         self.assertEqual(l.to_contacts, "to_contacts@example.com")
         self.assertEqual(l.technical_contacts, "technical_contact@example.com")
@@ -901,7 +921,7 @@ def test_liaison_add_attachment(self):
         file.name = "upload.txt"
         post_data = dict(
             from_groups = ','.join([ str(x.pk) for x in liaison.from_groups.all() ]),
-            from_contact = liaison.from_contact.address,
+            from_contact = liaison.from_contact,
             to_groups = ','.join([ str(x.pk) for x in liaison.to_groups.all() ]),
             to_contacts = 'to_contacts@example.com',
             purpose = liaison.purpose.slug,
diff --git a/ietf/liaisons/tests_forms.py b/ietf/liaisons/tests_forms.py
new file mode 100644
index 0000000000..c2afddea65
--- /dev/null
+++ b/ietf/liaisons/tests_forms.py
@@ -0,0 +1,229 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from ietf.group.factories import GroupFactory, RoleFactory
+from ietf.group.models import Group
+from ietf.liaisons.forms import (
+    flatten_choices,
+    choices_from_group_queryset,
+    all_internal_groups,
+    internal_groups_for_person,
+    external_groups_for_person,
+)
+from ietf.person.factories import PersonFactory
+from ietf.person.models import Person
+from ietf.utils.test_utils import TestCase
+
+
+class HelperTests(TestCase):
+    @staticmethod
+    def _alphabetically_by_acronym(group_list):
+        return sorted(group_list, key=lambda item: item.acronym)
+
+    def test_choices_from_group_queryset(self):
+        main_groups = list(Group.objects.filter(acronym__in=["ietf", "iab"]))
+        areas = GroupFactory.create_batch(2, type_id="area")
+        wgs = GroupFactory.create_batch(2)
+
+        # No groups
+        self.assertEqual(
+            choices_from_group_queryset(Group.objects.none()),
+            [],
+        )
+
+        # Main groups only
+        choices = choices_from_group_queryset(
+            Group.objects.filter(pk__in=[g.pk for g in main_groups])
+        )
+        self.assertEqual(len(choices), 1, "show one optgroup, hide empty ones")
+        self.assertEqual(choices[0][0], "Main IETF Entities")
+        self.assertEqual(
+            [val for val, _ in choices[0][1]],  # extract the choice value
+            [g.pk for g in self._alphabetically_by_acronym(main_groups)],
+        )
+
+        # Area groups only
+        choices = choices_from_group_queryset(
+            Group.objects.filter(pk__in=[g.pk for g in areas])
+        )
+        self.assertEqual(len(choices), 1, "show one optgroup, hide empty ones")
+        self.assertEqual(choices[0][0], "IETF Areas")
+        self.assertEqual(
+            [val for val, _ in choices[0][1]],  # extract the choice value
+            [g.pk for g in self._alphabetically_by_acronym(areas)],
+        )
+
+        # WGs only
+        choices = choices_from_group_queryset(
+            Group.objects.filter(pk__in=[g.pk for g in wgs])
+        )
+        self.assertEqual(len(choices), 1, "show one optgroup, hide empty ones")
+        self.assertEqual(choices[0][0], "IETF Working Groups")
+        self.assertEqual(
+            [val for val, _ in choices[0][1]],  # extract the choice value
+            [g.pk for g in self._alphabetically_by_acronym(wgs)],
+        )
+
+        # All together
+        choices = choices_from_group_queryset(
+            Group.objects.filter(pk__in=[g.pk for g in main_groups + areas + wgs])
+        )
+        self.assertEqual(len(choices), 3, "show all three optgroups")
+        self.assertEqual(
+            [optgroup_label for optgroup_label, _ in choices],
+            ["Main IETF Entities", "IETF Areas", "IETF Working Groups"],
+        )
+        self.assertEqual(
+            [val for val, _ in choices[0][1]],  # extract the choice value
+            [g.pk for g in self._alphabetically_by_acronym(main_groups)],
+        )
+        self.assertEqual(
+            [val for val, _ in choices[1][1]],  # extract the choice value
+            [g.pk for g in self._alphabetically_by_acronym(areas)],
+        )
+        self.assertEqual(
+            [val for val, _ in choices[2][1]],  # extract the choice value
+            [g.pk for g in self._alphabetically_by_acronym(wgs)],
+        )
+
+    def test_all_internal_groups(self):
+        # test relies on the data created in ietf.utils.test_data.make_immutable_test_data()
+        self.assertCountEqual(
+            all_internal_groups().values_list("acronym", flat=True),
+            {"ietf", "iab", "iesg", "farfut", "ops", "sops"},
+        )
+
+    def test_internal_groups_for_person(self):
+        # test relies on the data created in ietf.utils.test_data.make_immutable_test_data()
+        # todo add liaison coordinator when modeled
+        RoleFactory(
+            name_id="execdir",
+            group=Group.objects.get(acronym="iab"),
+            person__user__username="iab-execdir",
+        )
+        RoleFactory(
+            name_id="auth",
+            group__type_id="sdo",
+            group__acronym="sdo",
+            person__user__username="sdo-authperson",
+        )
+
+        self.assertQuerysetEqual(
+            internal_groups_for_person(None),
+            Group.objects.none(),
+            msg="no Person means no groups",
+        )
+        self.assertQuerysetEqual(
+            internal_groups_for_person(PersonFactory()),
+            Group.objects.none(),
+            msg="no Role means no groups",
+        )
+
+        for username in (
+            "secretary",
+            "ietf-chair",
+            "iab-chair",
+            "iab-execdir",
+            "sdo-authperson",
+        ):
+            returned_queryset = internal_groups_for_person(
+                Person.objects.get(user__username=username)
+            )
+            self.assertCountEqual(
+                returned_queryset.values_list("acronym", flat=True),
+                {"ietf", "iab", "iesg", "farfut", "ops", "sops"},
+                f"{username} should get all groups",
+            )
+
+        # "ops-ad" user is the AD of the "ops" area, which contains the "sops" wg
+        self.assertCountEqual(
+            internal_groups_for_person(
+                Person.objects.get(user__username="ops-ad")
+            ).values_list("acronym", flat=True),
+            {"ietf", "iesg", "ops", "sops"},
+            "area director should get only their area, its wgs, and the ietf/iesg groups",
+        )
+
+        self.assertCountEqual(
+            internal_groups_for_person(
+                Person.objects.get(user__username="sopschairman"),
+            ).values_list("acronym", flat=True),
+            {"sops"},
+            "wg chair should get only their wg",
+        )
+
+    def test_external_groups_for_person(self):
+        RoleFactory(
+            name_id="execdir",
+            group=Group.objects.get(acronym="iab"),
+            person__user__username="iab-execdir",
+        )
+        RoleFactory(name_id="liaison_coordinator", group__acronym="iab", person__user__username="liaison-coordinator")
+        the_sdo = GroupFactory(type_id="sdo", acronym="the-sdo")
+        liaison_manager = RoleFactory(name_id="liaiman", group=the_sdo).person
+        authperson = RoleFactory(name_id="auth", group=the_sdo).person
+
+        GroupFactory(acronym="other-sdo", type_id="sdo")
+        for username in (
+            "secretary",
+            "ietf-chair",
+            "iab-chair",
+            "iab-execdir",
+            "liaison-coordinator",
+            "ad",
+            "sopschairman",
+            "sopssecretary",
+        ):
+            person = Person.objects.get(user__username=username)
+            self.assertCountEqual(
+                external_groups_for_person(
+                    person,
+                ).values_list("acronym", flat=True),
+                {"the-sdo", "other-sdo"},
+                f"{username} should get all SDO groups",
+            )
+            tmp_role = RoleFactory(name_id="chair", group__type_id="wg", person=person)
+            self.assertCountEqual(
+                external_groups_for_person(
+                    person,
+                ).values_list("acronym", flat=True),
+                {"the-sdo", "other-sdo"},
+                f"{username} should still get all SDO groups when they also a liaison manager",
+            )
+            tmp_role.delete()
+
+        self.assertCountEqual(
+            external_groups_for_person(liaison_manager).values_list(
+                "acronym", flat=True
+            ),
+            {"the-sdo"},
+            "liaison manager should get only their SDO group",
+        )
+        self.assertCountEqual(
+            external_groups_for_person(authperson).values_list("acronym", flat=True),
+            {"the-sdo"},
+            "authorized individual should get only their SDO group",
+        )
+
+    def test_flatten_choices(self):
+        self.assertEqual(flatten_choices([]), [])
+        self.assertEqual(
+            flatten_choices(
+                (
+                    ("group A", ()),
+                    ("group B", (("val0", "label0"), ("val1", "label1"))),
+                    ("group C", (("val2", "label2"),)),
+                )
+            ),
+            [("val0", "label0"), ("val1", "label1"), ("val2", "label2")],
+        )
+
+
+class IncomingLiaisonFormTests(TestCase):
+    pass
+
+
+class OutgoingLiaisonFormTests(TestCase):
+    pass
+
+
+class EditLiaisonFormTests(TestCase):
+    pass
diff --git a/ietf/liaisons/utils.py b/ietf/liaisons/utils.py
index df48831917..ea06c5988e 100644
--- a/ietf/liaisons/utils.py
+++ b/ietf/liaisons/utils.py
@@ -4,6 +4,22 @@
 from ietf.liaisons.models import LiaisonStatement
 from ietf.ietfauth.utils import has_role, passes_test_decorator
 
+# Roles allowed to create and manage outgoing liaison statements.
+OUTGOING_LIAISON_ROLES = [
+    "Area Director",
+    "IAB Chair",
+    "IAB Executive Director",
+    "IETF Chair",
+    "Liaison Manager",
+    "Liaison Coordinator",
+    "Secretariat",
+    "WG Chair",
+    "WG Secretary",
+]
+
+# Roles allowed to create and manage incoming liaison statements.
+INCOMING_LIAISON_ROLES = ["Authorized Individual", "Liaison Manager", "Liaison Coordinator", "Secretariat"]
+
 can_submit_liaison_required = passes_test_decorator(
     lambda u, *args, **kwargs: can_add_liaison(u),
     "Restricted to participants who are authorized to submit liaison statements on behalf of the various IETF entities")
@@ -30,13 +46,13 @@ def can_edit_liaison(user, liaison):
     '''Returns True if user has edit / approval authority.
     
     True if:
-    - user is Secretariat
+    - user is Secretariat or Liaison Coordinator
     - liaison is outgoing and user has approval authority
     - user is liaison manager of all SDOs involved
     '''
     if not user.is_authenticated:
         return False
-    if has_role(user, "Secretariat"):
+    if has_role(user, "Secretariat") or has_role(user, "Liaison Coordinator"):
         return True
 
     if liaison.is_outgoing() and liaison in approvable_liaison_statements(user):
@@ -59,11 +75,10 @@ def get_person_for_user(user):
         return None
 
 def can_add_outgoing_liaison(user):
-    return has_role(user, ["Area Director","WG Chair","WG Secretary","IETF Chair","IAB Chair",
-        "IAB Executive Director","Liaison Manager","Secretariat"])
+    return has_role(user, OUTGOING_LIAISON_ROLES)
 
 def can_add_incoming_liaison(user):
-    return has_role(user, ["Liaison Manager","Authorized Individual","Secretariat"])
+    return has_role(user, INCOMING_LIAISON_ROLES)
 
 def can_add_liaison(user):
     return can_add_incoming_liaison(user) or can_add_outgoing_liaison(user)
diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py
index a8e80a5194..1b7e8d63bb 100644
--- a/ietf/liaisons/views.py
+++ b/ietf/liaisons/views.py
@@ -57,7 +57,7 @@ def _can_take_care(liaison, user):
         return False
 
     if user.is_authenticated:
-        if has_role(user, "Secretariat"):
+        if has_role(user, "Secretariat") or has_role(user, "Liaison Coordinator"):
             return True
         else:
             return _find_person_in_emails(liaison, get_person_for_user(user))
@@ -196,7 +196,13 @@ def post_only(group,person):
     - Authorized Individuals have full access for the group they are associated with
     - Liaison Managers can post only
     '''
-    if group.type_id == 'sdo' and ( not(has_role(person.user,"Secretariat") or group.role_set.filter(name='auth',person=person)) ):
+    if group.type_id == "sdo" and (
+        not (
+            has_role(person.user, "Secretariat")
+            or has_role(person.user, "Liaison Coordinator")
+            or group.role_set.filter(name="auth", person=person)
+        )
+    ):
         return True
     else:
         return False
diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json
index 15ae71d849..ebdda1a1fa 100644
--- a/ietf/name/fixtures/names.json
+++ b/ietf/name/fixtures/names.json
@@ -3341,7 +3341,7 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"ad\",\n    \"member\",\n    \"comdir\",\n    \"delegate\",\n    \"execdir\",\n    \"recman\",\n    \"secr\",\n    \"trac-editor\",\n    \"trac-admin\",\n    \"chair\"\n]",
+      "default_used_roles": "[\n    \"ad\",\n    \"member\",\n    \"comdir\",\n    \"delegate\",\n    \"execdir\",\n    \"recman\",\n    \"secr\",\n    \"chair\"\n]",
       "docman_roles": "[\n    \"chair\"\n]",
       "groupman_authroles": "[\n    \"Secretariat\"\n]",
       "groupman_roles": "[\n    \"chair\",\n    \"delegate\"\n]",
@@ -5392,6 +5392,21 @@
     "model": "mailtrigger.mailtrigger",
     "pk": "review_completed_opsdir_telechat"
   },
+  {
+    "fields": {
+      "cc": [
+        "ietf_last_call",
+        "review_doc_all_parties",
+        "review_doc_group_mail_list"
+      ],
+      "desc": "Recipients when a perfmetrdir Telechat review is completed",
+      "to": [
+        "review_team_mail_list"
+      ]
+    },
+    "model": "mailtrigger.mailtrigger",
+    "pk": "review_completed_perfmetrdir_telechat"
+  },
   {
     "fields": {
       "cc": [
@@ -11330,6 +11345,17 @@
     "model": "name.extresourcename",
     "pk": "mailing_list_archive"
   },
+  {
+    "fields": {
+      "desc": "ORCID",
+      "name": "ORCID",
+      "order": 0,
+      "type": "url",
+      "used": true
+    },
+    "model": "name.extresourcename",
+    "pk": "orcid"
+  },
   {
     "fields": {
       "desc": "Related Implementations",
@@ -13562,7 +13588,7 @@
       "desc": "",
       "name": "Liaison CC Contact",
       "order": 9,
-      "used": true
+      "used": false
     },
     "model": "name.rolename",
     "pk": "liaison_cc_contact"
@@ -13572,11 +13598,21 @@
       "desc": "",
       "name": "Liaison Contact",
       "order": 8,
-      "used": true
+      "used": false
     },
     "model": "name.rolename",
     "pk": "liaison_contact"
   },
+  {
+    "fields": {
+      "desc": "Coordinates liaison handling for the IAB",
+      "name": "Liaison Coordinator",
+      "order": 14,
+      "used": true
+    },
+    "model": "name.rolename",
+    "pk": "liaison_coordinator"
+  },
   {
     "fields": {
       "desc": "",
@@ -13662,7 +13698,7 @@
       "desc": "Assigned permission TRAC_ADMIN in datatracker-managed Trac Wiki instances",
       "name": "Trac Admin",
       "order": 0,
-      "used": true
+      "used": false
     },
     "model": "name.rolename",
     "pk": "trac-admin"
@@ -13672,7 +13708,7 @@
       "desc": "Provides log-in permission to restricted Trac instances.  Used by the generate_apache_perms management command, called from ../../scripts/Cron-runner",
       "name": "Trac Editor",
       "order": 0,
-      "used": true
+      "used": false
     },
     "model": "name.rolename",
     "pk": "trac-editor"
diff --git a/ietf/name/migrations/0018_alter_rolenames.py b/ietf/name/migrations/0018_alter_rolenames.py
new file mode 100644
index 0000000000..f931de2e97
--- /dev/null
+++ b/ietf/name/migrations/0018_alter_rolenames.py
@@ -0,0 +1,36 @@
+# Copyright The IETF Trust 2025, All Rights Reserved# Generated by Django 4.2.21 on 2025-05-30 16:35
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    RoleName = apps.get_model("name", "RoleName")
+    RoleName.objects.filter(slug__in=["liaison_contact", "liaison_cc_contact"]).update(
+        used=False
+    )
+    RoleName.objects.get_or_create(
+        slug="liaison_coordinator",
+        defaults={
+            "name": "Liaison Coordinator",
+            "desc": "Coordinates liaison handling for the IAB",
+            "order": 14,
+        },
+    )
+    RoleName.objects.filter(slug__contains="trac-").update(used=False)
+
+
+def reverse(apps, schema_editor):
+    RoleName = apps.get_model("name", "RoleName")
+    RoleName.objects.filter(slug__in=["liaison_contact", "liaison_cc_contact"]).update(
+        used=True
+    )
+    RoleName.objects.filter(slug="liaison_coordinator").delete()
+    # Intentionally not restoring trac-* RoleNames to used=True
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("name", "0017_populate_new_reg_names"),
+    ]
+
+    operations = [migrations.RunPython(forward, reverse)]
diff --git a/ietf/templates/liaisons/detail.html b/ietf/templates/liaisons/detail.html
index 4bf5b4d11b..d46c8d1c98 100644
--- a/ietf/templates/liaisons/detail.html
+++ b/ietf/templates/liaisons/detail.html
@@ -32,7 +32,7 @@ <h1>
         {% if liaison.from_contact %}
             <tr>
                 <th scope="row">From Contact</th>
-                <td>{% person_link liaison.from_contact.person %}</td>
+                <td>{{ liaison.from_contact }}</td>
             </tr>
         {% endif %}
         <tr>
diff --git a/ietf/templates/liaisons/liaison_mail.txt b/ietf/templates/liaisons/liaison_mail.txt
index c92c68ff76..6d6a07d7ef 100644
--- a/ietf/templates/liaisons/liaison_mail.txt
+++ b/ietf/templates/liaisons/liaison_mail.txt
@@ -2,7 +2,7 @@
 Submission Date: {{ liaison.submitted|date:"Y-m-d" }}
 URL of the IETF Web page: {{ liaison.get_absolute_url }}
 {% if liaison.deadline %}Please reply by {{ liaison.deadline }}{% endif %}
-From: {% if liaison.from_contact %}{{ liaison.from_contact.formatted_email }}{% endif %}
+From: {% if liaison.from_contact %}{{ liaison.from_contact }}{% endif %}
 To: {{ liaison.to_contacts }}
 Cc: {{ liaison.cc_contacts }}
 Response Contacts: {{ liaison.response_contacts }}
diff --git a/ietf/utils/validators.py b/ietf/utils/validators.py
index 8fe989df99..92a20f5a26 100644
--- a/ietf/utils/validators.py
+++ b/ietf/utils/validators.py
@@ -4,6 +4,8 @@
 
 import os
 import re
+from email.utils import parseaddr
+
 from pyquery import PyQuery
 from urllib.parse import urlparse, urlsplit, urlunsplit
 
@@ -11,7 +13,13 @@
 from django.apps import apps
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
-from django.core.validators import RegexValidator, URLValidator, EmailValidator, BaseValidator
+from django.core.validators import (
+    RegexValidator,
+    URLValidator,
+    BaseValidator,
+    validate_email,
+    ProhibitNullCharactersValidator,
+)
 from django.template.defaultfilters import filesizeformat
 from django.utils.deconstruct import deconstructible
 from django.utils.ipv6 import is_valid_ipv6_address
@@ -136,8 +144,17 @@ def validate_no_html_frame(file):
 # instantiations of sub-validiators used by the external_resource validator
 
 validate_url = URLValidator()
-validate_http_url = URLValidator(schemes=['http','https'])
-validate_email = EmailValidator()
+validate_http_url = URLValidator(schemes=["http", "https"])
+validate_no_nulls = ProhibitNullCharactersValidator()
+
+
+def validate_mailbox_address(s):
+    """Validate an RFC 5322 'mailbox' (e.g., "Some Person" <some@example.com>)"""
+    # parseaddr() returns ("", "") on err; validate_email() will reject that for us
+    name, addr = parseaddr(s)
+    validate_no_nulls(name)  # could be stricter...
+    validate_email(addr)
+
 
 def validate_ipv6_address(value):
     if not is_valid_ipv6_address(value):

From e502fb4646bbfd39a1b391c65e7415723a63542e Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 26 Jun 2025 00:03:36 -0400
Subject: [PATCH 370/656] docs: Update README.md

---
 docker/README.md | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docker/README.md b/docker/README.md
index f2161a173f..b40bc6f925 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -157,12 +157,19 @@ On Windows:
 docker compose down -v --rmi all
 docker image prune
 ```
+
 ### Updating an older environment
 
 If you already have a clone, such as from a previous codesprint, and are updating that clone, before starting the datatracker from the updated image:
-* rm ietf/settings_local.py   # The startup script will put a new one, appropriate to the current release, in place
+* `rm ietf/settings_local.py`   # The startup script will put a new one, appropriate to the current release, in place
 * Execute the `Clean all` sequence above.
 
+If the dev environment fails to start, even after running the `Clean all` sequence above, you can fully purge all docker cache, containers, images and volumes by running:
+```sh
+docker system prune -a --volumes
+```
+Note that this will delete everything docker-related, including non-datatracker docker resources you might have.
+
 ### Accessing PostgreSQL Port
 
 The port is exposed but not automatically mapped to `5432` to avoid potential conflicts with the host. To get the mapped port, run the command *(from the project `/docker` directory)*:

From 1f540109b9b414b8f6b682faf917e69cdaf69711 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 26 Jun 2025 00:09:49 -0400
Subject: [PATCH 371/656] docs: Update README.md

---
 docker/README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index b40bc6f925..e202ca5e20 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -164,11 +164,14 @@ If you already have a clone, such as from a previous codesprint, and are updatin
 * `rm ietf/settings_local.py`   # The startup script will put a new one, appropriate to the current release, in place
 * Execute the `Clean all` sequence above.
 
-If the dev environment fails to start, even after running the `Clean all` sequence above, you can fully purge all docker cache, containers, images and volumes by running:
+If the dev environment fails to start, even after running the `Clean all` sequence above, you can fully purge all docker cache, containers, images and volumes by running the command below.
+
+> [!CAUTION]
+> Note that this will delete everything docker-related, including non-datatracker docker resources you might have.
+
 ```sh
 docker system prune -a --volumes
 ```
-Note that this will delete everything docker-related, including non-datatracker docker resources you might have.
 
 ### Accessing PostgreSQL Port
 

From 75f6945fdb139c98305a0de3abaeb1b5e09ea2a4 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 26 Jun 2025 00:12:50 -0400
Subject: [PATCH 372/656] docs: Update README.md

---
 docker/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index e202ca5e20..9b94eab138 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -161,10 +161,10 @@ docker image prune
 ### Updating an older environment
 
 If you already have a clone, such as from a previous codesprint, and are updating that clone, before starting the datatracker from the updated image:
-* `rm ietf/settings_local.py`   # The startup script will put a new one, appropriate to the current release, in place
-* Execute the `Clean all` sequence above.
+1. `rm ietf/settings_local.py` *(The startup script will put a new one, appropriate to the current release, in place)*
+1. Execute the [Clean all](#clean-all) sequence above.
 
-If the dev environment fails to start, even after running the `Clean all` sequence above, you can fully purge all docker cache, containers, images and volumes by running the command below.
+If the dev environment fails to start, even after running the [Clean all](#clean-all) sequence above, you can fully purge all docker cache, containers, images and volumes by running the command below.
 
 > [!CAUTION]
 > Note that this will delete everything docker-related, including non-datatracker docker resources you might have.

From 3241d5798d7f1f2b13ce78fef28f9b9a7c3b9f8a Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 26 Jun 2025 00:19:40 -0400
Subject: [PATCH 373/656] docs: Update README.md

---
 docker/README.md | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index 9b94eab138..0ca79a6e89 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -1,5 +1,18 @@
 # Datatracker Development in Docker
 
+- [Getting started](#getting-started)
+- [Using Visual Studio Code](#using-visual-studio-code)
+    - [Initial Setup](#initial-setup)
+    - [Subsequent Launch](#subsequent-launch)
+    - [Usage](#usage)
+- [Using Other Editors / Generic](#using-other-editors--generic)
+    - [Exit Environment](#exit-environment)
+    - [Accessing PostgreSQL Port](#accessing-postgresql-port)
+- [Clean and Rebuild DB from latest image](#clean-and-rebuild-db-from-latest-image)
+- [Clean all](#clean-all)
+- [Updating an older environment](#updating-an-older-environment)
+- [Notes / Troubleshooting](#notes--troubleshooting)
+
 ## Getting started
 
 1. [Set up Docker](https://docs.docker.com/get-started/) on your preferred platform. On Windows, it is highly recommended to use the [WSL 2 *(Windows Subsystem for Linux)*](https://docs.docker.com/desktop/windows/wsl/) backend.
@@ -123,7 +136,14 @@ docker compose down
 
 to terminate the containers.
 
-### Clean and Rebuild DB from latest image
+### Accessing PostgreSQL Port
+
+The port is exposed but not automatically mapped to `5432` to avoid potential conflicts with the host. To get the mapped port, run the command *(from the project `/docker` directory)*:
+```sh
+docker compose port db 5432
+```
+
+## Clean and Rebuild DB from latest image
 
 To delete the active DB container, its volume and get the latest image / DB dump, simply run the following command:
 
@@ -141,7 +161,7 @@ docker compose pull db
 docker compose build --no-cache db
 ```
 
-### Clean all
+## Clean all
 
 To delete all containers for this project, its associated images and purge any remaining dangling images, simply run the following command:
 
@@ -158,7 +178,7 @@ docker compose down -v --rmi all
 docker image prune
 ```
 
-### Updating an older environment
+## Updating an older environment
 
 If you already have a clone, such as from a previous codesprint, and are updating that clone, before starting the datatracker from the updated image:
 1. `rm ietf/settings_local.py` *(The startup script will put a new one, appropriate to the current release, in place)*
@@ -173,13 +193,6 @@ If the dev environment fails to start, even after running the [Clean all](#clean
 docker system prune -a --volumes
 ```
 
-### Accessing PostgreSQL Port
-
-The port is exposed but not automatically mapped to `5432` to avoid potential conflicts with the host. To get the mapped port, run the command *(from the project `/docker` directory)*:
-```sh
-docker compose port db 5432
-```
-
 ## Notes / Troubleshooting
 
 ### Slow zsh prompt inside Docker

From e285ad862f5e9766ba6e8125c8ecf28bf4b9c638 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@holloway.co.nz>
Date: Sat, 28 Jun 2025 05:10:09 +1200
Subject: [PATCH 374/656] fix: dropdown menu positioning (#9061)

fixes #9019
---
 ietf/static/css/ietf.scss |  2 +-
 ietf/static/js/ietf.js    | 43 ++++++++++++++++++++++++++++++++++-----
 2 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss
index aee93dcfe1..014213c3e5 100644
--- a/ietf/static/css/ietf.scss
+++ b/ietf/static/css/ietf.scss
@@ -229,7 +229,7 @@ th,
 .group-menu .dropdown-menu {
     height: auto;
     width: auto;
-    max-height: 35em;
+    max-height: 95vh;
     overflow-x: hidden;
     overflow-y: auto;
 }
diff --git a/ietf/static/js/ietf.js b/ietf/static/js/ietf.js
index dde00c6d1e..09fa324e42 100644
--- a/ietf/static/js/ietf.js
+++ b/ietf/static/js/ietf.js
@@ -13,7 +13,7 @@ import "bootstrap/js/dist/scrollspy";
 import "bootstrap/js/dist/tab";
 // import "bootstrap/js/dist/toast";
 import "bootstrap/js/dist/tooltip";
-
+import { debounce } from 'lodash-es';
 import jquery from "jquery";
 
 window.$ = window.jQuery = jquery;
@@ -112,6 +112,34 @@ function overflowShadows(el) {
     }
 }
 
+function ensureDropdownOnscreen(elm) {
+    const handlePlacement = () => {
+        if(!(elm instanceof HTMLElement)) {
+            return
+        }
+        const rect = elm.getBoundingClientRect()
+        const BUFFER_PX = 5 // additional distance from bottom of viewport
+        const existingStyleTop = parseInt(elm.style.top, 10)
+        const offscreenBy = Math.round(window.innerHeight - (rect.top + rect.height) - BUFFER_PX)
+        if(existingStyleTop === offscreenBy) {
+            console.log(`Already set top to ${offscreenBy}. Ignoring`)
+            // already set, nothing to do
+            return
+        }
+        if(offscreenBy < 0) {
+            elm.style.top = `${offscreenBy}px`
+        } 
+    }
+
+    const debouncedHandler = debounce(handlePlacement, 100)
+        
+    const observer = new MutationObserver(debouncedHandler)
+
+    observer.observe(elm, {
+        attributes: true
+    })
+}
+
 $(document)
     .ready(function () {
         // load data for the menu
@@ -144,12 +172,17 @@ $(document)
                             g.acronym + " &mdash; " + g.name + "</a></li>");
                     }
                     menu.push("</ul>");
-                    for (i = 0; i < attachTo.length; i++) {
-                        attachTo.closest(".dropdown-menu");
-                    }
+
                     attachTo.append(menu.join(""));
 
-                    attachTo.find(".overflow-shadows").each(function(){ overflowShadows(this)})
+                    attachTo.find(".overflow-shadows").each(function(){
+                        // needs to be a function(){} so that we can access jQuery's `this`
+                        overflowShadows(this)
+                    })
+                    attachTo.find(".dropdown-menu").each(function(){
+                        // needs to be a function(){} so that we can access jQuery's `this`
+                        ensureDropdownOnscreen(this)
+                    })
                 }
             }
         });

From 2b2891f4c8e9916d83d6d74c1b749d66111d33cb Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 27 Jun 2025 14:12:18 -0300
Subject: [PATCH 375/656] chore(k8s): wait for blobdb migrations at startup
 (#9072)

* chore(k8s): wait for blobdb migrations at startup

* chore(k8s): ...and for celery containers, too
---
 dev/build/celery-start.sh      | 23 +++++++++++++++++++++--
 dev/build/datatracker-start.sh | 23 +++++++++++++++++++++--
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/dev/build/celery-start.sh b/dev/build/celery-start.sh
index f6284e84db..69dcd7bbda 100644
--- a/dev/build/celery-start.sh
+++ b/dev/build/celery-start.sh
@@ -5,10 +5,29 @@
 echo "Running Datatracker checks..."
 ./ietf/manage.py check
 
-if ! ietf/manage.py migrate --check ; then
+# Check whether the blobdb database exists - inspectdb will return a false
+# status if not.
+if ietf/manage.py inspectdb --database blobdb > /dev/null 2>&1; then
+    HAVE_BLOBDB="yes"
+fi
+
+migrations_applied_for () {
+    local DATABASE=${1:-default}
+    ietf/manage.py migrate --check --database "$DATABASE"
+}
+
+migrations_all_applied () {
+    if [[ "$HAVE_BLOBDB" == "yes" ]]; then
+        migrations_applied_for default && migrations_applied_for blobdb
+    else
+        migrations_applied_for default
+    fi
+}
+
+if ! migrations_all_applied; then
     echo "Unapplied migrations found, waiting to start..."
     sleep 5
-    while ! ietf/manage.py migrate --check ; do
+    while ! migrations_all_applied ; do 
         echo "... still waiting for migrations..."
         sleep 5
     done
diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh
index 5eb99c4745..a676415a26 100644
--- a/dev/build/datatracker-start.sh
+++ b/dev/build/datatracker-start.sh
@@ -3,10 +3,29 @@
 echo "Running Datatracker checks..."
 ./ietf/manage.py check
 
-if ! ietf/manage.py migrate --check ; then
+# Check whether the blobdb database exists - inspectdb will return a false
+# status if not.
+if ietf/manage.py inspectdb --database blobdb > /dev/null 2>&1; then
+    HAVE_BLOBDB="yes"
+fi
+
+migrations_applied_for () {
+    local DATABASE=${1:-default}
+    ietf/manage.py migrate --check --database "$DATABASE"
+}
+
+migrations_all_applied () {
+    if [[ "$HAVE_BLOBDB" == "yes" ]]; then
+        migrations_applied_for default && migrations_applied_for blobdb
+    else
+        migrations_applied_for default
+    fi
+}
+
+if ! migrations_all_applied; then
     echo "Unapplied migrations found, waiting to start..."
     sleep 5
-    while ! ietf/manage.py migrate --check ; do 
+    while ! migrations_all_applied ; do 
         echo "... still waiting for migrations..."
         sleep 5
     done

From b81bf8eea84d37cb1ec2fe847950a861a1f4c310 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Mon, 30 Jun 2025 10:35:36 -0400
Subject: [PATCH 376/656] fix: handle agenda room slug with single quote +
 ampersand (#9074)

---
 client/shared/xslugify.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/shared/xslugify.js b/client/shared/xslugify.js
index e1f7a34483..e1ac556ddf 100644
--- a/client/shared/xslugify.js
+++ b/client/shared/xslugify.js
@@ -1,5 +1,5 @@
 import slugify from 'slugify'
 
 export default (str) => {
-  return slugify(str.replaceAll('/', '-'), { lower: true })
+  return slugify(str.replaceAll('/', '-').replaceAll(/['&]/g, ''), { lower: true })
 }

From 06a4182e0287769673106d5d0c415f0ec92cdb46 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 2 Jul 2025 17:55:33 -0300
Subject: [PATCH 377/656] test: tolerate APP_API_TOKENS existing in settings
 (#9089)

---
 ietf/api/tests.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 29edd99b2b..abaf9f5ed2 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -1582,7 +1582,7 @@ def test_bad_post(self):
             data = self.response_data(r)
             self.assertEqual(data["result"], "failure")
             self.assertEqual(data["reason"], "invalid post")
-        
+
         bad = dict(authtoken=self.valid_token, username=self.valid_person.user.username, password=self.valid_password)
         r = self.client.post(self.url, bad)
         self.assertEqual(r.status_code, 200)
@@ -1590,8 +1590,9 @@ def test_bad_post(self):
         self.assertEqual(data["result"], "failure")
         self.assertEqual(data["reason"], "invalid post")       
 
+    @override_settings()
     def test_notokenstore(self):
-        self.assertFalse(hasattr(settings, "APP_API_TOKENS"))
+        del settings.APP_API_TOKENS  # only affects overridden copy of settings!
         r = self.client.post(self.url,self.valid_body_with_good_password)
         self.assertEqual(r.status_code, 200)
         data = self.response_data(r)

From 4c2877f5a34d8a5e8b1618a3bb7387ed7070b281 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 3 Jul 2025 20:19:42 -0400
Subject: [PATCH 378/656] chore: Update devcontainer.json to disable vue.volar
 extension

---
 .devcontainer/devcontainer.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index ac7854f265..9be6c09f20 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -23,7 +23,7 @@
                 "dbaeumer.vscode-eslint",
                 "eamodio.gitlens",
                 "editorconfig.editorconfig",
-                "vue.volar",
+                // "vue.volar", # causes crash in devcontainer...
                 "mrmlnc.vscode-duplicate",
                 "ms-azuretools.vscode-docker",
                 "ms-playwright.playwright",

From 2bf633bf70c40b9cb6baf428901615a0403e1ea5 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 3 Jul 2025 20:27:02 -0400
Subject: [PATCH 379/656] chore: Update devcontainer.json to pin vue.volar to
 older version

---
 .devcontainer/devcontainer.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 9be6c09f20..6b0fd79bb3 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -23,7 +23,8 @@
                 "dbaeumer.vscode-eslint",
                 "eamodio.gitlens",
                 "editorconfig.editorconfig",
-                // "vue.volar", # causes crash in devcontainer...
+                // Newer volar >=3.0.0 causes crashes in devcontainers
+                "vue.volar@2.2.10",
                 "mrmlnc.vscode-duplicate",
                 "ms-azuretools.vscode-docker",
                 "ms-playwright.playwright",

From 99775c4df25e68e2e4194d033107a7406353409a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 9 Jul 2025 19:03:30 -0300
Subject: [PATCH 380/656] fix: correct iCalendar text escaping (#9118)

Old implementation did not handle newlines and allowed many invalid chars to go through.
---
 ietf/doc/templatetags/ietf_filters.py | 54 ++++++++++++++++++++++++---
 1 file changed, 48 insertions(+), 6 deletions(-)

diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py
index d4adf96a27..5cabe1728d 100644
--- a/ietf/doc/templatetags/ietf_filters.py
+++ b/ietf/doc/templatetags/ietf_filters.py
@@ -285,7 +285,7 @@ def urlize_related_source_list(related, document_html=False):
                                                                       url=url)
         ))
     return links
-        
+
 @register.filter(name='urlize_related_target_list', is_safe=True, document_html=False)
 def urlize_related_target_list(related, document_html=False):
     """Convert a list of RelatedDocuments into list of links using the target document's canonical name"""
@@ -302,7 +302,7 @@ def urlize_related_target_list(related, document_html=False):
                                                                       url=url)
         ))
     return links
-        
+
 @register.filter(name='dashify')
 def dashify(string):
     """
@@ -521,10 +521,52 @@ def plural(text, seq, arg='s'):
     else:
         return text + pluralize(len(seq), arg)
 
+
+# Translation table to escape ICS characters. The {} | {} construction builds up a dict
+# mapping characters to arbitrary-length strings or None. Values in later dicts override
+# earlier ones prior to conversion to a translation table, so excluding a char and then
+# mapping it to an escape sequence results in its being escaped, not dropped.
+rfc5545_text_escapes = str.maketrans(
+    # text       = *(TSAFE-CHAR / ":" / DQUOTE / ESCAPED-CHAR)
+    # TSAFE-CHAR = WSP / %x21 / %x23-2B / %x2D-39 / %x3C-5B /
+    #                    %x5D-7E / NON-US-ASCII
+    {chr(c): None for c in range(0x00, 0x20)}  # strip 0x00-0x20
+    | {
+        # ESCAPED-CHAR = ("\\" / "\;" / "\," / "\N" / "\n")
+        "\n": r"\n",
+        ";": r"\;",
+        ",": r"\,",
+        "\\": r"\\",  # rhs is two backslashes!
+        "\t": "\t",  # htab ok (0x09)
+        " ": " ",  # space ok (0x20)
+    }
+)
+
+
 @register.filter
 def ics_esc(text):
-    text = re.sub(r"([\n,;\\])", r"\\\1", text)
-    return text
+    """Escape a string to use in an iCalendar text context
+    
+    >>> ics_esc('simple')
+    'simple'
+    
+    For the next tests, it helps to know:
+      chr(0x09) = "\t"
+      chr(0x0a) = "\n"
+      chr(0x0d) = "\r"
+      chr(0x5c) = "\\"
+    
+    >>> ics_esc(f'strips{chr(0x0d)}out{chr(0x0d)}LFs')
+    'stripsoutLFs'
+    
+    
+    >>> ics_esc(f'escapes;and,and{chr(0x5c)}and{chr(0x0a)}')
+    'escapes\\\\;and\\\\,and\\\\\\\\and\\\\n'
+    
+    >>> ics_esc(f"keeps spaces : and{chr(0x09)}tabs")
+    'keeps spaces : and\\ttabs'
+    """
+    return text.translate(rfc5545_text_escapes)
 
 
 @register.simple_tag
@@ -557,7 +599,7 @@ def ics_date_time(dt, tzname):
         return f':{timestamp}Z'
     else:
         return f';TZID={ics_esc(tzname)}:{timestamp}'
-    
+
 @register.filter
 def next_day(value):
     return value + datetime.timedelta(days=1)
@@ -676,7 +718,7 @@ def rfcbis(s):
 @stringfilter
 def urlize(value):
     raise RuntimeError("Use linkify from textfilters instead of urlize")
-    
+
 @register.filter
 @stringfilter
 def charter_major_rev(rev):

From 58bc7c9c385f88b665796ab358210282d8313aaa Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Jul 2025 11:43:18 -0300
Subject: [PATCH 381/656] fix: improve pw validation without js (#9113)

* fix: backend validators for PasswordForm

Only used for password reset

* refactor: share more code

* fix: reconcile with PersonPasswordForm

This form class hierarchy reallllly needs to be refactored more deeply

* fix: attach pw errors to specific fields

* fix: return val from clean_password_confirmation

* refactor: share PasswordForm code

* test: update test_change_password

* test: update test_create_existing_account

* test: update test_reset_password
---
 ietf/ietfauth/forms.py | 86 ++++++++++++++++++++++--------------------
 ietf/ietfauth/tests.py | 67 ++++++++++++++++++++++++--------
 ietf/ietfauth/views.py |  6 +--
 3 files changed, 99 insertions(+), 60 deletions(-)

diff --git a/ietf/ietfauth/forms.py b/ietf/ietfauth/forms.py
index 53bf7b5888..41828f2bf6 100644
--- a/ietf/ietfauth/forms.py
+++ b/ietf/ietfauth/forms.py
@@ -33,21 +33,56 @@ def clean_email(self):
         return email
 
 
+class PasswordStrengthField(forms.CharField):
+    widget = PasswordStrengthInput(
+        attrs={
+            "class": "password_strength",
+            "data-disable-strength-enforcement": "",  # usually removed in init
+        }
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        for pwval in password_validation.get_default_password_validators():
+            if isinstance(pwval, password_validation.MinimumLengthValidator):
+                self.widget.attrs["minlength"] = pwval.min_length
+            elif isinstance(pwval, StrongPasswordValidator):
+                self.widget.attrs.pop(
+                    "data-disable-strength-enforcement", None
+                )
+
+    
+
 class PasswordForm(forms.Form):
-    password = forms.CharField(widget=PasswordStrengthInput(attrs={'class':'password_strength'}))
+    password = PasswordStrengthField()
     password_confirmation = forms.CharField(widget=PasswordConfirmationInput(
                                                         confirm_with='password',
                                                         attrs={'class':'password_confirmation'}),
                                             help_text="Enter the same password as above, for verification.",)
-                                            
+
+    def __init__(self, *args, user=None, **kwargs):
+        # user is a kw-only argument to avoid interfering with the signature
+        # when this class is mixed with ModelForm in PersonPasswordForm
+        self.user = user
+        super().__init__(*args, **kwargs)
 
     def clean_password_confirmation(self):
-        password = self.cleaned_data.get("password", "")
-        password_confirmation = self.cleaned_data["password_confirmation"]
+        # clean fields here rather than a clean() method so validation is
+        # still enforced in PersonPasswordForm without having to override its
+        # clean() method
+        password = self.cleaned_data.get("password")
+        password_confirmation = self.cleaned_data.get("password_confirmation")
         if password != password_confirmation:
-            raise forms.ValidationError("The two password fields didn't match.")
+            raise ValidationError(
+                "The password confirmation is different than the new password" 
+            )
+        try:
+            password_validation.validate_password(password_confirmation, self.user)
+        except ValidationError as err:
+            self.add_error("password", err)
         return password_confirmation
 
+
 def ascii_cleaner(supposedly_ascii):
     outside_printable_ascii_pattern = r'[^\x20-\x7F]'
     if re.search(outside_printable_ascii_pattern, supposedly_ascii):
@@ -174,35 +209,13 @@ class Meta:
         exclude = ['by', 'time' ]
 
 
-class ChangePasswordForm(forms.Form):
+class ChangePasswordForm(PasswordForm):
     current_password = forms.CharField(widget=forms.PasswordInput)
+    field_order = ["current_password", "password", "password_confirmation"]
 
-    new_password = forms.CharField(
-        widget=PasswordStrengthInput(
-            attrs={
-                "class": "password_strength",
-                "data-disable-strength-enforcement": "",  # usually removed in init
-            }
-        ),
-    )
-    new_password_confirmation = forms.CharField(
-        widget=PasswordConfirmationInput(
-            confirm_with="new_password", attrs={"class": "password_confirmation"}
-        )
-    )
-
-    def __init__(self, user, data=None):
-        self.user = user
-        super().__init__(data)
-        # Check whether we have validators to enforce
-        new_password_field = self.fields["new_password"]
-        for pwval in password_validation.get_default_password_validators():
-            if isinstance(pwval, password_validation.MinimumLengthValidator):
-                new_password_field.widget.attrs["minlength"] = pwval.min_length
-            elif isinstance(pwval, StrongPasswordValidator):
-                new_password_field.widget.attrs.pop(
-                    "data-disable-strength-enforcement", None
-                )
+    def __init__(self, user, *args, **kwargs):
+        # user arg is optional in superclass, but required for this form
+        super().__init__(*args, user=user, **kwargs)
 
     def clean_current_password(self):
         # n.b., password = None is handled by check_password and results in a failed check
@@ -211,15 +224,6 @@ def clean_current_password(self):
             raise ValidationError("Invalid password")
         return password
 
-    def clean(self):
-        new_password = self.cleaned_data.get("new_password", "")
-        conf_password = self.cleaned_data.get("new_password_confirmation", "")
-        if new_password != conf_password:
-            raise ValidationError(
-                "The password confirmation is different than the new password"
-            )
-        password_validation.validate_password(conf_password, self.user)
-
 
 class ChangeUsernameForm(forms.Form):
     username = forms.ChoiceField(choices=[('-','--------')])
diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py
index dd23277b63..a77e5bd5d5 100644
--- a/ietf/ietfauth/tests.py
+++ b/ietf/ietfauth/tests.py
@@ -168,18 +168,40 @@ def register_and_verify(self, email):
         self.assertEqual(r.status_code, 200)
 
         # password mismatch
-        r = self.client.post(confirm_url, { 'password': 'secret', 'password_confirmation': 'nosecret' })
+        r = self.client.post(
+            confirm_url, {
+                "password": "secret-and-secure",
+                "password_confirmation": "not-secret-or-secure",
+            }
+        )
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(User.objects.filter(username=email).count(), 0)
+
+        # weak password
+        r = self.client.post(
+            confirm_url, {
+                "password": "password1234",
+                "password_confirmation": "password1234",
+            }
+        )
         self.assertEqual(r.status_code, 200)
         self.assertEqual(User.objects.filter(username=email).count(), 0)
 
         # confirm
-        r = self.client.post(confirm_url, { 'name': 'User Name', 'ascii': 'User Name', 'password': 'secret', 'password_confirmation': 'secret' })
+        r = self.client.post(
+            confirm_url,
+            {
+                "name": "User Name",
+                "ascii": "User Name",
+                "password": "secret-and-secure",
+                "password_confirmation": "secret-and-secure",
+            },
+        )
         self.assertEqual(r.status_code, 200)
         self.assertEqual(User.objects.filter(username=email).count(), 1)
         self.assertEqual(Person.objects.filter(user__username=email).count(), 1)
         self.assertEqual(Email.objects.filter(person__user__username=email).count(), 1)
 
-        
     # This also tests new account creation.
     def test_create_existing_account(self):
         # create account once
@@ -393,6 +415,7 @@ def test_nomcom_dressing_on_profile(self):
         self.assertTrue(q('#volunteered'))
 
     def test_reset_password(self):
+        WEAK_PASSWORD="password1234"
         VALID_PASSWORD = "complex-and-long-valid-password"
         ANOTHER_VALID_PASSWORD = "very-complicated-and-lengthy-password"
         url = urlreverse("ietf.ietfauth.views.password_reset")
@@ -450,6 +473,18 @@ def test_reset_password(self):
         q = PyQuery(r.content)
         self.assertTrue(len(q("form .is-invalid")) > 0)
 
+        # weak password
+        r = self.client.post(
+            confirm_url,
+            {
+                "password": WEAK_PASSWORD,
+                "password_confirmation": WEAK_PASSWORD,
+            },
+        )
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertTrue(len(q("form .is-invalid")) > 0)
+
         # confirm
         r = self.client.post(
             confirm_url,
@@ -636,8 +671,8 @@ def test_change_password(self):
             chpw_url,
             {
                 "current_password": "fiddlesticks",
-                "new_password": ANOTHER_VALID_PASSWORD,
-                "new_password_confirmation": ANOTHER_VALID_PASSWORD,
+                "password": ANOTHER_VALID_PASSWORD,
+                "password_confirmation": ANOTHER_VALID_PASSWORD,
             },
         )
         self.assertEqual(r.status_code, 200)
@@ -648,14 +683,14 @@ def test_change_password(self):
             chpw_url,
             {
                 "current_password": VALID_PASSWORD,
-                "new_password": ANOTHER_VALID_PASSWORD,
-                "new_password_confirmation": ANOTHER_VALID_PASSWORD[::-1],
+                "password": ANOTHER_VALID_PASSWORD,
+                "password_confirmation": ANOTHER_VALID_PASSWORD[::-1],
             },
         )
         self.assertEqual(r.status_code, 200)
         self.assertFormError(
             r.context["form"],
-            None,
+            "password_confirmation",
             "The password confirmation is different than the new password",
         )
 
@@ -664,14 +699,14 @@ def test_change_password(self):
             chpw_url,
             {
                 "current_password": VALID_PASSWORD,
-                "new_password": "sh0rtpw0rd",
-                "new_password_confirmation": "sh0rtpw0rd",
+                "password": "sh0rtpw0rd",
+                "password_confirmation": "sh0rtpw0rd",
             }
         )
         self.assertEqual(r.status_code, 200)
         self.assertFormError(
             r.context["form"],
-            None,
+            "password",
             "This password is too short. It must contain at least "
             f"{settings.PASSWORD_POLICY_MIN_LENGTH} characters."
         )
@@ -681,14 +716,14 @@ def test_change_password(self):
             chpw_url,
             {
                 "current_password": VALID_PASSWORD,
-                "new_password": "passwordpassword",
-                "new_password_confirmation": "passwordpassword",
+                "password": "passwordpassword",
+                "password_confirmation": "passwordpassword",
             }
         )
         self.assertEqual(r.status_code, 200)
         self.assertFormError(
             r.context["form"],
-            None,
+            "password",
             "This password does not meet complexity requirements "
             "and is easily guessable."
         )
@@ -698,8 +733,8 @@ def test_change_password(self):
             chpw_url,
             {
                 "current_password": VALID_PASSWORD,
-                "new_password": ANOTHER_VALID_PASSWORD,
-                "new_password_confirmation": ANOTHER_VALID_PASSWORD,
+                "password": ANOTHER_VALID_PASSWORD,
+                "password_confirmation": ANOTHER_VALID_PASSWORD,
             },
         )
         self.assertRedirects(r, prof_url)
diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py
index 84d5490873..4219747e12 100644
--- a/ietf/ietfauth/views.py
+++ b/ietf/ietfauth/views.py
@@ -529,7 +529,7 @@ def confirm_password_reset(request, auth):
         )
     success = False
     if request.method == 'POST':
-        form = PasswordForm(request.POST)
+        form = PasswordForm(user=user, data=request.POST)
         if form.is_valid():
             password = form.cleaned_data["password"]
 
@@ -538,7 +538,7 @@ def confirm_password_reset(request, auth):
 
             success = True
     else:
-        form = PasswordForm()
+        form = PasswordForm(user=user)
 
     hlibname, hashername = settings.PASSWORD_HASHERS[0].rsplit('.',1)
     hlib = importlib.import_module(hlibname)
@@ -669,7 +669,7 @@ def change_password(request):
     if request.method == 'POST':
         form = ChangePasswordForm(user, request.POST)
         if form.is_valid():
-            new_password = form.cleaned_data["new_password"]
+            new_password = form.cleaned_data["password"]
             
             user.set_password(new_password)
             user.save()

From 98f19cd317f2b05f988e74faeb3501ac766a4d25 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@holloway.co.nz>
Date: Fri, 11 Jul 2025 02:43:45 +1200
Subject: [PATCH 382/656] fix: agenda 'show meeting materials' button theme
 (#9119)

---
 client/agenda/AgendaScheduleList.vue | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index d38c7842eb..fc8b5fd30f 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -302,7 +302,7 @@ const meetingEvents = computed(() => {
           icon: 'collection',
           href: undefined,
           click: () => showMaterials(item.id),
-          color: 'black'
+          color: 'darkgray'
         })
         links.push({
           id: `lnk-${item.id}-tar`,
@@ -1155,7 +1155,7 @@ onBeforeUnmount(() => {
       .agenda-table-cell-links-buttons {
         white-space: nowrap;
 
-        > a, > i {
+        > a, > i, > button {
           margin-left: 3px;
           color: #666;
           cursor: pointer;
@@ -1197,6 +1197,18 @@ onBeforeUnmount(() => {
               background-color: rgba($orange-500, .3);
             }
           }
+          &.text-darkgray {
+            color: $gray-900;
+            background-color: rgba($gray-700, .1);
+
+            @at-root .theme-dark & {
+              color: $gray-100;
+            }
+
+            &:hover, &:focus {
+              background-color: rgba($gray-700, .3);
+            }
+          }
           &.text-blue {
             color: $blue-600;
             background-color: rgba($blue-300, .1);

From 6bb20be44cd64ff064c2eb4ea23905bb71a65374 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Jul 2025 12:12:35 -0300
Subject: [PATCH 383/656] fix: next -> continue (#9121)

---
 ietf/sync/rfceditor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index a3c6580452..76357ffbcc 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -676,11 +676,11 @@ def parse_relation_list(l):
                 subseries_slug = a[:3]
                 if subseries_slug not in ["bcp", "std", "fyi"]:
                     log(f"Unexpected 'also' relationship of {a} encountered for {doc}")
-                    next
+                    continue
                 maybe_number = a[3:].strip()
                 if not maybe_number.isdigit():
                     log(f"Unexpected 'also' subseries element identifier {a} encountered for {doc}")
-                    next
+                    continue
                 else:
                     subseries_number = int(maybe_number)
                     conditioned_also.append(f"{subseries_slug}{subseries_number}") # Note the lack of leading zeros

From 1eff7d17a9283d6f24977de130f194fc8121792d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 19 Jul 2025 04:55:54 -0400
Subject: [PATCH 384/656] chore(deps): bump ncipollo/release-action from 1.16.0
 to 1.18.0 (#9083)

Bumps [ncipollo/release-action](https://github.com/ncipollo/release-action) from 1.16.0 to 1.18.0.
- [Release notes](https://github.com/ncipollo/release-action/releases)
- [Commits](https://github.com/ncipollo/release-action/compare/v1.16.0...v1.18.0)

---
updated-dependencies:
- dependency-name: ncipollo/release-action
  dependency-version: 1.18.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index bc9680cd9d..9c24380764 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -98,7 +98,7 @@ jobs:
         echo "IS_RELEASE=true" >> $GITHUB_ENV
         
     - name: Create Draft Release
-      uses: ncipollo/release-action@v1.16.0
+      uses: ncipollo/release-action@v1.18.0
       if: ${{ github.ref_name == 'release' }}
       with:
         prerelease: true
@@ -317,7 +317,7 @@ jobs:
         histCoveragePath: historical-coverage.json
         
     - name: Create Release
-      uses: ncipollo/release-action@v1.16.0
+      uses: ncipollo/release-action@v1.18.0
       if: ${{ env.SHOULD_DEPLOY == 'true' }}
       with:
         allowUpdates: true
@@ -330,7 +330,7 @@ jobs:
         token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Update Baseline Coverage
-      uses: ncipollo/release-action@v1.16.0
+      uses: ncipollo/release-action@v1.18.0
       if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }}
       with:
         allowUpdates: true

From 0909583925189812ea63fbf0b5a5d540f002cc5e Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Sat, 19 Jul 2025 11:12:25 +0200
Subject: [PATCH 385/656] Merge pull request #8966 from
 jennifer-richards/remove-celery-image

chore(dev): remove the datatracker-celery image
---
 .github/workflows/build-celery-worker.yml     | 47 -------------------
 dev/celery/Dockerfile                         | 23 ---------
 dev/deploy-to-container/cli.js                | 13 ++---
 docker/celery.Dockerfile                      |  4 +-
 docker/docker-compose.celery.yml              | 47 -------------------
 .../scripts/app-init-celery.sh                | 11 +++--
 6 files changed, 12 insertions(+), 133 deletions(-)
 delete mode 100644 .github/workflows/build-celery-worker.yml
 delete mode 100644 dev/celery/Dockerfile
 delete mode 100644 docker/docker-compose.celery.yml
 rename dev/celery/docker-init.sh => docker/scripts/app-init-celery.sh (90%)

diff --git a/.github/workflows/build-celery-worker.yml b/.github/workflows/build-celery-worker.yml
deleted file mode 100644
index d14e4f2c8b..0000000000
--- a/.github/workflows/build-celery-worker.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-name: Build Celery Worker Docker Image
-
-on:
-  push:
-    branches:
-      - 'main'
-    paths:
-      - 'requirements.txt'
-      - 'dev/celery/**'
-      - '.github/workflows/build-celery-worker.yml'
-
-  workflow_dispatch: 
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-    
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
-
-    - name: Login to GitHub Container Registry
-      uses: docker/login-action@v3
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Docker Build & Push
-      uses: docker/build-push-action@v6
-      env:
-        DOCKER_BUILD_NO_SUMMARY: true
-      with:
-        context: .
-        file: dev/celery/Dockerfile
-        platforms: linux/amd64,linux/arm64
-        push: true
-        tags: ghcr.io/ietf-tools/datatracker-celery:latest
-
diff --git a/dev/celery/Dockerfile b/dev/celery/Dockerfile
deleted file mode 100644
index 12eb15eb81..0000000000
--- a/dev/celery/Dockerfile
+++ /dev/null
@@ -1,23 +0,0 @@
-# Dockerfile for celery worker
-#
-FROM ghcr.io/ietf-tools/datatracker-app-base:latest
-LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get purge -y imagemagick imagemagick-6-common
-
-# Copy the startup file
-COPY dev/celery/docker-init.sh /docker-init.sh
-RUN sed -i 's/\r$//' /docker-init.sh && \
-    chmod +x /docker-init.sh
-
-# Install current datatracker python dependencies
-COPY requirements.txt /tmp/pip-tmp/
-RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt
-RUN rm -rf /tmp/pip-tmp
-
-# Add watchmedo utility for dev containers
-RUN pip3 --disable-pip-version-check --no-cache-dir install watchdog[watchmedo]
-
-ENTRYPOINT [ "/docker-init.sh" ]
diff --git a/dev/deploy-to-container/cli.js b/dev/deploy-to-container/cli.js
index 4aee7ba1ed..1a2d993ac4 100644
--- a/dev/deploy-to-container/cli.js
+++ b/dev/deploy-to-container/cli.js
@@ -70,6 +70,7 @@ async function main () {
       .replace('__HOSTNAME__', hostname)
   )
   await fs.copy(path.join(basePath, 'docker/scripts/app-create-dirs.sh'), path.join(releasePath, 'app-create-dirs.sh'))
+  await fs.copy(path.join(basePath, 'docker/scripts/app-init-celery.sh'), path.join(releasePath, 'app-init-celery.sh'))
   await fs.copy(path.join(basePath, 'dev/deploy-to-container/start.sh'), path.join(releasePath, 'start.sh'))
   await fs.copy(path.join(basePath, 'test/data'), path.join(releasePath, 'test/data'))
   console.info('Updated configuration files.')
@@ -98,14 +99,6 @@ async function main () {
   })
   console.info('Pulled latest MQ docker image.')
 
-  // Pull latest Celery image
-  console.info('Pulling latest Celery docker image...')
-  const celeryImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-celery:latest')
-  await new Promise((resolve, reject) => {
-    dock.modem.followProgress(celeryImagePullStream, (err, res) => err ? reject(err) : resolve(res))
-  })
-  console.info('Pulled latest Celery docker image.')
-
   // Terminate existing containers
   console.info('Ensuring existing containers with same name are terminated...')
   const containers = await dock.listContainers({ all: true })
@@ -221,7 +214,7 @@ async function main () {
   const celeryContainers = {}
   for (const conConf of conConfs) {
     celeryContainers[conConf.name] = await dock.createContainer({
-      Image: 'ghcr.io/ietf-tools/datatracker-celery:latest',
+      Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest',
       name: `dt-${conConf.name}-${branch}`,
       Hostname: `dt-${conConf.name}-${branch}`,
       Env: [
@@ -243,7 +236,7 @@ async function main () {
           Name: 'unless-stopped'
         }
       },
-      Cmd: ['--loglevel=INFO']
+      Entrypoint: ['bash', '-c', 'chmod +x ./app-init-celery.sh && ./app-init-celery.sh']
     })
   }
   console.info('Created Celery docker containers successfully.')
diff --git a/docker/celery.Dockerfile b/docker/celery.Dockerfile
index e44200398c..e7c7b9cc3f 100644
--- a/docker/celery.Dockerfile
+++ b/docker/celery.Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-celery:latest
+FROM ghcr.io/ietf-tools/datatracker-app-base:latest
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -34,7 +34,7 @@ RUN bash /tmp/library-scripts/docker-setup-python.sh "none" "/usr/local" "${PIPX
 RUN rm -rf /tmp/library-scripts
 
 # Copy the startup file
-COPY dev/celery/docker-init.sh /docker-init.sh
+COPY docker/scripts/app-init-celery.sh /docker-init.sh
 RUN sed -i 's/\r$//' /docker-init.sh && \
     chmod +x /docker-init.sh
 
diff --git a/docker/docker-compose.celery.yml b/docker/docker-compose.celery.yml
deleted file mode 100644
index b6cc3d09e8..0000000000
--- a/docker/docker-compose.celery.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-services:
-  mq:
-    image: rabbitmq:3-alpine
-    user: '${RABBITMQ_UID:-499:499}'
-    hostname: datatracker-mq
-#    deploy:
-#      resources:
-#        limits:
-#          memory: 1gb # coordinate with settings in rabbitmq.conf
-#        reservations:
-#          memory: 512mb
-    mem_limit: 1gb  # coordinate with settings in rabbitmq.conf
-    ports:
-      - '${MQ_PORT:-5672}:5672'
-    volumes:
-      - ./lib.rabbitmq:/var/lib/rabbitmq
-      - ./rabbitmq.conf:/etc/rabbitmq/conf.d/90-ietf.conf
-      - ./definitions.json:/ietf-conf/definitions.json
-    restart: unless-stopped
-    logging:
-      driver: "syslog"
-      options:
-        syslog-address: 'unixgram:///dev/log'
-        tag: 'docker/{{.Name}}'
-#        syslog-address: "tcp://ietfa.amsl.com:514"
-
-  celery:
-    image: ghcr.io/ietf-tools/datatracker-celery:latest
-    environment:
-      CELERY_APP: ietf
-      # UPDATE_REQUIREMENTS: 1  # uncomment to update Python requirements on startup
-    command:
-      - '--loglevel=INFO'
-    user: '${CELERY_UID:-499:499}'
-    volumes:
-      - '${DATATRACKER_PATH:-..}:/workspace'
-      - '${MYSQL_SOCKET_PATH:-/run/mysql}:/run/mysql'
-    depends_on:
-      - mq
-    network_mode: 'service:mq'
-    restart: unless-stopped
-    logging:
-      driver: "syslog"
-      options:
-        syslog-address: 'unixgram:///dev/log'
-        tag: 'docker/{{.Name}}'
-#        syslog-address: "tcp://ietfa.amsl.com:514"
diff --git a/dev/celery/docker-init.sh b/docker/scripts/app-init-celery.sh
similarity index 90%
rename from dev/celery/docker-init.sh
rename to docker/scripts/app-init-celery.sh
index 6a36cb6d78..5788b943da 100755
--- a/dev/celery/docker-init.sh
+++ b/docker/scripts/app-init-celery.sh
@@ -90,19 +90,22 @@ if [[ "${CELERY_ROLE}" == "worker" ]]; then
     run_as_celery_uid /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check
 fi
 
+USER_BIN_PATH="/home/dev/.local/bin"
+WATCHMEDO="$USER_BIN_PATH/watchmedo"
+CELERY="$USER_BIN_PATH/celery"
 trap 'trap "" TERM; cleanup' TERM
 # start celery in the background so we can trap the TERM signal
-if [[ -n "${DEV_MODE}" ]]; then
-  watchmedo auto-restart \
+if [[ -n "${DEV_MODE}" && -x "${WATCHMEDO}" ]]; then
+  $WATCHMEDO auto-restart \
             --patterns '*.py' \
             --directory 'ietf' \
             --recursive \
             --debounce-interval 5 \
             -- \
-            celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" $@ &
+            $CELERY --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" $@ &
   celery_pid=$!
 else
-  celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" "$@" &
+  $CELERY --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" "$@" &
   celery_pid=$!
 fi
 

From 4ea16888201116de81d183703a04d899e8662101 Mon Sep 17 00:00:00 2001
From: Eric Vyncke <eric@vyncke.org>
Date: Sat, 19 Jul 2025 11:20:02 +0200
Subject: [PATCH 386/656] fix: use bootstrap body color in highcharts (#9171)

---
 ietf/templates/doc/ad_list.html | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html
index 1d7b6e2b54..7f7e95a873 100644
--- a/ietf/templates/doc/ad_list.html
+++ b/ietf/templates/doc/ad_list.html
@@ -8,15 +8,19 @@
 {% endblock %}
 {% block morecss %}
     table .border-bottom { border-bottom-color: var(--highcharts-neutral-color-80) !important; }
-    .highcharts-container .highcharts-axis-labels { font-size: .7rem; }
+    .highcharts-container .highcharts-axis-labels { 
+        font-size: .7rem; 
+        fill: var(--bs-body-color)
+    }
     .highcharts-container .highcharts-graph { stroke-width: 2.5; }
     .highcharts-container .highcharts-color-0 {
-        fill: var(--bs-primary);
+        fill: var(--bs-body-color);
         stroke: var(--bs-primary);
     }
     .highcharts-container .highcharts-data-label text {
         font-size: 1rem;
         font-weight: inherit;
+        fill: var(--bs-body-color)
     }
 {% endblock %}
 {% block title %}IESG Dashboard{% endblock %}

From ae60b41f60939a25f6268e35c2800d0a20fcd3df Mon Sep 17 00:00:00 2001
From: Peter Yee <github@houseofyee.com>
Date: Sat, 19 Jul 2025 11:33:20 +0200
Subject: [PATCH 387/656] feat: Add "IESG" group at top of Groups menu (#9144)

* feat: Add "IESG" group at top of Groups menu
Creates an IESG group at the top of the Group dropdown menu that
points at https://datatracker.ietf.org/group/iesg/about/. Maybe
that value shouldn't be hardcoded into menu.html. This feature
is in response to issue #9018.

* feat: Add "IESG" group at top of Groups menu
This feature is in response to issue #9018. This commit is an
update to the original commit to add proper support for the
left menu and use a non-hardcoded URL for the IESG.
---
 ietf/templates/base/menu.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html
index 3ae0d3dc57..81f012a679 100644
--- a/ietf/templates/base/menu.html
+++ b/ietf/templates/base/menu.html
@@ -19,6 +19,7 @@
             <span class="fw-bolder">Groups</span>
         </li>
     {% endif %}
+    <li><a class="dropdown-item" href="{% url "ietf.group.views.group_about" acronym="iesg" %}">IESG</a></li>
     {% if flavor == 'top' %}<li class="dropdown-header">By area/parent</li>{% endif %}
     {% wg_menu flavor %}
     <li class="dropend">

From aae03d94cfb6aaeb29e033b6f1f4557217218537 Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Sat, 19 Jul 2025 12:50:09 +0200
Subject: [PATCH 388/656] fix: fix issue cancelling unpaid registration (#9170)

* fix: fix issue cancelling unpaid registration

* fix: add logging to reg API
---
 ietf/meeting/utils.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index db67f79b93..ee1a3b46d7 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -1423,6 +1423,7 @@ def sync_registration_data(meeting):
     # Delete registrations that exist in the DB but not in registration data, they've been cancelled
     emails_to_delete = existing_emails - reg_emails
     if emails_to_delete:
+        log(f"sync_reg: emails marked for deletion: {emails_to_delete}")
         result = Registration.objects.filter(
             email__in=emails_to_delete,
             meeting=meeting
@@ -1432,7 +1433,6 @@ def sync_registration_data(meeting):
         else:
             deleted_count = 0
         stats['deleted'] = deleted_count
-
     # set meeting.attendees
     count = Registration.objects.onsite().filter(meeting=meeting, checkedin=True).count()
     if meeting.attendees != count:
@@ -1474,14 +1474,16 @@ def process_single_registration(reg_data, meeting):
             target = registration.tickets.filter(
                 attendance_type__slug=ticket['attendance_type'],
                 ticket_type__slug=ticket['ticket_type']).first()
-            target.delete()
+            if target:
+                target.delete()
         if registration.tickets.count() == 0:
             registration.delete()
+        log(f"sync_reg: cancelled registration {reg_data['email']}")
         return (None, 'deleted')
 
     person = Person.objects.filter(email__address=reg_data['email']).first()
     if not person:
-        log.log(f"ERROR: meeting registration email unknown {reg_data['email']}")
+        log(f"ERROR: meeting registration email unknown {reg_data['email']}")
 
     registration, created = Registration.objects.get_or_create(
         email=reg_data['email'],
@@ -1500,6 +1502,7 @@ def process_single_registration(reg_data, meeting):
     if not created:
         for field in ['first_name', 'last_name', 'affiliation', 'country_code', 'checkedin']:
             if getattr(registration, field) != reg_data[field]:
+                log(f"sync_reg: found update {reg_data['email']}, {field} different, data from reg: {reg_data}")
                 setattr(registration, field, reg_data[field])
                 fields_updated = True
 
@@ -1536,6 +1539,7 @@ def process_single_registration(reg_data, meeting):
             ).order_by('id')  # Use a consistent order for deterministic deletion
 
             # Delete the required number
+            log(f"sync_reg: deleting {tickets_to_delete} of {ticket_type[0]}:{ticket_type[1]} of {reg_data['email']}")
             for ticket in matching_tickets[:tickets_to_delete]:
                 ticket.delete()
             tickets_modified = True
@@ -1545,6 +1549,7 @@ def process_single_registration(reg_data, meeting):
             tickets_to_add = new_count - existing_count
 
             # Create the new tickets
+            log(f"sync_reg: adding {tickets_to_add} of {ticket_type[0]}:{ticket_type[1]} of {reg_data['email']}")
             for _ in range(tickets_to_add):
                 try:
                     RegistrationTicket.objects.create(
@@ -1555,7 +1560,6 @@ def process_single_registration(reg_data, meeting):
                     tickets_modified = True
                 except IntegrityError as e:
                     log(f"Error adding RegistrationTicket {e}")
-
     # handle nomcom volunteer
     if reg_data['is_nomcom_volunteer'] and person:
         try:
@@ -1574,6 +1578,7 @@ def process_single_registration(reg_data, meeting):
 
     # set action_taken
     if created:
+        log(f"sync_reg: created record. {reg_data['email']}")
         action_taken = 'created'
     elif fields_updated or tickets_modified:
         action_taken = 'updated'

From e89302c5937c49ddc98afcb82f3c51c8c0777fdb Mon Sep 17 00:00:00 2001
From: Eric Vyncke <eric@vyncke.org>
Date: Sat, 19 Jul 2025 14:10:25 +0200
Subject: [PATCH 389/656] feat: Add "IESG Statements" and "IESG Appeals" as
 menu items to Document menu (#9174)

* Add IESG nested drop-down menu

* Remove aria id to please Python tests

* More suitable menu ordering
---
 ietf/templates/base/menu.html | 52 ++++++++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 16 deletions(-)

diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html
index 81f012a679..695445c578 100644
--- a/ietf/templates/base/menu.html
+++ b/ietf/templates/base/menu.html
@@ -99,35 +99,55 @@
             Submit an Internet-Draft
         </a>
     </li>
+    {% if user and user.is_authenticated %}
     <li>
         <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
-           href="{% url 'ietf.doc.views_search.ad_workload' %}">
-            IESG dashboard
+            href="{% url "ietf.community.views.view_list" user.username %}">
+            My tracked docs
         </a>
     </li>
-    {% if user and user.is_authenticated %}
+    {% if user|has_role:"Area Director,Secretariat" %}
+        {% if flavor == 'top' %}<li><hr class="dropdown-divider"></li>{% endif %}
         <li>
             <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
-               href="{% url "ietf.community.views.view_list" user.username %}">
-                My tracked docs
+                href="{% url 'ietf.doc.views_status_change.rfc_status_changes' %}">
+                RFC status changes
             </a>
         </li>
-        {% if user|has_role:"Area Director,Secretariat" %}
-            {% if flavor == 'top' %}<li><hr class="dropdown-divider"></li>{% endif %}
+    {% endif %}
+    {% if flavor == 'top' %}<li><hr class="dropdown-divider"></li>{% endif %}
+    <li class="dropend">
+      <a class="dropdown-item dropdown-toggle" href="#">
+        IESG
+      </a>
+      <ul class="dropdown-menu">
             <li>
                 <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
-                   href="{% url 'ietf.doc.views_status_change.rfc_status_changes' %}">
-                    RFC status changes
+                href="{% url 'ietf.group.views.appeals' acronym='iesg' %}">
+                    IESG appeals
                 </a>
             </li>
-        {% endif %}
-        {% if flavor == 'top' %}<li><hr class="dropdown-divider"></li>{% endif %}
-        <li>
-            <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
-               href="{% url 'ietf.doc.views_ballot.irsg_ballot_status' %}">
-                IRSG ballot status
-            </a>
+            <li>
+                <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                href="{% url 'ietf.doc.views_search.ad_workload' %}">
+                    IESG dashboard
+                </a>
+            </li>
+            <li>
+                <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                href="{% url 'ietf.group.views.statements' acronym='iesg' %}">
+                    IESG statements
+                </a>
+            </li>
+        </ul>
         </li>
+        <li><hr class="dropdown-divider"></li>
+    <li>
+        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+            href="{% url 'ietf.doc.views_ballot.irsg_ballot_status' %}">
+            IRSG ballot status
+        </a>
+    </li>
         {% if user|has_role:"WG Chair,RG Chair" %}
             {% if flavor == 'top' %}<li><hr class="dropdown-divider"></li>{% endif %}
             <li {% if flavor == 'top' %}class="dropdown-header"{% else %}class="nav-item fw-bolder"{% endif %}>Manage</li>

From 2c8a7cda366d34d8eb1e0ee105d2b5c7b43a8cf3 Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Sat, 19 Jul 2025 15:32:57 +0200
Subject: [PATCH 390/656] fix: add tickets to Registration admin (#9177)

---
 ietf/meeting/admin.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ietf/meeting/admin.py b/ietf/meeting/admin.py
index 5d6adb7294..1c5d5c67b5 100644
--- a/ietf/meeting/admin.py
+++ b/ietf/meeting/admin.py
@@ -227,6 +227,10 @@ def queryset(self, request, queryset):
         if self.value():
             return queryset.filter(meeting__id=self.value())
         return queryset
+
+class RegistrationTicketInline(admin.TabularInline):
+    model = RegistrationTicket
+
 class RegistrationAdmin(admin.ModelAdmin):
     model = Registration
     # list_filter = [('meeting', Meeting.objects.filter(type='ietf')), ]
@@ -234,6 +238,7 @@ class RegistrationAdmin(admin.ModelAdmin):
     list_display = ['meeting', 'first_name', 'last_name', 'affiliation', 'country_code', 'person', 'email', ]
     search_fields = ['meeting__number', 'first_name', 'last_name', 'affiliation', 'country_code', 'email', ]
     raw_id_fields = ['person']
+    inlines = [RegistrationTicketInline, ]
 admin.site.register(Registration, RegistrationAdmin)
 
 class RegistrationTicketAdmin(admin.ModelAdmin):

From c78e360b4215988f1c462d36c8f0e3d9b77cec29 Mon Sep 17 00:00:00 2001
From: Eric Vyncke <eric@vyncke.org>
Date: Sat, 19 Jul 2025 15:42:23 +0200
Subject: [PATCH 391/656] Add a idnits-v3 button displayed only to IESG members
 (test phase) (#9175)

---
 ietf/settings.py                       |  1 +
 ietf/templates/doc/document_draft.html | 11 +++++++++++
 2 files changed, 12 insertions(+)

diff --git a/ietf/settings.py b/ietf/settings.py
index 5e33673611..ada98bd627 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -658,6 +658,7 @@ def skip_unreadable_post(record):
 IDTRACKER_BASE_URL = "https://datatracker.ietf.org"
 RFCDIFF_BASE_URL = "https://author-tools.ietf.org/iddiff"
 IDNITS_BASE_URL = "https://author-tools.ietf.org/api/idnits"
+IDNITS3_BASE_URL = "https://author-tools.ietf.org/idnits3/results"
 IDNITS_SERVICE_URL = "https://author-tools.ietf.org/idnits"
 
 # Content security policy configuration (django-csp)
diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html
index d5ea074422..6414538283 100644
--- a/ietf/templates/doc/document_draft.html
+++ b/ietf/templates/doc/document_draft.html
@@ -661,6 +661,17 @@
                 </i>
                 Nits
             </a>
+            {% if user|has_role:"Area Director" %}
+            {# IDNITS3 is an experimental service, so only show it to Area Directors #}
+             <a class="btn btn-primary btn-sm"
+               href="{{ settings.IDNITS3_BASE_URL }}?url=https://www.ietf.org/archive/id/{{ doc.filename_with_rev }}"
+               rel="nofollow"
+               target="_blank">
+                <i class="bi bi-exclamation-diamond">
+                </i>
+                Nits-v3 (Experimental)
+            </a>           
+            {% endif %}
             <a class="btn btn-primary btn-sm"
                href="https://mailarchive.ietf.org/arch/search/?q=%22{{ doc.name }}%22"
                rel="nofollow"

From 3d50a211fa6a98ad3bffdb973043d5152f77d89a Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Sat, 19 Jul 2025 16:01:52 +0200
Subject: [PATCH 392/656] refactor: remove old reg API functions (#9176)

---
 ietf/api/tests.py                             | 126 -----------------
 ietf/api/urls.py                              |   1 -
 ietf/api/views.py                             |  95 -------------
 ietf/meeting/tasks.py                         |  18 ---
 ietf/meeting/tests_utils.py                   |  89 +-----------
 ietf/meeting/utils.py                         | 112 ---------------
 ietf/settings.py                              |   1 -
 .../commands/fetch_meeting_attendance.py      |  42 ------
 ietf/stats/tasks.py                           |  27 ----
 ietf/stats/tests.py                           | 124 -----------------
 ietf/stats/utils.py                           | 127 +-----------------
 11 files changed, 3 insertions(+), 759 deletions(-)
 delete mode 100644 ietf/stats/management/commands/fetch_meeting_attendance.py
 delete mode 100644 ietf/stats/tasks.py

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index abaf9f5ed2..93a2195467 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -37,7 +37,6 @@
 from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year
 from ietf.person.factories import PersonFactory, random_faker, EmailFactory, PersonalApiKeyFactory
 from ietf.person.models import Email, User
-from ietf.stats.models import MeetingRegistration
 from ietf.utils.mail import empty_outbox, outbox, get_payload_text
 from ietf.utils.models import DumpInfo
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, reload_db_objects
@@ -705,131 +704,6 @@ def test_api_v2_person_export_view(self):
         self.assertEqual(data['ascii'], robot.ascii)
         self.assertEqual(data['user']['email'], robot.user.email)
 
-    def test_api_new_meeting_registration(self):
-        meeting = MeetingFactory(type_id='ietf')
-        reg = {
-            'apikey': 'invalid',
-            'affiliation': "Alguma Corporação",
-            'country_code': 'PT',
-            'email': 'foo@example.pt',
-            'first_name': 'Foo',
-            'last_name': 'Bar',
-            'meeting': meeting.number,
-            'reg_type': 'hackathon',
-            'ticket_type': '',
-            'checkedin': 'False',
-            'is_nomcom_volunteer': 'False',
-        }
-        url = urlreverse('ietf.api.views.api_new_meeting_registration')
-        r = self.client.post(url, reg)
-        self.assertContains(r, 'Invalid apikey', status_code=403)
-        oidcp = PersonFactory(user__is_staff=True)
-        # Make sure 'oidcp' has an acceptable role
-        RoleFactory(name_id='robot', person=oidcp, email=oidcp.email(), group__acronym='secretariat')
-        key = PersonalApiKeyFactory(person=oidcp, endpoint=url)
-        reg['apikey'] = key.hash()
-        #
-        # Test valid POST
-        # FIXME: sometimes, there seems to be something in the outbox?
-        old_len = len(outbox)
-        r = self.client.post(url, reg)
-        self.assertContains(r, "Accepted, New registration, Email sent", status_code=202)
-        #
-        # Check outgoing mail
-        self.assertEqual(len(outbox), old_len + 1)
-        body = get_payload_text(outbox[-1])
-        self.assertIn(reg['email'], outbox[-1]['To'] )
-        self.assertIn(reg['email'], body)
-        self.assertIn('account creation request', body)
-        #
-        # Check record
-        obj = MeetingRegistration.objects.get(email=reg['email'], meeting__number=reg['meeting'])
-        for key in ['affiliation', 'country_code', 'first_name', 'last_name', 'person', 'reg_type', 'ticket_type', 'checkedin']:
-            self.assertEqual(getattr(obj, key), False if key=='checkedin' else reg.get(key) , "Bad data for field '%s'" % key)
-        #
-        # Test with existing user
-        person = PersonFactory()
-        reg['email'] = person.email().address
-        reg['first_name'] = person.first_name()
-        reg['last_name'] = person.last_name()
-        #
-        r = self.client.post(url, reg)
-        self.assertContains(r, "Accepted, New registration", status_code=202)
-        #
-        # There should be no new outgoing mail
-        self.assertEqual(len(outbox), old_len + 1)
-        #
-        # Test multiple reg types
-        reg['reg_type'] = 'remote'
-        reg['ticket_type'] = 'full_week_pass'
-        r = self.client.post(url, reg)
-        self.assertContains(r, "Accepted, New registration", status_code=202)
-        objs = MeetingRegistration.objects.filter(email=reg['email'], meeting__number=reg['meeting'])
-        self.assertEqual(len(objs), 2)
-        self.assertEqual(objs.filter(reg_type='hackathon').count(), 1)
-        self.assertEqual(objs.filter(reg_type='remote', ticket_type='full_week_pass').count(), 1)
-        self.assertEqual(len(outbox), old_len + 1)
-        #
-        # Test incomplete POST
-        drop_fields = ['affiliation', 'first_name', 'reg_type']
-        for field in drop_fields:
-            del reg[field]
-        r = self.client.post(url, reg)        
-        self.assertContains(r, 'Missing parameters:', status_code=400)
-        err, fields = r.content.decode().split(':', 1)
-        missing_fields = [f.strip() for f in fields.split(',')]
-        self.assertEqual(set(missing_fields), set(drop_fields))
-
-    def test_api_new_meeting_registration_nomcom_volunteer(self):
-        '''Test that Volunteer is created if is_nomcom_volunteer=True
-           is submitted to API
-        '''
-        meeting = MeetingFactory(type_id='ietf')
-        reg = {
-            'apikey': 'invalid',
-            'affiliation': "Alguma Corporação",
-            'country_code': 'PT',
-            'meeting': meeting.number,
-            'reg_type': 'onsite',
-            'ticket_type': '',
-            'checkedin': 'False',
-            'is_nomcom_volunteer': 'False',
-        }
-        person = PersonFactory()
-        reg['email'] = person.email().address
-        reg['first_name'] = person.first_name()
-        reg['last_name'] = person.last_name()
-        now = datetime.datetime.now()
-        if now.month > 10:
-            year = now.year + 1
-        else:
-            year = now.year
-        # create appropriate group and nomcom objects
-        nomcom = NomComFactory.create(is_accepting_volunteers=True, **nomcom_kwargs_for_year(year))
-        url = urlreverse('ietf.api.views.api_new_meeting_registration')
-        oidcp = PersonFactory(user__is_staff=True)
-        # Make sure 'oidcp' has an acceptable role
-        RoleFactory(name_id='robot', person=oidcp, email=oidcp.email(), group__acronym='secretariat')
-        key = PersonalApiKeyFactory(person=oidcp, endpoint=url)
-        reg['apikey'] = key.hash()
-
-        # first test is_nomcom_volunteer False
-        r = self.client.post(url, reg)
-        self.assertContains(r, "Accepted, New registration", status_code=202)
-        # assert no Volunteers exists
-        self.assertEqual(Volunteer.objects.count(), 0)
-
-        # test is_nomcom_volunteer True
-        reg['is_nomcom_volunteer'] = 'True'
-        r = self.client.post(url, reg)
-        self.assertContains(r, "Accepted, Updated registration", status_code=202)
-        # assert Volunteer exists
-        self.assertEqual(Volunteer.objects.count(), 1)
-        volunteer = Volunteer.objects.last()
-        self.assertEqual(volunteer.person, person)
-        self.assertEqual(volunteer.nomcom, nomcom)
-        self.assertEqual(volunteer.origin, 'registration')
-
     @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]})
     def test_api_new_meeting_registration_v2(self):
         meeting = MeetingFactory(type_id='ietf')
diff --git a/ietf/api/urls.py b/ietf/api/urls.py
index bafd5c5b76..6f2efb3c1e 100644
--- a/ietf/api/urls.py
+++ b/ietf/api/urls.py
@@ -67,7 +67,6 @@
     url(r'^notify/session/polls/?$', meeting_views.api_upload_polls),    
     # Let the registration system notify us about registrations
     url(r'^notify/meeting/registration/v2/?', api_views.api_new_meeting_registration_v2),
-    url(r'^notify/meeting/registration/?', api_views.api_new_meeting_registration),
     # OpenID authentication provider
     url(r'^openid/$', TemplateView.as_view(template_name='api/openid-issuer.html'), name='ietf.api.urls.oidc_issuer'),
     url(r'^openid/', include('oidc_provider.urls', namespace='oidc_provider')),
diff --git a/ietf/api/views.py b/ietf/api/views.py
index e8e38b25b4..b4dd7f05d6 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -16,8 +16,6 @@
 from django.contrib.auth import authenticate
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.models import User
-from django.core.exceptions import ValidationError
-from django.core.validators import validate_email
 from django.http import HttpResponse, Http404, JsonResponse, HttpResponseBadRequest
 from django.shortcuts import render, get_object_or_404
 from django.urls import reverse
@@ -43,14 +41,11 @@
 from ietf.doc.utils import DraftAliasGenerator, fuzzy_find_documents
 from ietf.group.utils import GroupAliasGenerator, role_holder_emails
 from ietf.ietfauth.utils import role_required
-from ietf.ietfauth.views import send_account_creation_email
 from ietf.ipr.utils import ingest_response_email as ipr_ingest_response_email
 from ietf.meeting.models import Meeting
 from ietf.meeting.utils import import_registration_json_validator, process_single_registration
-from ietf.nomcom.models import Volunteer, NomCom
 from ietf.nomcom.utils import ingest_feedback_email as nomcom_ingest_feedback_email
 from ietf.person.models import Person, Email
-from ietf.stats.models import MeetingRegistration
 from ietf.sync.iana import ingest_review_email as iana_ingest_review_email
 from ietf.utils import log
 from ietf.utils.decorators import require_api_key
@@ -151,96 +146,6 @@ def post(self, request):
 #     else:
 #         return HttpResponse(status=405)
 
-@require_api_key
-@role_required('Robot')
-@csrf_exempt
-def api_new_meeting_registration(request):
-    '''REST API to notify the datatracker about a new meeting registration'''
-    def err(code, text):
-        return HttpResponse(text, status=code, content_type='text/plain')
-    required_fields = [ 'meeting', 'first_name', 'last_name', 'affiliation', 'country_code',
-                        'email', 'reg_type', 'ticket_type', 'checkedin', 'is_nomcom_volunteer']
-    fields = required_fields + []
-    if request.method == 'POST':
-        # parameters:
-        #   apikey:
-        #   meeting
-        #   name
-        #   email
-        #   reg_type (In Person, Remote, Hackathon Only)
-        #   ticket_type (full_week, one_day, student)
-        #   
-        data = {'attended': False, }
-        missing_fields = []
-        for item in fields:
-            value = request.POST.get(item, None)
-            if value is None and item in required_fields:
-                missing_fields.append(item)
-            data[item] = value
-        if missing_fields:
-            return err(400, "Missing parameters: %s" % ', '.join(missing_fields))
-        number = data['meeting']
-        try:
-            meeting = Meeting.objects.get(number=number)
-        except Meeting.DoesNotExist:
-            return err(400, "Invalid meeting value: '%s'" % (number, ))
-        reg_type = data['reg_type']
-        email = data['email']
-        try:
-            validate_email(email)
-        except ValidationError:
-            return err(400, "Invalid email value: '%s'" % (email, ))
-        if request.POST.get('cancelled', 'false') == 'true':
-            MeetingRegistration.objects.filter(
-                meeting_id=meeting.pk,
-                email=email,
-                reg_type=reg_type).delete()
-            return HttpResponse('OK', status=200, content_type='text/plain')
-        else:
-            object, created = MeetingRegistration.objects.get_or_create(
-                meeting_id=meeting.pk,
-                email=email,
-                reg_type=reg_type)
-            try:
-                # Update attributes
-                for key in set(data.keys())-set(['attended', 'apikey', 'meeting', 'email']):
-                    if key == 'checkedin':
-                        new = bool(data.get(key).lower() == 'true')
-                    else:
-                        new = data.get(key)
-                    setattr(object, key, new)
-                person = Person.objects.filter(email__address=email)
-                if person.exists():
-                    object.person = person.first()
-                object.save()
-            except ValueError as e:
-                return err(400, "Unexpected POST data: %s" % e)
-            response = "Accepted, New registration" if created else "Accepted, Updated registration"
-            if User.objects.filter(username__iexact=email).exists() or Email.objects.filter(address=email).exists():
-                pass
-            else:
-                send_account_creation_email(request, email)
-                response += ", Email sent"
-
-            # handle nomcom volunteer
-            if request.POST.get('is_nomcom_volunteer', 'false').lower() == 'true' and object.person:
-                try:
-                    nomcom = NomCom.objects.get(is_accepting_volunteers=True)
-                except (NomCom.DoesNotExist, NomCom.MultipleObjectsReturned):
-                    nomcom = None
-                if nomcom:
-                    Volunteer.objects.get_or_create(
-                        nomcom=nomcom,
-                        person=object.person,
-                        defaults={
-                            "affiliation": data["affiliation"],
-                            "origin": "registration"
-                        }
-                    )
-            return HttpResponse(response, status=202, content_type='text/plain')
-    else:
-        return HttpResponse(status=405)
-
 
 @requires_api_token
 @csrf_exempt
diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py
index dc3fbc99ec..784eb00d87 100644
--- a/ietf/meeting/tasks.py
+++ b/ietf/meeting/tasks.py
@@ -9,7 +9,6 @@
 from .models import Meeting
 from .utils import generate_proceedings_content
 from .views import generate_agenda_data
-from .utils import migrate_registrations, check_migrate_registrations
 from .utils import fetch_attendance_from_meetings
 
 
@@ -18,23 +17,6 @@ def agenda_data_refresh():
     generate_agenda_data(force_refresh=True)
 
 
-@shared_task
-def migrate_registrations_task(initial=False):
-    """ Migrate ietf.stats.MeetingRegistration to ietf.meeting.Registration
-        If initial is True, migrate all meetings otherwise only future meetings.
-        This function is idempotent. It can be run regularly from cron.
-    """
-    migrate_registrations(initial=initial)
-
-
-@shared_task
-def check_migrate_registrations_task():
-    """ Compare MeetingRegistration with Registration to ensure
-        all records migrated
-    """
-    check_migrate_registrations()
-
-
 @shared_task
 def proceedings_content_refresh_task(*, all=False):
     """Refresh meeting proceedings cache
diff --git a/ietf/meeting/tests_utils.py b/ietf/meeting/tests_utils.py
index 8d912158ce..11ea63df4f 100644
--- a/ietf/meeting/tests_utils.py
+++ b/ietf/meeting/tests_utils.py
@@ -12,101 +12,14 @@
 from django.http import HttpResponse, JsonResponse
 from ietf.meeting.factories import MeetingFactory, RegistrationFactory, RegistrationTicketFactory
 from ietf.meeting.models import Registration
-from ietf.meeting.utils import (migrate_registrations, get_preferred, process_single_registration,
+from ietf.meeting.utils import (process_single_registration,
     get_registration_data, sync_registration_data, fetch_attendance_from_meetings)
 from ietf.nomcom.models import Volunteer
 from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year
 from ietf.person.factories import PersonFactory
-from ietf.stats.factories import MeetingRegistrationFactory
 from ietf.utils.test_utils import TestCase
 
 
-class MigrateRegistrationsTests(TestCase):
-    def test_new_meeting_registration(self):
-        meeting = MeetingFactory(type_id='ietf', number='109')
-        reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
-        self.assertEqual(Registration.objects.count(), 0)
-        migrate_registrations(initial=True)
-        self.assertEqual(Registration.objects.count(), 1)
-        new = Registration.objects.first()
-        self.assertEqual(new.first_name, reg.first_name)
-        self.assertEqual(new.last_name, reg.last_name)
-        self.assertEqual(new.email, reg.email)
-        self.assertEqual(new.person, reg.person)
-        self.assertEqual(new.meeting, meeting)
-        self.assertEqual(new.affiliation, reg.affiliation)
-        self.assertEqual(new.country_code, reg.country_code)
-        self.assertEqual(new.checkedin, reg.checkedin)
-        self.assertEqual(new.attended, reg.attended)
-
-    def test_migrate_non_initial(self):
-        # with only old meeting
-        meeting = MeetingFactory(type_id='ietf', number='109')
-        MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
-        self.assertEqual(Registration.objects.count(), 0)
-        migrate_registrations()
-        self.assertEqual(Registration.objects.count(), 0)
-        # with new meeting
-        new_meeting = MeetingFactory(type_id='ietf', number='150')
-        new_meeting.date = datetime.date.today() + datetime.timedelta(days=30)
-        new_meeting.save()
-        MeetingRegistrationFactory(meeting=new_meeting, reg_type='onsite', ticket_type='week_pass')
-        migrate_registrations()
-        self.assertEqual(Registration.objects.count(), 1)
-
-    def test_updated_meeting_registration(self):
-        # setup test initial conditions
-        meeting = MeetingFactory(type_id='ietf', number='109')
-        reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
-        migrate_registrations(initial=True)
-        # change first_name and save
-        original = reg.first_name
-        reg.first_name = 'NewBob'
-        reg.save()
-        new = Registration.objects.first()
-        self.assertEqual(new.first_name, original)
-        migrate_registrations(initial=True)
-        new.refresh_from_db()
-        self.assertEqual(new.first_name, reg.first_name)
-
-    def test_additional_ticket(self):
-        # setup test initial conditions
-        meeting = MeetingFactory(type_id='ietf', number='109')
-        reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
-        migrate_registrations(initial=True)
-        new = Registration.objects.first()
-        self.assertEqual(new.tickets.count(), 1)
-        # add a second ticket
-        reg.reg_type = 'remote'
-        reg.pk = None
-        reg.save()
-        migrate_registrations(initial=True)
-        # new.refresh_from_db()
-        self.assertEqual(new.tickets.count(), 2)
-
-    def test_cancelled_registration(self):
-        # setup test initial conditions
-        meeting = MeetingFactory(type_id='ietf', number='109')
-        reg = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
-        migrate_registrations(initial=True)
-        reg.delete()
-        # do test
-        migrate_registrations(initial=True)
-        self.assertEqual(Registration.objects.count(), 0)
-
-    def test_get_preferred(self):
-        meeting = MeetingFactory(type_id='ietf', number='109')
-        onsite = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', ticket_type='week_pass')
-        remote = MeetingRegistrationFactory(meeting=meeting, reg_type='remote', ticket_type='week_pass')
-        hackathon = MeetingRegistrationFactory(meeting=meeting, reg_type='hackathon_onsite', ticket_type='week_pass')
-        result = get_preferred([remote, onsite, hackathon])
-        self.assertEqual(result, onsite)
-        result = get_preferred([hackathon, remote])
-        self.assertEqual(result, remote)
-        result = get_preferred([hackathon])
-        self.assertEqual(result, hackathon)
-
-
 class JsonResponseWithJson(JsonResponse):
     def json(self):
         return json.loads(self.content)
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index ee1a3b46d7..1047783a0c 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -37,7 +37,6 @@
 from ietf.group.utils import can_manage_materials
 from ietf.name.models import SessionStatusName, ConstraintName, DocTypeName
 from ietf.person.models import Person
-from ietf.stats.models import MeetingRegistration
 from ietf.utils.html import clean_html
 from ietf.utils.log import log
 from ietf.utils.timezone import date_today
@@ -1017,117 +1016,6 @@ def participants_for_meeting(meeting):
     return (checked_in, attended)
 
 
-def get_preferred(regs):
-    """ If there are multiple registrations return preferred in
-        this order: onsite, remote, any (ie hackathon_onsite)
-    """
-    if len(regs) == 1:
-        return regs[0]
-    reg_types = [r.reg_type for r in regs]
-    if 'onsite' in reg_types:
-        return regs[reg_types.index('onsite')]
-    elif 'remote' in reg_types:
-        return regs[reg_types.index('remote')]
-    else:
-        return regs[0]
-
-
-def migrate_registrations(initial=False):
-    """ Migrate ietf.stats.MeetingRegistration to ietf.meeting.Registration
-        If initial is True, migrate all meetings otherwise only future meetings.
-        This function is idempotent. It can be run regularly from cron.
-    """
-    if initial:
-        meetings = Meeting.objects.filter(type='ietf')
-        MeetingRegistration.objects.filter(reg_type='hackathon').update(reg_type='hackathon_remote')
-        MeetingRegistration.objects.filter(ticket_type='full_week_pass').update(ticket_type='week_pass')
-        MeetingRegistration.objects.filter(pk=49645).update(ticket_type='one_day')
-        MeetingRegistration.objects.filter(pk=50804).update(ticket_type='week_pass')
-        MeetingRegistration.objects.filter(pk=42386).update(ticket_type='week_pass')
-        MeetingRegistration.objects.filter(pk=42782).update(ticket_type='one_day')
-        MeetingRegistration.objects.filter(pk=43464).update(ticket_type='week_pass')
-    else:
-        # still process records during week of meeting
-        one_week_ago = datetime.date.today() - datetime.timedelta(days=7)
-        meetings = Meeting.objects.filter(type='ietf', date__gt=one_week_ago)
-
-    for meeting in meetings:
-        # gather all MeetingRegistrations by person (email)
-        emails = {}
-        for meeting_reg in MeetingRegistration.objects.filter(meeting=meeting):
-            if meeting_reg.email in emails:
-                emails[meeting_reg.email].append(meeting_reg)
-            else:
-                emails[meeting_reg.email] = [meeting_reg]
-        # process each person's registrations
-        for email, meeting_regs in emails.items():
-            preferred_reg = get_preferred(meeting_regs)
-            reg, created = Registration.objects.get_or_create(
-                meeting=meeting,
-                email=email,
-                defaults={
-                    'first_name': preferred_reg.first_name,
-                    'last_name': preferred_reg.last_name,
-                    'affiliation': preferred_reg.affiliation,
-                    'country_code': preferred_reg.country_code,
-                    'person': preferred_reg.person,
-                    'attended': preferred_reg.attended,
-                    'checkedin': preferred_reg.checkedin,
-                }
-            )
-            if created:
-                for meeting_reg in meeting_regs:
-                    reg.tickets.create(
-                        attendance_type_id=meeting_reg.reg_type or 'unknown',
-                        ticket_type_id=meeting_reg.ticket_type or 'unknown',
-                    )
-            else:
-                # check if tickets differ
-                reg_tuple_list = [(t.attendance_type_id, t.ticket_type_id) for t in reg.tickets.all()]
-                meeting_reg_tuple_list = [(mr.reg_type or 'unknown', mr.ticket_type or 'unknown') for mr in meeting_regs]
-                if not set(reg_tuple_list) == set(meeting_reg_tuple_list):
-                    # update tickets
-                    reg.tickets.all().delete()
-                    for meeting_reg in meeting_regs:
-                        reg.tickets.create(
-                            attendance_type_id=meeting_reg.reg_type or 'unknown',
-                            ticket_type_id=meeting_reg.ticket_type or 'unknown',
-                        )
-                # check fields for updates
-                fields_to_check = [
-                    'first_name', 'last_name', 'affiliation', 'country_code',
-                    'attended', 'checkedin'
-                ]
-
-                changed = False
-                for field in fields_to_check:
-                    new_value = getattr(preferred_reg, field)
-                    if getattr(reg, field) != new_value:
-                        setattr(reg, field, new_value)
-                        changed = True
-
-                if changed:
-                    reg.save()
-        # delete cancelled Registrations
-        meeting_reg_email_set = set(emails.keys())
-        reg_email_set = set(Registration.objects.filter(meeting=meeting).values_list('email', flat=True))
-        for email in reg_email_set - meeting_reg_email_set:
-            Registration.objects.filter(meeting=meeting, email=email).delete()
-
-    return
-
-
-def check_migrate_registrations():
-    """A simple utility function to test that all MeetingRegistration
-    records got migrated
-    """
-    for mr in MeetingRegistration.objects.all():
-        reg = Registration.objects.get(meeting=mr.meeting, email=mr.email)
-        assert reg.tickets.filter(
-            attendance_type__slug=mr.reg_type or 'unknown',
-            ticket_type__slug=mr.ticket_type or 'unknown').exists()
-
-
 def generate_proceedings_content(meeting, force_refresh=False):
     """Render proceedings content for a meeting and update cache
     
diff --git a/ietf/settings.py b/ietf/settings.py
index ada98bd627..d76a8749b2 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -1142,7 +1142,6 @@ def skip_unreadable_post(record):
     "--outdir"
 ]
 
-STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/'
 REGISTRATION_PARTICIPANTS_API_URL = 'https://registration.ietf.org/api/v1/participants-dt/'
 REGISTRATION_PARTICIPANTS_API_KEY = 'changeme'
 
diff --git a/ietf/stats/management/commands/fetch_meeting_attendance.py b/ietf/stats/management/commands/fetch_meeting_attendance.py
deleted file mode 100644
index e17ae567fa..0000000000
--- a/ietf/stats/management/commands/fetch_meeting_attendance.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright The IETF Trust 2017-2019, All Rights Reserved
-# Copyright 2016 IETF Trust
-
-from django.core.management.base import BaseCommand, CommandError
-from django.utils import timezone
-
-import debug                            # pyflakes:ignore
-
-from ietf.meeting.models import Meeting
-from ietf.stats.utils import fetch_attendance_from_meetings
-from ietf.utils import log
-
-
-class Command(BaseCommand):
-    help = "Fetch meeting attendee figures from ietf.org/registration/attendees."
-
-    def add_arguments(self, parser):
-        parser.add_argument("--meeting", help="meeting to fetch data for")
-        parser.add_argument("--all", action="store_true", help="fetch data for all meetings")
-        parser.add_argument("--latest", type=int, help="fetch data for latest N meetings")
-
-    def handle(self, *args, **options):
-        self.verbosity = options['verbosity']
-
-        meetings = Meeting.objects.none()
-        if options['meeting']:
-            meetings = Meeting.objects.filter(number=options['meeting'], type="ietf")
-        elif options['all']:
-            meetings = Meeting.objects.filter(type="ietf").order_by("date")
-        elif options['latest']:
-            meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by("-date")[:options['latest']]
-        else:
-            raise CommandError("Please use one of --meeting, --all or --latest")
-
-        for meeting, stats in zip(meetings, fetch_attendance_from_meetings(meetings)):
-            msg = "Fetched data for meeting {:>3}: {:4d} processed, {:4d} added, {:4d} in table".format(
-                meeting.number, stats.processed, stats.added, stats.total
-            )
-            if self.stdout.isatty():
-                self.stdout.write(msg+'\n') # make debugging a bit easier
-            else:
-                log.log(msg)
diff --git a/ietf/stats/tasks.py b/ietf/stats/tasks.py
deleted file mode 100644
index 808e797a40..0000000000
--- a/ietf/stats/tasks.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright The IETF Trust 2024, All Rights Reserved
-#
-# Celery task definitions
-#
-from celery import shared_task
-from django.utils import timezone
-
-from ietf.meeting.models import Meeting
-from ietf.stats.utils import fetch_attendance_from_meetings
-from ietf.utils import log
-
-
-@shared_task
-def fetch_meeting_attendance_task():
-    # fetch most recent two meetings
-    meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by("-date")[:2]
-    try:
-        stats = fetch_attendance_from_meetings(meetings)
-    except RuntimeError as err:
-        log.log(f"Error in fetch_meeting_attendance_task: {err}")
-    else:
-        for meeting, meeting_stats in zip(meetings, stats):
-            log.log(
-                "Fetched data for meeting {:>3}: {:4d} processed, {:4d} added, {:4d} in table".format(
-                    meeting.number, meeting_stats.processed, meeting_stats.added, meeting_stats.total
-                )
-            )
diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py
index 47027277be..48552c8fba 100644
--- a/ietf/stats/tests.py
+++ b/ietf/stats/tests.py
@@ -3,12 +3,9 @@
 
 
 import calendar
-import datetime
 import json
 
-from mock import patch
 from pyquery import PyQuery
-from requests import Response
 
 import debug    # pyflakes:ignore
 
@@ -19,12 +16,8 @@
 
 
 from ietf.group.factories import RoleFactory
-from ietf.meeting.factories import MeetingFactory
 from ietf.person.factories import PersonFactory
 from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory
-from ietf.stats.models import MeetingRegistration
-from ietf.stats.tasks import fetch_meeting_attendance_task
-from ietf.stats.utils import get_meeting_registration_data, FetchStats, fetch_attendance_from_meetings
 from ietf.utils.timezone import date_today
 
 
@@ -116,120 +109,3 @@ def test_review_stats(self):
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertTrue(q('.review-stats td:contains("1")'))
-
-    @patch('requests.get')
-    def test_get_meeting_registration_data(self, mock_get):
-        '''Test function to get reg data.  Confirm leading/trailing spaces stripped'''
-        person = PersonFactory()
-        data = {
-            'LastName': person.last_name() + ' ',
-            'FirstName': person.first_name(),
-            'Company': 'ABC',
-            'Country': 'US',
-            'Email': person.email().address,
-            'RegType': 'onsite',
-            'TicketType': 'week_pass',
-            'CheckedIn': 'True',
-        }
-        data2 = data.copy()
-        data2['RegType'] = 'hackathon'
-        response_a = Response()
-        response_a.status_code = 200
-        response_a._content = json.dumps([data, data2]).encode('utf8')
-        # second response one less record, it's been deleted
-        response_b = Response()
-        response_b.status_code = 200
-        response_b._content = json.dumps([data]).encode('utf8')
-        # mock_get.return_value = response
-        mock_get.side_effect = [response_a, response_b]
-        meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016, 7, 14), number="96")
-        get_meeting_registration_data(meeting)
-        query = MeetingRegistration.objects.filter(
-            first_name=person.first_name(),
-            last_name=person.last_name(),
-            country_code='US')
-        self.assertEqual(query.count(), 2)
-        self.assertEqual(query.filter(reg_type='onsite').count(), 1)
-        self.assertEqual(query.filter(reg_type='hackathon').count(), 1)
-        onsite = query.get(reg_type='onsite')
-        self.assertEqual(onsite.ticket_type, 'week_pass')
-        self.assertEqual(onsite.checkedin, True)
-        # call a second time to test delete
-        get_meeting_registration_data(meeting)
-        query = MeetingRegistration.objects.filter(meeting=meeting, email=person.email())
-        self.assertEqual(query.count(), 1)
-        self.assertEqual(query.filter(reg_type='onsite').count(), 1)
-        self.assertEqual(query.filter(reg_type='hackathon').count(), 0)
-
-    @patch('requests.get')
-    def test_get_meeting_registration_data_duplicates(self, mock_get):
-        '''Test that get_meeting_registration_data does not create duplicate
-           MeetingRegistration records
-        '''
-        person = PersonFactory()
-        data = {
-            'LastName': person.last_name() + ' ',
-            'FirstName': person.first_name(),
-            'Company': 'ABC',
-            'Country': 'US',
-            'Email': person.email().address,
-            'RegType': 'onsite',
-            'TicketType': 'week_pass',
-            'CheckedIn': 'True',
-        }
-        data2 = data.copy()
-        data2['RegType'] = 'hackathon'
-        response = Response()
-        response.status_code = 200
-        response._content = json.dumps([data, data2, data]).encode('utf8')
-        mock_get.return_value = response
-        meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016, 7, 14), number="96")
-        self.assertEqual(MeetingRegistration.objects.count(), 0)
-        get_meeting_registration_data(meeting)
-        query = MeetingRegistration.objects.all()
-        self.assertEqual(query.count(), 2)
-
-    @patch("ietf.stats.utils.get_meeting_registration_data")
-    def test_fetch_attendance_from_meetings(self, mock_get_mtg_reg_data):
-        mock_meetings = [object(), object(), object()]
-        mock_get_mtg_reg_data.side_effect = (
-            (1, 2, 3),
-            (4, 5, 6),
-            (7, 8, 9),
-        )
-        stats = fetch_attendance_from_meetings(mock_meetings)
-        self.assertEqual(
-            [mock_get_mtg_reg_data.call_args_list[n][0][0] for n in range(3)],
-            mock_meetings,
-        )
-        self.assertEqual(
-            stats,
-            [
-                FetchStats(1, 2, 3),
-                FetchStats(4, 5, 6),
-                FetchStats(7, 8, 9),
-            ]
-        )
-
-
-class TaskTests(TestCase):
-    @patch("ietf.stats.tasks.fetch_attendance_from_meetings")
-    def test_fetch_meeting_attendance_task(self, mock_fetch_attendance):
-        today = date_today()
-        meetings = [
-            MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=1)),
-            MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=2)),
-            MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=3)),
-        ]
-        mock_fetch_attendance.return_value = [FetchStats(1,2,3), FetchStats(1,2,3)]
-
-        fetch_meeting_attendance_task()
-        self.assertEqual(mock_fetch_attendance.call_count, 1)
-        self.assertCountEqual(mock_fetch_attendance.call_args[0][0], meetings[0:2])
-
-        # test handling of RuntimeError
-        mock_fetch_attendance.reset_mock()
-        mock_fetch_attendance.side_effect = RuntimeError
-        fetch_meeting_attendance_task()
-        self.assertTrue(mock_fetch_attendance.called)
-        # Good enough that we got here without raising an exception
diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py
index f2e1d9801d..a13e87a4f4 100644
--- a/ietf/stats/utils.py
+++ b/ietf/stats/utils.py
@@ -3,18 +3,12 @@
 
 
 import re
-import requests
-from collections import defaultdict, namedtuple
-
-from django.conf import settings
-from django.db.models import Q
+from collections import defaultdict
 
 import debug                            # pyflakes:ignore
 
-from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias, MeetingRegistration
+from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias
 from ietf.name.models import CountryName
-from ietf.person.models import Email
-from ietf.utils.log import log
 
 import logging
 logger = logging.getLogger('django')
@@ -221,120 +215,3 @@ def compute_hirsch_index(citation_counts):
         i += 1
 
     return i
-
-
-def get_meeting_registration_data(meeting):
-    """"Retrieve registration attendee data and summary statistics.  Returns number
-    of Registration records created.
-
-    MeetingRegistration records are created in realtime as people register for a
-    meeting. This function serves as an audit / reconciliation. Most records are
-    expected to already exist. The function has been optimized with this in mind.
-    """
-    num_created = 0
-    num_processed = 0
-    try:
-        response = requests.get(
-            settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
-            timeout=settings.DEFAULT_REQUESTS_TIMEOUT,
-        )
-    except requests.Timeout as exc:
-        log(f'GET request timed out for [{settings.STATS_REGISTRATION_ATTENDEES_JSON_URL}]: {exc}')
-        raise RuntimeError("Timeout retrieving data from registrations API") from exc
-    if response.status_code == 200:
-        decoded = []
-        try:
-            decoded = response.json()
-        except ValueError:
-            if response.content.strip() == 'Invalid meeting':
-                logger.info('Invalid meeting: {}'.format(meeting.number))
-                return (0,0,0)
-            else:
-                raise RuntimeError("Could not decode response from registrations API: '%s...'" % (response.content[:64], ))
-
-        records = MeetingRegistration.objects.filter(meeting_id=meeting.pk).select_related('person')
-        meeting_registrations = {(r.email, r.reg_type):r for r in records}
-        for registration in decoded:
-            person = None
-            # capture the stripped registration values for later use
-            first_name      = registration['FirstName'].strip()
-            last_name       = registration['LastName'].strip()
-            affiliation     = registration['Company'].strip()
-            country_code    = registration['Country'].strip()
-            address         = registration['Email'].strip()
-            reg_type        = registration['RegType'].strip()
-            ticket_type     = registration['TicketType'].strip()
-            checkedin       = bool(registration['CheckedIn'].strip().lower() == 'true')
-
-            if (address, reg_type) in meeting_registrations:
-                object = meeting_registrations.pop((address, reg_type))
-                created = False
-            else:
-                object, created = MeetingRegistration.objects.get_or_create(
-                    meeting_id=meeting.pk,
-                    email=address,
-                    reg_type=reg_type)
-            
-            if (object.first_name != first_name[:200] or
-                object.last_name != last_name[:200] or
-                object.affiliation != affiliation or
-                object.country_code != country_code or
-                object.ticket_type != ticket_type or
-                object.checkedin != checkedin):
-                    object.first_name=first_name[:200]
-                    object.last_name=last_name[:200]
-                    object.affiliation=affiliation
-                    object.country_code=country_code
-                    object.ticket_type=ticket_type
-                    object.checkedin=checkedin
-                    object.save()
-
-            # Add a Person object to MeetingRegistration object
-            # if valid email is available
-            if object and not object.person and address:
-                # If the person already exists do not try to create a new one
-                emails = Email.objects.filter(address=address)
-                # there can only be on Email object with a unique email address (primary key)
-                if emails.exists():
-                    person = emails.first().person
-                # Create a new Person object
-                else:
-                    logger.error("No Person record for registration. email={}".format(address))
-                # update the person object to an actual value
-                object.person = person
-                object.save()
-            
-            if created:
-                num_created += 1
-            num_processed += 1
-
-        # any registrations left in meeting_registrations no longer exist in reg
-        # so must have been deleted
-        for r in meeting_registrations:
-            try:
-                MeetingRegistration.objects.get(meeting=meeting,email=r[0],reg_type=r[1]).delete()
-                logger.info('Removing deleted registration. email={}, reg_type={}'.format(r[0], r[1]))
-            except MeetingRegistration.DoesNotExist:
-                pass
-    else:
-        raise RuntimeError("Bad response from registrations API: %s, '%s'" % (response.status_code, response.content))
-    num_total = MeetingRegistration.objects.filter(
-        meeting_id=meeting.pk,
-        reg_type__in=['onsite', 'remote']
-    ).filter(
-        Q(attended=True) | Q(checkedin=True)
-    ).count()
-    if meeting.attendees is None or num_total > meeting.attendees:
-        meeting.attendees = num_total
-        meeting.save()
-    return num_created, num_processed, num_total
-
-
-FetchStats = namedtuple("FetchStats", "added processed total")
-
-
-def fetch_attendance_from_meetings(meetings):
-    stats = [
-        FetchStats(*get_meeting_registration_data(meeting)) for meeting in meetings
-    ]
-    return stats

From aa2770372287cad937e2aacacc95aefa90d9c6b3 Mon Sep 17 00:00:00 2001
From: Eric Vyncke <eric@vyncke.org>
Date: Sat, 19 Jul 2025 21:27:26 +0200
Subject: [PATCH 393/656] fix: AD dashboard shows undead documents returned to
 the WG (#9183)

* Do not display expired I-D while in IESG state of 'idexists'

* Do not remove AD&shepherd display
---
 ietf/doc/views_search.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py
index 3433a9ca31..67ff0c2f21 100644
--- a/ietf/doc/views_search.py
+++ b/ietf/doc/views_search.py
@@ -665,6 +665,7 @@ def sort_key(doc):
                 r.get_state_slug("draft-iesg") == "dead"
                 or r.get_state_slug("draft") == "repl"
                 or r.get_state_slug("draft") == "rfc"
+                or (r.get_state_slug("draft") == "expired" and r.get_state_slug("draft-iesg") == "idexists")
             )
         )
     ]

From eba6191213413b56ccc0a999d82f0dea6e7c0484 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 21 Jul 2025 03:05:13 -0400
Subject: [PATCH 394/656] chore(deps): bump luxon in /dev/coverage-action in
 the npm group (#9139)

Bumps the npm group in /dev/coverage-action with 1 update: [luxon](https://github.com/moment/luxon).


Updates `luxon` from 3.6.1 to 3.7.1
- [Changelog](https://github.com/moment/luxon/blob/master/CHANGELOG.md)
- [Commits](https://github.com/moment/luxon/compare/3.6.1...3.7.1)

---
updated-dependencies:
- dependency-name: luxon
  dependency-version: 3.7.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/coverage-action/package-lock.json | 15 +++++++--------
 dev/coverage-action/package.json      |  2 +-
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/dev/coverage-action/package-lock.json b/dev/coverage-action/package-lock.json
index 4a1ff1e03d..09570ee0e4 100644
--- a/dev/coverage-action/package-lock.json
+++ b/dev/coverage-action/package-lock.json
@@ -12,7 +12,7 @@
         "@actions/core": "1.11.1",
         "@actions/github": "6.0.1",
         "lodash": "4.17.21",
-        "luxon": "3.6.1"
+        "luxon": "3.7.1"
       }
     },
     "node_modules/@actions/core": {
@@ -245,10 +245,9 @@
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
     },
     "node_modules/luxon": {
-      "version": "3.6.1",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.6.1.tgz",
-      "integrity": "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ==",
-      "license": "MIT",
+      "version": "3.7.1",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.1.tgz",
+      "integrity": "sha512-RkRWjA926cTvz5rAb1BqyWkKbbjzCGchDUIKMCUvNi17j6f6j8uHGDV82Aqcqtzd+icoYpELmG3ksgGiFNNcNg==",
       "engines": {
         "node": ">=12"
       }
@@ -480,9 +479,9 @@
       "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
     },
     "luxon": {
-      "version": "3.6.1",
-      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.6.1.tgz",
-      "integrity": "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ=="
+      "version": "3.7.1",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.1.tgz",
+      "integrity": "sha512-RkRWjA926cTvz5rAb1BqyWkKbbjzCGchDUIKMCUvNi17j6f6j8uHGDV82Aqcqtzd+icoYpELmG3ksgGiFNNcNg=="
     },
     "once": {
       "version": "1.4.0",
diff --git a/dev/coverage-action/package.json b/dev/coverage-action/package.json
index d59a8de22c..3f72b78028 100644
--- a/dev/coverage-action/package.json
+++ b/dev/coverage-action/package.json
@@ -9,6 +9,6 @@
     "@actions/core": "1.11.1",
     "@actions/github": "6.0.1",
     "lodash": "4.17.21",
-    "luxon": "3.6.1"
+    "luxon": "3.7.1"
   }
 }

From ef99ffe11d0c92806a6fd57328b8f252cd2849a5 Mon Sep 17 00:00:00 2001
From: Russ Housley <housley@vigilsec.com>
Date: Mon, 21 Jul 2025 10:27:20 -0400
Subject: [PATCH 395/656] fix: correct RFC count in the activity report. fixes
 #9140 (#9178)

---
 ietf/meeting/tests_utils.py                 | 43 ++++++++++++++++++++-
 ietf/meeting/utils.py                       | 15 +++----
 ietf/templates/meeting/activity_report.html |  4 +-
 3 files changed, 51 insertions(+), 11 deletions(-)

diff --git a/ietf/meeting/tests_utils.py b/ietf/meeting/tests_utils.py
index 11ea63df4f..391e017d68 100644
--- a/ietf/meeting/tests_utils.py
+++ b/ietf/meeting/tests_utils.py
@@ -12,12 +12,19 @@
 from django.http import HttpResponse, JsonResponse
 from ietf.meeting.factories import MeetingFactory, RegistrationFactory, RegistrationTicketFactory
 from ietf.meeting.models import Registration
-from ietf.meeting.utils import (process_single_registration,
-    get_registration_data, sync_registration_data, fetch_attendance_from_meetings)
+from ietf.meeting.utils import (
+    process_single_registration,
+    get_registration_data, 
+    sync_registration_data, 
+    fetch_attendance_from_meetings, 
+    get_activity_stats
+)
 from ietf.nomcom.models import Volunteer
 from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year
 from ietf.person.factories import PersonFactory
 from ietf.utils.test_utils import TestCase
+from ietf.meeting.test_data import make_meeting_test_data
+from ietf.doc.factories import NewRevisionDocEventFactory, DocEventFactory
 
 
 class JsonResponseWithJson(JsonResponse):
@@ -25,6 +32,38 @@ def json(self):
         return json.loads(self.content)
 
 
+class ActivityStatsTests(TestCase):
+
+    def test_activity_stats(self):
+        utc = datetime.timezone.utc
+        make_meeting_test_data()
+        sdate = datetime.date(2016,4,3)
+        edate = datetime.date(2016,7,14)
+        MeetingFactory(type_id='ietf', date=sdate, number="96")
+        MeetingFactory(type_id='ietf', date=edate, number="97")
+
+        NewRevisionDocEventFactory(time=datetime.datetime(2016,4,5,12,0,0,0,tzinfo=utc))
+        NewRevisionDocEventFactory(time=datetime.datetime(2016,4,6,12,0,0,0,tzinfo=utc))
+        NewRevisionDocEventFactory(time=datetime.datetime(2016,4,7,12,0,0,0,tzinfo=utc))
+
+        NewRevisionDocEventFactory(time=datetime.datetime(2016,6,30,12,0,0,0,tzinfo=utc))
+        NewRevisionDocEventFactory(time=datetime.datetime(2016,6,30,13,0,0,0,tzinfo=utc))
+
+        DocEventFactory(doc__std_level_id="ps", doc__type_id="rfc", type="published_rfc", time=datetime.datetime(2016,4,5,12,0,0,0,tzinfo=utc))
+        DocEventFactory(doc__std_level_id="bcp", doc__type_id="rfc", type="published_rfc", time=datetime.datetime(2016,4,6,12,0,0,0,tzinfo=utc))
+        DocEventFactory(doc__std_level_id="inf", doc__type_id="rfc", type="published_rfc", time=datetime.datetime(2016,4,7,12,0,0,0,tzinfo=utc))
+        DocEventFactory(doc__std_level_id="exp", doc__type_id="rfc", type="published_rfc", time=datetime.datetime(2016,4,8,12,0,0,0,tzinfo=utc))
+
+        data = get_activity_stats(sdate, edate)
+        self.assertEqual(data['new_drafts_count'], len(data['new_docs']))
+        self.assertEqual(data['ffw_new_count'], 2)
+        self.assertEqual(data['ffw_new_percent'], '40%')
+        rfc_count = 0
+        for c in data['counts']:
+            rfc_count += data['counts'].get(c)
+        self.assertEqual(rfc_count, len(data['rfcs']))
+
+
 class GetRegistrationsTests(TestCase):
 
     @patch('ietf.meeting.utils.requests.get')
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index 1047783a0c..f6925269aa 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -949,13 +949,14 @@ def get_activity_stats(sdate, edate):
     data['ffw_update_count'] = ffw_update_count
     data['ffw_update_percent'] = ffw_update_percent
 
-    rfcs = events.filter(type='published_rfc')
-    data['rfcs'] = rfcs.select_related('doc').select_related('doc__group').select_related('doc__intended_std_level')
-
-    data['counts'] = {'std': rfcs.filter(doc__intended_std_level__in=('ps', 'ds', 'std')).count(),
-                      'bcp': rfcs.filter(doc__intended_std_level='bcp').count(),
-                      'exp': rfcs.filter(doc__intended_std_level='exp').count(),
-                      'inf': rfcs.filter(doc__intended_std_level='inf').count()}
+    rfcs_events = DocEvent.objects.filter(doc__type='rfc', time__gte=sdatetime, time__lt=edatetime)
+    rfcs = rfcs_events.filter(type='published_rfc')
+    data['rfcs'] = rfcs.select_related('doc').select_related('doc__group').select_related('doc__std_level')
+
+    data['counts'] = {'std': rfcs.filter(doc__std_level__in=('ps', 'ds', 'std')).count(),
+                      'bcp': rfcs.filter(doc__std_level='bcp').count(),
+                      'exp': rfcs.filter(doc__std_level='exp').count(),
+                      'inf': rfcs.filter(doc__std_level='inf').count()}
 
     data['new_groups'] = Group.objects.filter(
         type='wg',
diff --git a/ietf/templates/meeting/activity_report.html b/ietf/templates/meeting/activity_report.html
index 0de8b88c65..dbfc717dab 100644
--- a/ietf/templates/meeting/activity_report.html
+++ b/ietf/templates/meeting/activity_report.html
@@ -40,14 +40,14 @@ <h3 class="mt-3">{{ rfcs.count }} RFCs published this period</h3>
                 <th scope="col" data-sort="title">Title</th>
             </tr>
         </thead>
-        {% if rfcs %}
+        {% if rfcs|length > 0 %}
             <tbody>
                 {% for rfc in rfcs %}
                     <tr>
                         <td class="text-nowrap">
                             <a href="{{ rfc.doc.get_absolute_url }}">{{ rfc.doc.name|prettystdname }}</a>
                         </td>
-                        <td class="text-nowrap">{{ rfc.doc.intended_std_level.name }}</td>
+                        <td class="text-nowrap">{{ rfc.doc.std_level.name }}</td>
                         <td>
                             <a href="{{ rfc.doc.group.about_url }}">{{ rfc.doc.group.acronym }}</a>
                         </td>

From 8c1d881ff0ce1d10d02b18984fdc869fbaca865d Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 21 Jul 2025 16:32:11 +0200
Subject: [PATCH 396/656] fix: workaround magic false positives (#9207)

* fix: workaround magic false positives

* chore: ruff
---
 ietf/utils/mime.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/ietf/utils/mime.py b/ietf/utils/mime.py
index ab21cfe5c6..1f9b75b4df 100644
--- a/ietf/utils/mime.py
+++ b/ietf/utils/mime.py
@@ -5,6 +5,7 @@
 import magic
 import re
 
+
 def get_mime_type(content):
     # try to fixup encoding
     if hasattr(magic, "open"):
@@ -13,15 +14,17 @@ def get_mime_type(content):
         filetype = m.buffer(content)
     else:
         m = magic.Magic()
-        m.cookie = magic.magic_open(magic.MAGIC_NONE | magic.MAGIC_MIME | magic.MAGIC_MIME_ENCODING)
+        m.cookie = magic.magic_open(
+            magic.MAGIC_NONE | magic.MAGIC_MIME | magic.MAGIC_MIME_ENCODING
+        )
         magic.magic_load(m.cookie, None)
         filetype = m.from_buffer(content)
     # Work around silliness in libmagic on OpenSUSE 15.1
-    filetype = filetype.replace('text/x-Algol68;', 'text/plain;')
-    if ';' in filetype and 'charset=' in filetype:
-        mimetype, charset = re.split('; *charset=', filetype)
+    filetype = filetype.replace("text/x-Algol68;", "text/plain;")
+    filetype = filetype.replace("application/vnd.hp-HPGL;", "text/plain;")
+    if ";" in filetype and "charset=" in filetype:
+        mimetype, charset = re.split("; *charset=", filetype)
     else:
-        mimetype = re.split(';', filetype)[0]
-        charset = 'utf-8'
+        mimetype = re.split(";", filetype)[0]
+        charset = "utf-8"
     return mimetype, charset
-

From d2b70aea3efbbc847111ed493d01c403186d30af Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 21 Jul 2025 16:49:43 +0200
Subject: [PATCH 397/656] chore: move IESG menu item deeper into the menu
 structure (#9172)

---
 ietf/templates/base/menu.html    | 1 -
 ietf/templates/base/menu_wg.html | 3 +++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html
index 695445c578..bd8c0bf3cd 100644
--- a/ietf/templates/base/menu.html
+++ b/ietf/templates/base/menu.html
@@ -19,7 +19,6 @@
             <span class="fw-bolder">Groups</span>
         </li>
     {% endif %}
-    <li><a class="dropdown-item" href="{% url "ietf.group.views.group_about" acronym="iesg" %}">IESG</a></li>
     {% if flavor == 'top' %}<li class="dropdown-header">By area/parent</li>{% endif %}
     {% wg_menu flavor %}
     <li class="dropend">
diff --git a/ietf/templates/base/menu_wg.html b/ietf/templates/base/menu_wg.html
index 09fef8c7b0..5ca1be8a4e 100644
--- a/ietf/templates/base/menu_wg.html
+++ b/ietf/templates/base/menu_wg.html
@@ -2,6 +2,9 @@
 {% load origin %}
 {% origin %}
 {% for p in parents %}
+    {% if p.acronym == "iab" %}
+        <li><a class="dropdown-item" href="{% url "ietf.group.views.group_about" acronym="iesg" %}">IESG</a></li>
+    {% endif%}
     <li class="dropend group-menu group-parent-{{ p.id }}">
         <a class="dropdown-item dropdown-toggle {% if flavor != 'top' %}text-wrap{% endif %}"
            href="{{ p.menu_url }}">

From e043c85dc218da5802ee63adec634cba5c30cb4e Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Tue, 22 Jul 2025 04:11:50 +1200
Subject: [PATCH 398/656] fix: Community feed (#9184)

* style: Apply black / ruff formatting

* fix: Community feed
---
 ietf/community/tests.py           | 416 ++++++++++++++++++++++--------
 ietf/community/views.py           | 259 ++++++++++++-------
 ietf/templates/community/atom.xml |   8 +-
 3 files changed, 477 insertions(+), 206 deletions(-)

diff --git a/ietf/community/tests.py b/ietf/community/tests.py
index 9bd7789958..1255ba46eb 100644
--- a/ietf/community/tests.py
+++ b/ietf/community/tests.py
@@ -6,13 +6,20 @@
 
 from django.test.utils import override_settings
 from django.urls import reverse as urlreverse
+from lxml import etree
 
-import debug                            # pyflakes:ignore
+import debug  # pyflakes:ignore
 
 from ietf.community.models import CommunityList, SearchRule, EmailSubscription
 from ietf.community.signals import notify_of_event
-from ietf.community.utils import docs_matching_community_list_rule, community_list_rules_matching_doc
-from ietf.community.utils import reset_name_contains_index_for_rule, notify_event_to_subscribers
+from ietf.community.utils import (
+    docs_matching_community_list_rule,
+    community_list_rules_matching_doc,
+)
+from ietf.community.utils import (
+    reset_name_contains_index_for_rule,
+    notify_event_to_subscribers,
+)
 from ietf.community.tasks import notify_event_to_subscribers_task
 import ietf.community.views
 from ietf.group.models import Group
@@ -26,35 +33,80 @@
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.person.factories import PersonFactory, EmailFactory, AliasFactory
 
+
 class CommunityListTests(TestCase):
     def test_rule_matching(self):
-        plain = PersonFactory(user__username='plain')
-        ad = Person.objects.get(user__username='ad')
+        plain = PersonFactory(user__username="plain")
+        ad = Person.objects.get(user__username="ad")
         draft = WgDraftFactory(
-            group__parent=Group.objects.get(acronym='farfut' ),
+            group__parent=Group.objects.get(acronym="farfut"),
             authors=[ad],
             ad=ad,
             shepherd=plain.email(),
-            states=[('draft-iesg','lc'),('draft','active')],
+            states=[("draft-iesg", "lc"), ("draft", "active")],
         )
 
         clist = CommunityList.objects.create(person=plain)
 
-        rule_group = SearchRule.objects.create(rule_type="group", group=draft.group, state=State.objects.get(type="draft", slug="active"), community_list=clist)
-        rule_group_rfc = SearchRule.objects.create(rule_type="group_rfc", group=draft.group, state=State.objects.get(type="rfc", slug="published"), community_list=clist)
-        rule_area = SearchRule.objects.create(rule_type="area", group=draft.group.parent, state=State.objects.get(type="draft", slug="active"), community_list=clist)
+        rule_group = SearchRule.objects.create(
+            rule_type="group",
+            group=draft.group,
+            state=State.objects.get(type="draft", slug="active"),
+            community_list=clist,
+        )
+        rule_group_rfc = SearchRule.objects.create(
+            rule_type="group_rfc",
+            group=draft.group,
+            state=State.objects.get(type="rfc", slug="published"),
+            community_list=clist,
+        )
+        rule_area = SearchRule.objects.create(
+            rule_type="area",
+            group=draft.group.parent,
+            state=State.objects.get(type="draft", slug="active"),
+            community_list=clist,
+        )
 
-        rule_state_iesg = SearchRule.objects.create(rule_type="state_iesg", state=State.objects.get(type="draft-iesg", slug="lc"), community_list=clist)
+        rule_state_iesg = SearchRule.objects.create(
+            rule_type="state_iesg",
+            state=State.objects.get(type="draft-iesg", slug="lc"),
+            community_list=clist,
+        )
 
-        rule_author = SearchRule.objects.create(rule_type="author", state=State.objects.get(type="draft", slug="active"), person=Person.objects.filter(documentauthor__document=draft).first(), community_list=clist)
+        rule_author = SearchRule.objects.create(
+            rule_type="author",
+            state=State.objects.get(type="draft", slug="active"),
+            person=Person.objects.filter(documentauthor__document=draft).first(),
+            community_list=clist,
+        )
 
-        rule_ad = SearchRule.objects.create(rule_type="ad", state=State.objects.get(type="draft", slug="active"), person=draft.ad, community_list=clist)
+        rule_ad = SearchRule.objects.create(
+            rule_type="ad",
+            state=State.objects.get(type="draft", slug="active"),
+            person=draft.ad,
+            community_list=clist,
+        )
 
-        rule_shepherd = SearchRule.objects.create(rule_type="shepherd", state=State.objects.get(type="draft", slug="active"), person=draft.shepherd.person, community_list=clist)
+        rule_shepherd = SearchRule.objects.create(
+            rule_type="shepherd",
+            state=State.objects.get(type="draft", slug="active"),
+            person=draft.shepherd.person,
+            community_list=clist,
+        )
 
-        rule_group_exp = SearchRule.objects.create(rule_type="group_exp", group=draft.group, state=State.objects.get(type="draft", slug="expired"), community_list=clist)
+        rule_group_exp = SearchRule.objects.create(
+            rule_type="group_exp",
+            group=draft.group,
+            state=State.objects.get(type="draft", slug="expired"),
+            community_list=clist,
+        )
 
-        rule_name_contains = SearchRule.objects.create(rule_type="name_contains", state=State.objects.get(type="draft", slug="active"), text="draft-.*" + "-".join(draft.name.split("-")[2:]), community_list=clist)
+        rule_name_contains = SearchRule.objects.create(
+            rule_type="name_contains",
+            state=State.objects.get(type="draft", slug="active"),
+            text="draft-.*" + "-".join(draft.name.split("-")[2:]),
+            community_list=clist,
+        )
         reset_name_contains_index_for_rule(rule_name_contains)
 
         # doc -> rules
@@ -71,29 +123,44 @@ def test_rule_matching(self):
 
         # rule -> docs
         self.assertTrue(draft in list(docs_matching_community_list_rule(rule_group)))
-        self.assertTrue(draft not in list(docs_matching_community_list_rule(rule_group_rfc)))
+        self.assertTrue(
+            draft not in list(docs_matching_community_list_rule(rule_group_rfc))
+        )
         self.assertTrue(draft in list(docs_matching_community_list_rule(rule_area)))
-        self.assertTrue(draft in list(docs_matching_community_list_rule(rule_state_iesg)))
+        self.assertTrue(
+            draft in list(docs_matching_community_list_rule(rule_state_iesg))
+        )
         self.assertTrue(draft in list(docs_matching_community_list_rule(rule_author)))
         self.assertTrue(draft in list(docs_matching_community_list_rule(rule_ad)))
         self.assertTrue(draft in list(docs_matching_community_list_rule(rule_shepherd)))
-        self.assertTrue(draft in list(docs_matching_community_list_rule(rule_name_contains)))
-        self.assertTrue(draft not in list(docs_matching_community_list_rule(rule_group_exp)))
+        self.assertTrue(
+            draft in list(docs_matching_community_list_rule(rule_name_contains))
+        )
+        self.assertTrue(
+            draft not in list(docs_matching_community_list_rule(rule_group_exp))
+        )
 
-        draft.set_state(State.objects.get(type='draft', slug='expired'))
+        draft.set_state(State.objects.get(type="draft", slug="expired"))
 
         # doc -> rules
         matching_rules = list(community_list_rules_matching_doc(draft))
         self.assertTrue(rule_group_exp in matching_rules)
 
         # rule -> docs
-        self.assertTrue(draft in list(docs_matching_community_list_rule(rule_group_exp)))
+        self.assertTrue(
+            draft in list(docs_matching_community_list_rule(rule_group_exp))
+        )
 
     def test_view_list_duplicates(self):
-        person = PersonFactory(name="John Q. Public", user__username="bazquux@example.com")
+        person = PersonFactory(
+            name="John Q. Public", user__username="bazquux@example.com"
+        )
         PersonFactory(name="John Q. Public", user__username="foobar@example.com")
 
-        url = urlreverse(ietf.community.views.view_list, kwargs={ "email_or_name": person.plain_name()})
+        url = urlreverse(
+            ietf.community.views.view_list,
+            kwargs={"email_or_name": person.plain_name()},
+        )
         r = self.client.get(url)
         self.assertEqual(r.status_code, 404)
 
@@ -104,20 +171,23 @@ def complex_person(self, *args, **kwargs):
         return person
 
     def email_or_name_set(self, person):
-        return [e for e in Email.objects.filter(person=person)] + \
-               [a for a in Alias.objects.filter(person=person)]
+        return [e for e in Email.objects.filter(person=person)] + [
+            a for a in Alias.objects.filter(person=person)
+        ]
 
     def do_view_list_test(self, person):
         draft = WgDraftFactory()
         # without list
         for id in self.email_or_name_set(person):
-            url = urlreverse(ietf.community.views.view_list, kwargs={ "email_or_name": id })
+            url = urlreverse(
+                ietf.community.views.view_list, kwargs={"email_or_name": id}
+            )
             r = self.client.get(url)
             self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'")
 
         # with list
         clist = CommunityList.objects.create(person=person)
-        if not draft in clist.added_docs.all():
+        if draft not in clist.added_docs.all():
             clist.added_docs.add(draft)
         SearchRule.objects.create(
             community_list=clist,
@@ -126,31 +196,37 @@ def do_view_list_test(self, person):
             text="test",
         )
         for id in self.email_or_name_set(person):
-            url = urlreverse(ietf.community.views.view_list, kwargs={ "email_or_name": id })
+            url = urlreverse(
+                ietf.community.views.view_list, kwargs={"email_or_name": id}
+            )
             r = self.client.get(url)
             self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'")
             self.assertContains(r, draft.name)
 
     def test_view_list(self):
-        person = self.complex_person(user__username='plain')
+        person = self.complex_person(user__username="plain")
         self.do_view_list_test(person)
-        
+
     def test_view_list_without_active_email(self):
-        person = self.complex_person(user__username='plain')
+        person = self.complex_person(user__username="plain")
         person.email_set.update(active=False)
         self.do_view_list_test(person)
 
     def test_manage_personal_list(self):
-        person = self.complex_person(user__username='plain')
-        ad = Person.objects.get(user__username='ad')
+        person = self.complex_person(user__username="plain")
+        ad = Person.objects.get(user__username="ad")
         draft = WgDraftFactory(authors=[ad])
 
-        url = urlreverse(ietf.community.views.manage_list, kwargs={ "email_or_name": person.email() })
+        url = urlreverse(
+            ietf.community.views.manage_list, kwargs={"email_or_name": person.email()}
+        )
         login_testing_unauthorized(self, "plain", url)
 
         for id in self.email_or_name_set(person):
-            url = urlreverse(ietf.community.views.manage_list, kwargs={ "email_or_name": id })
-            r = self.client.get(url, user='plain')
+            url = urlreverse(
+                ietf.community.views.manage_list, kwargs={"email_or_name": id}
+            )
+            r = self.client.get(url, user="plain")
             self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'")
 
             # We can't call post() with follow=True because that 404's if
@@ -158,11 +234,13 @@ def test_manage_personal_list(self):
             # apparently re-encodes the already-encoded url.
             def follow(r):
                 redirect_url = r.url or url
-                return self.client.get(redirect_url, user='plain')
+                return self.client.get(redirect_url, user="plain")
 
             # add document
-            self.assertContains(r, 'add_document')
-            r = self.client.post(url, {'action': 'add_documents', 'documents': draft.pk})
+            self.assertContains(r, "add_document")
+            r = self.client.post(
+                url, {"action": "add_documents", "documents": draft.pk}
+            )
             self.assertEqual(r.status_code, 302, msg=f"id='{id}', url='{url}'")
             clist = CommunityList.objects.get(person__user__username="plain")
             self.assertTrue(clist.added_docs.filter(pk=draft.pk))
@@ -170,8 +248,10 @@ def follow(r):
             self.assertContains(r, draft.name, status_code=200)
 
             # remove document
-            self.assertContains(r, 'remove_document_%s' % draft.pk)
-            r = self.client.post(url, {'action': 'remove_document', 'document': draft.pk})
+            self.assertContains(r, "remove_document_%s" % draft.pk)
+            r = self.client.post(
+                url, {"action": "remove_document", "document": draft.pk}
+            )
             self.assertEqual(r.status_code, 302, msg=f"id='{id}', url='{url}'")
             clist = CommunityList.objects.get(person__user__username="plain")
             self.assertTrue(not clist.added_docs.filter(pk=draft.pk))
@@ -179,23 +259,37 @@ def follow(r):
             self.assertNotContains(r, draft.name, status_code=200)
 
             # add rule
-            r = self.client.post(url, {
-                "action": "add_rule",
-                "rule_type": "author_rfc",
-                "author_rfc-person": Person.objects.filter(documentauthor__document=draft).first().pk,
-            "author_rfc-state": State.objects.get(type="rfc", slug="published").pk,
-            })
+            r = self.client.post(
+                url,
+                {
+                    "action": "add_rule",
+                    "rule_type": "author_rfc",
+                    "author_rfc-person": Person.objects.filter(
+                        documentauthor__document=draft
+                    )
+                    .first()
+                    .pk,
+                    "author_rfc-state": State.objects.get(
+                        type="rfc", slug="published"
+                    ).pk,
+                },
+            )
             self.assertEqual(r.status_code, 302, msg=f"id='{id}', url='{url}'")
             clist = CommunityList.objects.get(person__user__username="plain")
             self.assertTrue(clist.searchrule_set.filter(rule_type="author_rfc"))
 
             # add name_contains rule
-            r = self.client.post(url, {
-                "action": "add_rule",
-                "rule_type": "name_contains",
-                "name_contains-text": "draft.*mars",
-                "name_contains-state": State.objects.get(type="draft", slug="active").pk,
-            })
+            r = self.client.post(
+                url,
+                {
+                    "action": "add_rule",
+                    "rule_type": "name_contains",
+                    "name_contains-text": "draft.*mars",
+                    "name_contains-state": State.objects.get(
+                        type="draft", slug="active"
+                    ).pk,
+                },
+            )
             self.assertEqual(r.status_code, 302, msg=f"id='{id}', url='{url}'")
             clist = CommunityList.objects.get(person__user__username="plain")
             self.assertTrue(clist.searchrule_set.filter(rule_type="name_contains"))
@@ -205,22 +299,31 @@ def follow(r):
             self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'")
             rule = clist.searchrule_set.filter(rule_type="author_rfc").first()
             q = PyQuery(r.content)
-            self.assertEqual(len(q('#r%s' % rule.pk)), 1)
+            self.assertEqual(len(q("#r%s" % rule.pk)), 1)
 
             # remove rule
-            r = self.client.post(url, {
-                "action": "remove_rule",
-                "rule": rule.pk,
-            })
+            r = self.client.post(
+                url,
+                {
+                    "action": "remove_rule",
+                    "rule": rule.pk,
+                },
+            )
 
             clist = CommunityList.objects.get(person__user__username="plain")
             self.assertTrue(not clist.searchrule_set.filter(rule_type="author_rfc"))
 
     def test_manage_group_list(self):
-        draft = WgDraftFactory(group__acronym='mars')
-        RoleFactory(group__acronym='mars',name_id='chair',person=PersonFactory(user__username='marschairman'))
+        draft = WgDraftFactory(group__acronym="mars")
+        RoleFactory(
+            group__acronym="mars",
+            name_id="chair",
+            person=PersonFactory(user__username="marschairman"),
+        )
 
-        url = urlreverse(ietf.community.views.manage_list, kwargs={ "acronym": draft.group.acronym })
+        url = urlreverse(
+            ietf.community.views.manage_list, kwargs={"acronym": draft.group.acronym}
+        )
         setup_default_community_list_for_group(draft.group)
         login_testing_unauthorized(self, "marschairman", url)
 
@@ -229,27 +332,41 @@ def test_manage_group_list(self):
         self.assertEqual(r.status_code, 200)
 
         # Verify GET also works with non-WG and RG groups
-        for gtype in ['area','program']:
+        for gtype in ["area", "program"]:
             g = GroupFactory.create(type_id=gtype)
             # make sure the group's features have been initialized to improve coverage
-            _ = g.features # pyflakes:ignore
+            _ = g.features  # pyflakes:ignore
             p = PersonFactory()
-            g.role_set.create(name_id={'area':'ad','program':'lead'}[gtype],person=p, email=p.email())
-            url = urlreverse(ietf.community.views.manage_list, kwargs={ "acronym": g.acronym })
+            g.role_set.create(
+                name_id={"area": "ad", "program": "lead"}[gtype],
+                person=p,
+                email=p.email(),
+            )
+            url = urlreverse(
+                ietf.community.views.manage_list, kwargs={"acronym": g.acronym}
+            )
             setup_default_community_list_for_group(g)
-            self.client.login(username=p.user.username,password=p.user.username+"+password")
+            self.client.login(
+                username=p.user.username, password=p.user.username + "+password"
+            )
             r = self.client.get(url)
             self.assertEqual(r.status_code, 200)
 
     def test_track_untrack_document(self):
-        person = self.complex_person(user__username='plain')
+        person = self.complex_person(user__username="plain")
         draft = WgDraftFactory()
 
-        url = urlreverse(ietf.community.views.track_document, kwargs={ "email_or_name": person.email(), "name": draft.name })
+        url = urlreverse(
+            ietf.community.views.track_document,
+            kwargs={"email_or_name": person.email(), "name": draft.name},
+        )
         login_testing_unauthorized(self, "plain", url)
 
         for id in self.email_or_name_set(person):
-            url = urlreverse(ietf.community.views.track_document, kwargs={ "email_or_name": id, "name": draft.name })
+            url = urlreverse(
+                ietf.community.views.track_document,
+                kwargs={"email_or_name": id, "name": draft.name},
+            )
 
             # track
             r = self.client.get(url)
@@ -261,7 +378,10 @@ def test_track_untrack_document(self):
             self.assertEqual(list(clist.added_docs.all()), [draft])
 
             # untrack
-            url = urlreverse(ietf.community.views.untrack_document, kwargs={ "email_or_name": id, "name": draft.name })
+            url = urlreverse(
+                ietf.community.views.untrack_document,
+                kwargs={"email_or_name": id, "name": draft.name},
+            )
             r = self.client.get(url)
             self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'")
 
@@ -271,36 +391,47 @@ def test_track_untrack_document(self):
             self.assertEqual(list(clist.added_docs.all()), [])
 
     def test_track_untrack_document_through_ajax(self):
-        person = self.complex_person(user__username='plain')
+        person = self.complex_person(user__username="plain")
         draft = WgDraftFactory()
 
-        url = urlreverse(ietf.community.views.track_document, kwargs={ "email_or_name": person.email(), "name": draft.name })
+        url = urlreverse(
+            ietf.community.views.track_document,
+            kwargs={"email_or_name": person.email(), "name": draft.name},
+        )
         login_testing_unauthorized(self, "plain", url)
 
         for id in self.email_or_name_set(person):
-            url = urlreverse(ietf.community.views.track_document, kwargs={ "email_or_name": id, "name": draft.name })
+            url = urlreverse(
+                ietf.community.views.track_document,
+                kwargs={"email_or_name": id, "name": draft.name},
+            )
 
             # track
-            r = self.client.post(url, HTTP_X_REQUESTED_WITH='XMLHttpRequest')
+            r = self.client.post(url, HTTP_X_REQUESTED_WITH="XMLHttpRequest")
             self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'")
             self.assertEqual(r.json()["success"], True)
             clist = CommunityList.objects.get(person__user__username="plain")
             self.assertEqual(list(clist.added_docs.all()), [draft])
 
             # untrack
-            url = urlreverse(ietf.community.views.untrack_document, kwargs={ "email_or_name": id, "name": draft.name })
-            r = self.client.post(url, HTTP_X_REQUESTED_WITH='XMLHttpRequest')
+            url = urlreverse(
+                ietf.community.views.untrack_document,
+                kwargs={"email_or_name": id, "name": draft.name},
+            )
+            r = self.client.post(url, HTTP_X_REQUESTED_WITH="XMLHttpRequest")
             self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'")
             self.assertEqual(r.json()["success"], True)
             clist = CommunityList.objects.get(person__user__username="plain")
             self.assertEqual(list(clist.added_docs.all()), [])
 
     def test_csv(self):
-        person = self.complex_person(user__username='plain')
+        person = self.complex_person(user__username="plain")
         draft = WgDraftFactory()
 
         for id in self.email_or_name_set(person):
-            url = urlreverse(ietf.community.views.export_to_csv, kwargs={ "email_or_name": id })
+            url = urlreverse(
+                ietf.community.views.export_to_csv, kwargs={"email_or_name": id}
+            )
 
             # without list
             r = self.client.get(url)
@@ -308,7 +439,7 @@ def test_csv(self):
 
             # with list
             clist = CommunityList.objects.create(person=person)
-            if not draft in clist.added_docs.all():
+            if draft not in clist.added_docs.all():
                 clist.added_docs.add(draft)
             SearchRule.objects.create(
                 community_list=clist,
@@ -324,7 +455,9 @@ def test_csv(self):
     def test_csv_for_group(self):
         draft = WgDraftFactory()
 
-        url = urlreverse(ietf.community.views.export_to_csv, kwargs={ "acronym": draft.group.acronym })
+        url = urlreverse(
+            ietf.community.views.export_to_csv, kwargs={"acronym": draft.group.acronym}
+        )
 
         setup_default_community_list_for_group(draft.group)
 
@@ -333,11 +466,11 @@ def test_csv_for_group(self):
         self.assertEqual(r.status_code, 200)
 
     def test_feed(self):
-        person = self.complex_person(user__username='plain')
+        person = self.complex_person(user__username="plain")
         draft = WgDraftFactory()
 
         for id in self.email_or_name_set(person):
-            url = urlreverse(ietf.community.views.feed, kwargs={ "email_or_name": id })
+            url = urlreverse(ietf.community.views.feed, kwargs={"email_or_name": id})
 
             # without list
             r = self.client.get(url)
@@ -345,7 +478,7 @@ def test_feed(self):
 
             # with list
             clist = CommunityList.objects.create(person=person)
-            if not draft in clist.added_docs.all():
+            if draft not in clist.added_docs.all():
                 clist.added_docs.add(draft)
             SearchRule.objects.create(
                 community_list=clist,
@@ -357,31 +490,53 @@ def test_feed(self):
             self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'")
             self.assertContains(r, draft.name)
 
+            # test atom xml
+            xml = etree.fromstring(r.content)
+            ns = {"atom": "http://www.w3.org/2005/Atom"}
+            updated = xml.xpath("/atom:feed/atom:updated", namespaces=ns)[0].text
+            entries = xml.xpath("/atom:feed/atom:entry", namespaces=ns)
+            self.assertIn("+00:00", updated)  # RFC 3339 compatible UTC TZ
+            for entry in entries:
+                updated = entry.xpath("atom:updated", namespaces=ns)[0].text
+                published = entry.xpath("atom:published", namespaces=ns)[0].text
+                entry_id = entry.xpath("atom:id", namespaces=ns)[0].text
+                self.assertIn("+00:00", updated)
+                self.assertIn("+00:00", published)
+                self.assertIn(
+                    "urn:datatracker-ietf-org:event:", entry_id
+                )  # atom:entry:id must be a valid URN
+
             # only significant
             r = self.client.get(url + "?significant=1")
             self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'")
-            self.assertNotContains(r, '<entry>')
+            self.assertNotContains(r, "<entry>")
 
     def test_feed_for_group(self):
         draft = WgDraftFactory()
 
-        url = urlreverse(ietf.community.views.feed, kwargs={ "acronym": draft.group.acronym })
+        url = urlreverse(
+            ietf.community.views.feed, kwargs={"acronym": draft.group.acronym}
+        )
 
         setup_default_community_list_for_group(draft.group)
 
         # test GET, rest is tested with personal list
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-        
+
     def test_subscription(self):
-        person = self.complex_person(user__username='plain')
+        person = self.complex_person(user__username="plain")
         draft = WgDraftFactory()
 
-        url = urlreverse(ietf.community.views.subscription, kwargs={ "email_or_name": person.email() })
+        url = urlreverse(
+            ietf.community.views.subscription, kwargs={"email_or_name": person.email()}
+        )
         login_testing_unauthorized(self, "plain", url)
 
         for id in self.email_or_name_set(person):
-            url = urlreverse(ietf.community.views.subscription, kwargs={ "email_or_name": id })
+            url = urlreverse(
+                ietf.community.views.subscription, kwargs={"email_or_name": id}
+            )
 
             # subscription without list
             r = self.client.get(url)
@@ -389,7 +544,7 @@ def test_subscription(self):
 
         # subscription with list
         clist = CommunityList.objects.create(person=person)
-        if not draft in clist.added_docs.all():
+        if draft not in clist.added_docs.all():
             clist.added_docs.add(draft)
         SearchRule.objects.create(
             community_list=clist,
@@ -399,29 +554,49 @@ def test_subscription(self):
         )
 
         for email in Email.objects.filter(person=person):
-            url = urlreverse(ietf.community.views.subscription, kwargs={ "email_or_name": email })
+            url = urlreverse(
+                ietf.community.views.subscription, kwargs={"email_or_name": email}
+            )
 
             r = self.client.get(url)
             self.assertEqual(r.status_code, 200)
 
             # subscribe
-            r = self.client.post(url, { "email": email.pk, "notify_on": "significant", "action": "subscribe" })
+            r = self.client.post(
+                url,
+                {"email": email.pk, "notify_on": "significant", "action": "subscribe"},
+            )
             self.assertEqual(r.status_code, 302)
 
-            subscription = EmailSubscription.objects.filter(community_list=clist, email=email, notify_on="significant").first()
+            subscription = EmailSubscription.objects.filter(
+                community_list=clist, email=email, notify_on="significant"
+            ).first()
 
             self.assertTrue(subscription)
 
             # delete subscription
-            r = self.client.post(url, { "subscription_id": subscription.pk, "action": "unsubscribe" })
+            r = self.client.post(
+                url, {"subscription_id": subscription.pk, "action": "unsubscribe"}
+            )
             self.assertEqual(r.status_code, 302)
-            self.assertEqual(EmailSubscription.objects.filter(community_list=clist, email=email, notify_on="significant").count(), 0)
+            self.assertEqual(
+                EmailSubscription.objects.filter(
+                    community_list=clist, email=email, notify_on="significant"
+                ).count(),
+                0,
+            )
 
     def test_subscription_for_group(self):
-        draft = WgDraftFactory(group__acronym='mars')
-        RoleFactory(group__acronym='mars',name_id='chair',person=PersonFactory(user__username='marschairman'))
+        draft = WgDraftFactory(group__acronym="mars")
+        RoleFactory(
+            group__acronym="mars",
+            name_id="chair",
+            person=PersonFactory(user__username="marschairman"),
+        )
 
-        url = urlreverse(ietf.community.views.subscription, kwargs={ "acronym": draft.group.acronym })
+        url = urlreverse(
+            ietf.community.views.subscription, kwargs={"acronym": draft.group.acronym}
+        )
 
         setup_default_community_list_for_group(draft.group)
 
@@ -434,7 +609,7 @@ def test_subscription_for_group(self):
     @mock.patch("ietf.community.signals.notify_of_event")
     def test_notification_signal_receiver(self, mock_notify_of_event):
         """Saving a newly created DocEvent should notify subscribers
-        
+
         This implicitly tests that notify_of_event_receiver is hooked up to the post_save signal.
         """
         # Arbitrary model that's not a DocEvent
@@ -442,19 +617,26 @@ def test_notification_signal_receiver(self, mock_notify_of_event):
         mock_notify_of_event.reset_mock()  # clear any calls that resulted from the factories
         person.save()
         self.assertFalse(mock_notify_of_event.called)
-    
+
         # build a DocEvent that is not yet persisted
         doc = DocumentFactory()
         event = DocEventFactory.build(by=person, doc=doc)  # builds but does not save...
         mock_notify_of_event.reset_mock()  # clear any calls that resulted from the factories
         event.save()
-        self.assertEqual(mock_notify_of_event.call_count, 1, "notify_task should be run on creation of DocEvent")
+        self.assertEqual(
+            mock_notify_of_event.call_count,
+            1,
+            "notify_task should be run on creation of DocEvent",
+        )
         self.assertEqual(mock_notify_of_event.call_args, mock.call(event))
 
-        # save the existing DocEvent and see that no notification is sent    
+        # save the existing DocEvent and see that no notification is sent
         mock_notify_of_event.reset_mock()
         event.save()
-        self.assertFalse(mock_notify_of_event.called, "notify_task should not be run save of on existing DocEvent")
+        self.assertFalse(
+            mock_notify_of_event.called,
+            "notify_task should not be run save of on existing DocEvent",
+        )
 
     # Mock out the on_commit call so we can tell whether the task was actually queued
     @mock.patch("ietf.submit.views.transaction.on_commit", side_effect=lambda x: x())
@@ -468,7 +650,10 @@ def test_notify_of_event(self, mock_notify_task, mock_on_commit):
         # under test does not make this call when in "test" mode
         with override_settings(SERVER_MODE="not-test"):
             notify_of_event(event)
-        self.assertTrue(mock_notify_task.delay.called, "notify_task should run for a DocEvent on a draft")
+        self.assertTrue(
+            mock_notify_task.delay.called,
+            "notify_task should run for a DocEvent on a draft",
+        )
         mock_notify_task.reset_mock()
 
         event.skip_community_list_notification = True
@@ -476,22 +661,28 @@ def test_notify_of_event(self, mock_notify_task, mock_on_commit):
         # under test does not make this call when in "test" mode
         with override_settings(SERVER_MODE="not-test"):
             notify_of_event(event)
-        self.assertFalse(mock_notify_task.delay.called, "notify_task should not run when skip_community_list_notification is set")
+        self.assertFalse(
+            mock_notify_task.delay.called,
+            "notify_task should not run when skip_community_list_notification is set",
+        )
 
         event = DocEventFactory.build(by=person, doc=DocumentFactory(type_id="rfc"))
         # be careful overriding SERVER_MODE - we do it here because the method
         # under test does not make this call when in "test" mode
         with override_settings(SERVER_MODE="not-test"):
             notify_of_event(event)
-        self.assertFalse(mock_notify_task.delay.called, "notify_task should not run on a document with type 'rfc'")
+        self.assertFalse(
+            mock_notify_task.delay.called,
+            "notify_task should not run on a document with type 'rfc'",
+        )
 
     @mock.patch("ietf.utils.mail.send_mail_text")
     def test_notify_event_to_subscribers(self, mock_send_mail_text):
-        person = PersonFactory(user__username='plain')
+        person = PersonFactory(user__username="plain")
         draft = WgDraftFactory()
 
         clist = CommunityList.objects.create(person=person)
-        if not draft in clist.added_docs.all():
+        if draft not in clist.added_docs.all():
             clist.added_docs.add(draft)
 
         sub_to_significant = EmailSubscription.objects.create(
@@ -522,11 +713,13 @@ def test_notify_event_to_subscribers(self, mock_send_mail_text):
         mock_send_mail_text.reset_mock()
         notify_event_to_subscribers(event)
         self.assertEqual(mock_send_mail_text.call_count, 2)
-        addresses = [call_args[0][1] for call_args in mock_send_mail_text.call_args_list]
+        addresses = [
+            call_args[0][1] for call_args in mock_send_mail_text.call_args_list
+        ]
         subjects = {call_args[0][3] for call_args in mock_send_mail_text.call_args_list}
         contents = {call_args[0][4] for call_args in mock_send_mail_text.call_args_list}
         self.assertCountEqual(
-            addresses, 
+            addresses,
             [sub_to_significant.email.address, sub_to_all.email.address],
         )
         self.assertEqual(len(subjects), 1)
@@ -545,4 +738,3 @@ def test_notify_event_to_subscribers_task(self, mock_notify):
         d.delete()
         notify_event_to_subscribers_task(event_id=d.pk)
         self.assertFalse(mock_notify.called)
-
diff --git a/ietf/community/views.py b/ietf/community/views.py
index 923ec556f3..08b1c24fe5 100644
--- a/ietf/community/views.py
+++ b/ietf/community/views.py
@@ -13,13 +13,24 @@
 from django.utils import timezone
 from django.utils.html import strip_tags
 
-import debug                            # pyflakes:ignore
+import debug  # pyflakes:ignore
 
 from ietf.community.models import CommunityList, EmailSubscription, SearchRule
-from ietf.community.forms import SearchRuleTypeForm, SearchRuleForm, AddDocumentsForm, SubscriptionForm
+from ietf.community.forms import (
+    SearchRuleTypeForm,
+    SearchRuleForm,
+    AddDocumentsForm,
+    SubscriptionForm,
+)
 from ietf.community.utils import can_manage_community_list
-from ietf.community.utils import docs_tracked_by_community_list, docs_matching_community_list_rule
-from ietf.community.utils import states_of_significant_change, reset_name_contains_index_for_rule
+from ietf.community.utils import (
+    docs_tracked_by_community_list,
+    docs_matching_community_list_rule,
+)
+from ietf.community.utils import (
+    states_of_significant_change,
+    reset_name_contains_index_for_rule,
+)
 from ietf.group.models import Group
 from ietf.doc.models import DocEvent, Document
 from ietf.doc.utils_search import prepare_document_table
@@ -31,43 +42,61 @@
 
 def lookup_community_list(request, email_or_name=None, acronym=None):
     """Finds a CommunityList for a person or group
-    
+
     Instantiates an unsaved CommunityList if one is not found.
-    
+
     If the person or group cannot be found and uniquely identified, raises an Http404 exception
     """
     assert email_or_name or acronym
 
     if acronym:
         group = get_object_or_404(Group, acronym=acronym)
-        clist = CommunityList.objects.filter(group=group).first() or CommunityList(group=group)
+        clist = CommunityList.objects.filter(group=group).first() or CommunityList(
+            group=group
+        )
     else:
         persons = lookup_persons(email_or_name)
         if len(persons) > 1:
-            if hasattr(request.user, 'person') and request.user.person in persons:
+            if hasattr(request.user, "person") and request.user.person in persons:
                 person = request.user.person
             else:
-                raise Http404(f"Unable to identify the CommunityList for {email_or_name}")
+                raise Http404(
+                    f"Unable to identify the CommunityList for {email_or_name}"
+                )
         else:
             person = persons[0]
-        clist = CommunityList.objects.filter(person=person).first() or CommunityList(person=person)
+        clist = CommunityList.objects.filter(person=person).first() or CommunityList(
+            person=person
+        )
     return clist
 
+
 def view_list(request, email_or_name=None):
     clist = lookup_community_list(request, email_or_name)  # may raise Http404
     docs = docs_tracked_by_community_list(clist)
     docs, meta = prepare_document_table(request, docs, request.GET)
 
-    subscribed = request.user.is_authenticated and (EmailSubscription.objects.none() if clist.pk is None else EmailSubscription.objects.filter(community_list=clist, email__person__user=request.user))
+    subscribed = request.user.is_authenticated and (
+        EmailSubscription.objects.none()
+        if clist.pk is None
+        else EmailSubscription.objects.filter(
+            community_list=clist, email__person__user=request.user
+        )
+    )
+
+    return render(
+        request,
+        "community/view_list.html",
+        {
+            "clist": clist,
+            "docs": docs,
+            "meta": meta,
+            "can_manage_list": can_manage_community_list(request.user, clist),
+            "subscribed": subscribed,
+            "email_or_name": email_or_name,
+        },
+    )
 
-    return render(request, 'community/view_list.html', {
-        'clist': clist,
-        'docs': docs,
-        'meta': meta,
-        'can_manage_list': can_manage_community_list(request.user, clist),
-        'subscribed': subscribed,
-        "email_or_name": email_or_name,
-    })
 
 @login_required
 @ignore_view_kwargs("group_type")
@@ -79,24 +108,24 @@ def manage_list(request, email_or_name=None, acronym=None):
     if not can_manage_community_list(request.user, clist):
         permission_denied(request, "You do not have permission to access this view")
 
-    action = request.POST.get('action')
+    action = request.POST.get("action")
 
-    if request.method == 'POST' and action == 'add_documents':
+    if request.method == "POST" and action == "add_documents":
         add_doc_form = AddDocumentsForm(request.POST)
         if add_doc_form.is_valid():
             if clist.pk is None:
                 clist.save()
 
-            for d in add_doc_form.cleaned_data['documents']:
-                if not d in clist.added_docs.all():
+            for d in add_doc_form.cleaned_data["documents"]:
+                if d not in clist.added_docs.all():
                     clist.added_docs.add(d)
 
             return HttpResponseRedirect("")
     else:
         add_doc_form = AddDocumentsForm()
 
-    if request.method == 'POST' and action == 'remove_document':
-        document_id = request.POST.get('document')
+    if request.method == "POST" and action == "remove_document":
+        document_id = request.POST.get("document")
         if clist.pk is not None and document_id:
             document = get_object_or_404(clist.added_docs, id=document_id)
             clist.added_docs.remove(document)
@@ -104,16 +133,16 @@ def manage_list(request, email_or_name=None, acronym=None):
             return HttpResponseRedirect("")
 
     rule_form = None
-    if request.method == 'POST' and action == 'add_rule':
+    if request.method == "POST" and action == "add_rule":
         rule_type_form = SearchRuleTypeForm(request.POST)
         if rule_type_form.is_valid():
-            rule_type = rule_type_form.cleaned_data['rule_type']
+            rule_type = rule_type_form.cleaned_data["rule_type"]
             if rule_type:
                 rule_form = SearchRuleForm(clist, rule_type, request.POST)
                 if rule_form.is_valid():
                     if clist.pk is None:
                         clist.save()
-    
+
                     rule = rule_form.save(commit=False)
                     rule.community_list = clist
                     rule.rule_type = rule_type
@@ -125,8 +154,8 @@ def manage_list(request, email_or_name=None, acronym=None):
     else:
         rule_type_form = SearchRuleTypeForm()
 
-    if request.method == 'POST' and action == 'remove_rule':
-        rule_pk = request.POST.get('rule')
+    if request.method == "POST" and action == "remove_rule":
+        rule_pk = request.POST.get("rule")
         if clist.pk is not None and rule_pk:
             rule = get_object_or_404(SearchRule, pk=rule_pk, community_list=clist)
             rule.delete()
@@ -137,23 +166,35 @@ def manage_list(request, email_or_name=None, acronym=None):
     for r in rules:
         r.matching_documents_count = docs_matching_community_list_rule(r).count()
 
-    empty_rule_forms = { rule_type: SearchRuleForm(clist, rule_type) for rule_type, _ in SearchRule.RULE_TYPES }
+    empty_rule_forms = {
+        rule_type: SearchRuleForm(clist, rule_type)
+        for rule_type, _ in SearchRule.RULE_TYPES
+    }
 
     total_count = docs_tracked_by_community_list(clist).count()
 
-    all_forms = [f for f in [rule_type_form, rule_form, add_doc_form, *empty_rule_forms.values()]
-                 if f is not None]
-    return render(request, 'community/manage_list.html', {
-        'clist': clist,
-        'rules': rules,
-        'individually_added': clist.added_docs.all() if clist.pk is not None else [],
-        'rule_type_form': rule_type_form,
-        'rule_form': rule_form,
-        'empty_rule_forms': empty_rule_forms,
-        'total_count': total_count,
-        'add_doc_form': add_doc_form,
-        'all_forms': all_forms,
-    })
+    all_forms = [
+        f
+        for f in [rule_type_form, rule_form, add_doc_form, *empty_rule_forms.values()]
+        if f is not None
+    ]
+    return render(
+        request,
+        "community/manage_list.html",
+        {
+            "clist": clist,
+            "rules": rules,
+            "individually_added": (
+                clist.added_docs.all() if clist.pk is not None else []
+            ),
+            "rule_type_form": rule_type_form,
+            "rule_form": rule_form,
+            "empty_rule_forms": empty_rule_forms,
+            "total_count": total_count,
+            "add_doc_form": add_doc_form,
+            "all_forms": all_forms,
+        },
+    )
 
 
 @login_required
@@ -161,24 +202,33 @@ def track_document(request, name, email_or_name=None, acronym=None):
     doc = get_object_or_404(Document, name=name)
 
     if request.method == "POST":
-        clist = lookup_community_list(request, email_or_name, acronym)  # may raise Http404
+        clist = lookup_community_list(
+            request, email_or_name, acronym
+        )  # may raise Http404
         if not can_manage_community_list(request.user, clist):
             permission_denied(request, "You do not have permission to access this view")
 
         if clist.pk is None:
             clist.save()
 
-        if not doc in clist.added_docs.all():
+        if doc not in clist.added_docs.all():
             clist.added_docs.add(doc)
 
         if is_ajax(request):
-            return HttpResponse(json.dumps({ 'success': True }), content_type='application/json')
+            return HttpResponse(
+                json.dumps({"success": True}), content_type="application/json"
+            )
         else:
             return HttpResponseRedirect(clist.get_absolute_url())
 
-    return render(request, "community/track_document.html", {
-        "name": doc.name,
-    })
+    return render(
+        request,
+        "community/track_document.html",
+        {
+            "name": doc.name,
+        },
+    )
+
 
 @login_required
 def untrack_document(request, name, email_or_name=None, acronym=None):
@@ -192,28 +242,34 @@ def untrack_document(request, name, email_or_name=None, acronym=None):
             clist.added_docs.remove(doc)
 
         if is_ajax(request):
-            return HttpResponse(json.dumps({ 'success': True }), content_type='application/json')
+            return HttpResponse(
+                json.dumps({"success": True}), content_type="application/json"
+            )
         else:
             return HttpResponseRedirect(clist.get_absolute_url())
 
-    return render(request, "community/untrack_document.html", {
-        "name": doc.name,
-    })
+    return render(
+        request,
+        "community/untrack_document.html",
+        {
+            "name": doc.name,
+        },
+    )
 
 
 @ignore_view_kwargs("group_type")
 def export_to_csv(request, email_or_name=None, acronym=None):
     clist = lookup_community_list(request, email_or_name, acronym)  # may raise Http404
-    response = HttpResponse(content_type='text/csv')
+    response = HttpResponse(content_type="text/csv")
 
     if clist.group:
         filename = "%s-draft-list.csv" % clist.group.acronym
     else:
         filename = "draft-list.csv"
 
-    response['Content-Disposition'] = 'attachment; filename=%s' % filename
+    response["Content-Disposition"] = "attachment; filename=%s" % filename
 
-    writer = csv.writer(response, dialect=csv.excel, delimiter=str(','))
+    writer = csv.writer(response, dialect=csv.excel, delimiter=str(","))
 
     header = [
         "Name",
@@ -226,12 +282,12 @@ def export_to_csv(request, email_or_name=None, acronym=None):
     ]
     writer.writerow(header)
 
-    docs = docs_tracked_by_community_list(clist).select_related('type', 'group', 'ad')
+    docs = docs_tracked_by_community_list(clist).select_related("type", "group", "ad")
     for doc in docs.prefetch_related("states", "tags"):
         row = []
         row.append(doc.name)
         row.append(doc.title)
-        e = doc.latest_event(type='new_revision')
+        e = doc.latest_event(type="new_revision")
         row.append(e.time.strftime("%Y-%m-%d") if e else "")
         row.append(strip_tags(doc.friendly_state()))
         row.append(doc.group.acronym if doc.group else "")
@@ -242,39 +298,54 @@ def export_to_csv(request, email_or_name=None, acronym=None):
 
     return response
 
+
 @ignore_view_kwargs("group_type")
 def feed(request, email_or_name=None, acronym=None):
     clist = lookup_community_list(request, email_or_name, acronym)  # may raise Http404
-    significant = request.GET.get('significant', '') == '1'
-
-    documents = docs_tracked_by_community_list(clist).values_list('pk', flat=True)
-    since = timezone.now() - datetime.timedelta(days=14)
-
-    events = DocEvent.objects.filter(
-        doc__id__in=documents,
-        time__gte=since,
-    ).distinct().order_by('-time', '-id').select_related("doc")
+    significant = request.GET.get("significant", "") == "1"
+
+    documents = docs_tracked_by_community_list(clist).values_list("pk", flat=True)
+    updated = timezone.now()
+    since = updated - datetime.timedelta(days=14)
+
+    events = (
+        DocEvent.objects.filter(
+            doc__id__in=documents,
+            time__gte=since,
+        )
+        .distinct()
+        .order_by("-time", "-id")
+        .select_related("doc")
+    )
 
     if significant:
-        events = events.filter(type="changed_state", statedocevent__state__in=list(states_of_significant_change()))
+        events = events.filter(
+            type="changed_state",
+            statedocevent__state__in=list(states_of_significant_change()),
+        )
 
     host = request.get_host()
-    feed_url = 'https://%s%s' % (host, request.get_full_path())
+    feed_url = "https://%s%s" % (host, request.get_full_path())
     feed_id = uuid.uuid5(uuid.NAMESPACE_URL, str(feed_url))
-    title = '%s RSS Feed' % clist.long_name()
+    title = "%s RSS Feed" % clist.long_name()
     if significant:
-        subtitle = 'Significant document changes'
+        subtitle = "Significant document changes"
     else:
-        subtitle = 'Document changes'
-
-    return render(request, 'community/atom.xml', {
-        'clist': clist,
-        'entries': events[:50],
-        'title': title,
-        'subtitle': subtitle,
-        'id': feed_id.urn,
-        'updated': timezone.now(),
-    }, content_type='text/xml')
+        subtitle = "Document changes"
+
+    return render(
+        request,
+        "community/atom.xml",
+        {
+            "clist": clist,
+            "entries": events[:50],
+            "title": title,
+            "subtitle": subtitle,
+            "id": feed_id.urn,
+            "updated": updated,
+        },
+        content_type="text/xml",
+    )
 
 
 @login_required
@@ -286,9 +357,11 @@ def subscription(request, email_or_name=None, acronym=None):
 
     person = request.user.person
 
-    existing_subscriptions = EmailSubscription.objects.filter(community_list=clist, email__person=person)
+    existing_subscriptions = EmailSubscription.objects.filter(
+        community_list=clist, email__person=person
+    )
 
-    if request.method == 'POST':
+    if request.method == "POST":
         action = request.POST.get("action")
         if action == "subscribe":
             form = SubscriptionForm(person, clist, request.POST)
@@ -300,14 +373,20 @@ def subscription(request, email_or_name=None, acronym=None):
                 return HttpResponseRedirect("")
 
         elif action == "unsubscribe":
-            existing_subscriptions.filter(pk=request.POST.get("subscription_id")).delete()
+            existing_subscriptions.filter(
+                pk=request.POST.get("subscription_id")
+            ).delete()
 
             return HttpResponseRedirect("")
     else:
         form = SubscriptionForm(person, clist)
 
-    return render(request, 'community/subscription.html', {
-        'clist': clist,
-        'form': form,
-        'existing_subscriptions': existing_subscriptions,
-    })
+    return render(
+        request,
+        "community/subscription.html",
+        {
+            "clist": clist,
+            "form": form,
+            "existing_subscriptions": existing_subscriptions,
+        },
+    )
diff --git a/ietf/templates/community/atom.xml b/ietf/templates/community/atom.xml
index 32e3b00292..01dcdfeee7 100644
--- a/ietf/templates/community/atom.xml
+++ b/ietf/templates/community/atom.xml
@@ -3,7 +3,7 @@
     <title type="text">{{ title }}</title>
     <subtitle type="text">{{ subtitle }}</subtitle>
     <id>{{ id }}</id>
-    <updated>{{ updated|date:"Y-m-d\TH:i:sO" }}</updated>
+    <updated>{{ updated.isoformat }}</updated>
     <link rel="alternate" type="text/html" hreflang="en" href="https://{{ request.get_host }}/"/>
     <link rel="self" type="application/atom+xml" href="https://{{ request.get_host }}{{ request.get_full_path }}"/>
 
@@ -17,11 +17,11 @@
 
         <link href="{{ entry.doc.get_absolute_url }}"/>
 
-        <id>{{ entry.id }}</id>
+        <id>urn:datatracker-ietf-org:event:{{ entry.id }}</id>
         
-        <updated>{{ entry.time|date:"Y-m-d\TH:i:sO" }}</updated>
+        <updated>{{ entry.time.isoformat }}</updated>
         
-        <published>{{ entry.time|date:"Y-m-d\TH:i:sO" }}</published>
+        <published>{{ entry.time.isoformat }}</published>
         
         <author>
             <name>{{ entry.by }}</name>

From 7c968139d0068d54a2de10013f62afe0a51522b3 Mon Sep 17 00:00:00 2001
From: Peter Yee <github@houseofyee.com>
Date: Tue, 22 Jul 2025 09:37:47 +0200
Subject: [PATCH 399/656] feat: Add calendar buttons for future group meetings
 (#9182)

The buttons appear when there are future meetings for a group, on
https://datatracker.ietf.org/wg/GROUPNAME/meetings/
The buttons are basically the same two calendar buttons that are
on https://datatracker.ietf.org/meeting/upcoming?show=GROUPNAME
This change addresses issue #7524. (Separately, the description
of #7526 might need to be updated to refer to the new buttons.)
---
 ietf/group/views.py                | 14 ++++++++++++++
 ietf/templates/group/meetings.html | 12 ++++++------
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/ietf/group/views.py b/ietf/group/views.py
index f30569d230..f59b270a08 100644
--- a/ietf/group/views.py
+++ b/ietf/group/views.py
@@ -58,6 +58,7 @@
 from django.utils import timezone
 from django.utils.html import escape
 from django.views.decorators.cache import cache_page, cache_control
+from django.urls import reverse
 
 import debug                            # pyflakes:ignore
 
@@ -898,6 +899,18 @@ def meetings(request, acronym, group_type=None):
                 far_past.append(s)
         past = recent_past
 
+    # Add calendar actions
+    cal_actions = []
+
+    cal_actions.append(dict(
+        label='Download as .ics',
+        url=reverse('ietf.meeting.views.upcoming_ical')+"?show="+group.acronym)
+    )
+    cal_actions.append(dict(
+        label='Subscribe with webcal',
+        url='webcal://'+request.get_host()+reverse('ietf.meeting.views.upcoming_ical')+"?show="+group.acronym)
+    )
+
     return render(
         request,
         "group/meetings.html",
@@ -915,6 +928,7 @@ def meetings(request, acronym, group_type=None):
                 "far_past": far_past,
                 "can_edit": can_edit,
                 "can_always_edit": can_always_edit,
+                "cal_actions": cal_actions,
             },
         ),
     )
diff --git a/ietf/templates/group/meetings.html b/ietf/templates/group/meetings.html
index deaea1e675..bee8111025 100644
--- a/ietf/templates/group/meetings.html
+++ b/ietf/templates/group/meetings.html
@@ -40,12 +40,12 @@ <h2 class="mt-5" id="inprogressmeets">Meetings in progress</h2>
     {% if future %}
         <h2 class="mt-5" id="futuremeets">
             Future Meetings
-            <a class="ms-2"
-               aria-label="icalendar entry for all scheduled future {{ group.acronym }} meetings"
-               title="icalendar entry for all scheduled future {{ group.acronym }} meetings"
-               href="{% url 'ietf.meeting.views.upcoming_ical' %}?show={{ group.acronym }}">
-                <i class="bi bi-calendar"></i>
-            </a>
+            {% for cal_action in cal_actions %}
+                <a class="btn btn-primary"
+                   href="{{ cal_action.url }}">
+                    {{ cal_action.label }}
+                </a>
+            {% endfor %}
         </h2>
         <table class="table table-sm table-striped tablesorter">
             <thead>

From 0e2df84f21d05a1a58ed5d99db4e213458396c89 Mon Sep 17 00:00:00 2001
From: Eric Vyncke <eric@vyncke.org>
Date: Wed, 23 Jul 2025 15:58:56 +0200
Subject: [PATCH 400/656] feat: Sort IESG and IAB Statements Pages with Active
 Statements at the Top (#9198)

* Use statement.state to sort entries

* remove spurious tabs

* Uncomment the URL

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/group/views.py                  |  2 +-
 ietf/templates/group/statements.html | 19 ++++++++++++++-----
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/ietf/group/views.py b/ietf/group/views.py
index f59b270a08..0c89302c6a 100644
--- a/ietf/group/views.py
+++ b/ietf/group/views.py
@@ -2201,7 +2201,7 @@ def statements(request, acronym, group_type=None):
                 ).values_list("state__slug", flat=True)[:1]
             )
         )
-        .order_by("-published")
+        .order_by("status", "-published")
     )
     return render(
         request,
diff --git a/ietf/templates/group/statements.html b/ietf/templates/group/statements.html
index 035c3bc967..6bbe3cb394 100644
--- a/ietf/templates/group/statements.html
+++ b/ietf/templates/group/statements.html
@@ -12,8 +12,8 @@ <h2 class="my-3">{{group.acronym|upper}} Statements</h2>
     {% if request.user|has_role:"Secretariat" %}
     <div class="buttonlist">
         <a id="start_button"
-           class="btn btn-primary"
-           href="{% url 'ietf.doc.views_statement.new_statement' %}">
+        class="btn btn-primary"
+        href="{% url 'ietf.doc.views_statement.new_statement' %}">
             Start New Statement
         </a>
     </div>
@@ -25,16 +25,25 @@ <h2 class="my-3">{{group.acronym|upper}} Statements</h2>
                 <th scope="col" data-sort="statement">Statement</th>
             </tr>
         </thead>
+{% regroup statements by status as grouped_statements %}
+{% for statement_group in grouped_statements %}
         <tbody>
-            {% for statement in statements %}
+            <tr class="table-info">
+                <th scope="col" colspan="2">
+                {{ statement_group.grouper|title }} {{"Statement"|plural:statement_group.list }} ({{ statement_group.list|length }} {{"hit"|plural:statement_group.list }})
+                </th>
+            </tr>
+        </tbody>
+        <tbody>
+        {% for statement in statement_group.list %}
             <tr>
                 <td title="{{ statement.published|date:'Y-m-d H:i:s O' }}">{{ statement.published|date:"Y-m-d" }}</td>
                 <td><a href="{% url 'ietf.doc.views_doc.document_main' name=statement.name %}">{{statement.title}}</a>
-                    {% if statement.status == "replaced" %}<span class="badge rounded-pill text-bg-warning">Replaced</span>{% endif %}
                 </td>
             </tr>
-            {% endfor %}
+        {% endfor %}   
         </tbody>
+{% endfor %}
     </table>
 {% endblock %}
 {% block js %}

From 5a862b2022cbb4650596f8a2074b273b74b04660 Mon Sep 17 00:00:00 2001
From: Rich Salz <rsalz@akamai.com>
Date: Wed, 23 Jul 2025 10:01:48 -0400
Subject: [PATCH 401/656] feat: Append ascii name if any 8bit UTF8 chars
 (#9173)

Fixes: 7167
---
 ietf/utils/tests.py    | 8 ++++++++
 ietf/utils/xmldraft.py | 6 ++++++
 2 files changed, 14 insertions(+)

diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py
index 872aa366b9..88484000f7 100644
--- a/ietf/utils/tests.py
+++ b/ietf/utils/tests.py
@@ -709,6 +709,14 @@ def test_render_author_name(self):
             )),
             "Joanna Q. Public",
         )
+        self.assertEqual(
+            XMLDraft.render_author_name(lxml.etree.Element(
+                "author",
+                fullname=chr(340)+"ich",
+                asciiFullname="Rich UTF-8",
+            )),
+            chr(340)+"ich (Rich UTF-8)",
+        )
         self.assertEqual(
             XMLDraft.render_author_name(lxml.etree.Element(
                 "author",
diff --git a/ietf/utils/xmldraft.py b/ietf/utils/xmldraft.py
index 7ef6605c78..f555a0a16a 100644
--- a/ietf/utils/xmldraft.py
+++ b/ietf/utils/xmldraft.py
@@ -233,6 +233,12 @@ def render_author_name(author_elt):
         # Use fullname attribute, if present
         fullname = author_elt.attrib.get("fullname", "").strip()
         if fullname:
+            # If any 8bit chars in the fullname, try to append the author's
+            # name in ascii.
+            if any([x >= 0x80 for x in fullname.encode('utf8')]):
+                asciifullname = author_elt.attrib.get("asciiFullname", "").strip()
+                if asciifullname:
+                    fullname = fullname + ' (' + asciifullname + ')'
             return fullname
         surname = author_elt.attrib.get("surname", "").strip()
         initials = author_elt.attrib.get("initials", "").strip()

From 47afc6ed95895f2f7d03947813fec48417b5c2d6 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 23 Jul 2025 10:08:52 -0400
Subject: [PATCH 402/656] fix: prevent navigation on doc search header sort
 click (#9227)

---
 ietf/static/js/list.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ietf/static/js/list.js b/ietf/static/js/list.js
index c16111ba63..756a75001a 100644
--- a/ietf/static/js/list.js
+++ b/ietf/static/js/list.js
@@ -247,7 +247,8 @@ $(document)
 
                 $(table)
                     .find(".sort")
-                    .on("click", function () {
+                    .on("click", function (ev) {
+                        ev.preventDefault()
                         var order = $(this)
                             .hasClass("asc") ? "desc" : "asc";
                         $.each(list_instance, (_, e) => {

From ced120fd0e5ffc713b910054087149d9c6fe8933 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 23 Jul 2025 16:11:58 +0200
Subject: [PATCH 403/656] feat: improve template coverage cfg (#9230)

* feat: template coverage ignores using path

* refactor: reduce repeatitive redundancy

* chore: copyrights

* refactor: use pathlib to compute BASE_DIR

* feat: settings.PROJECT_DIR

* fix: glob from PROJECT_DIR not BASE_DIR

* chore: remove reference code
---
 ietf/settings.py          | 22 +++++++++++++--------
 ietf/utils/test_runner.py | 41 ++++++++++++++++++++-------------------
 ietf/utils/tests.py       |  9 ++-------
 3 files changed, 37 insertions(+), 35 deletions(-)

diff --git a/ietf/settings.py b/ietf/settings.py
index d76a8749b2..64679ca1d8 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2007-2024, All Rights Reserved
+# Copyright The IETF Trust 2007-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -9,6 +9,7 @@
 import os
 import sys
 import datetime
+import pathlib
 import warnings
 from hashlib import sha384
 from typing import Any, Dict, List, Tuple # pyflakes:ignore
@@ -27,8 +28,12 @@
 warnings.filterwarnings("ignore", message="Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated", module="bleach")
 warnings.filterwarnings("ignore", message="HTTPResponse.getheader\\(\\) is deprecated", module='selenium.webdriver')
 
-BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.abspath(BASE_DIR + "/.."))
+base_path = pathlib.Path(__file__).resolve().parent
+BASE_DIR = str(base_path)
+
+project_path = base_path.parent
+PROJECT_DIR = str(project_path)  
+sys.path.append(PROJECT_DIR)
 
 from ietf import __version__
 import debug
@@ -717,12 +722,13 @@ def skip_unreadable_post(record):
 ]
 
 # These are filename globs.  They are used by test_parse_templates() and
-# get_template_paths()
+# get_template_paths(). Globs are applied via pathlib.Path().match, using
+# the path to the template from the project root.
 TEST_TEMPLATE_IGNORE = [
-    ".*",                             # dot-files
-    "*~",                             # tilde temp-files
-    "#*",                             # files beginning with a hashmark
-    "500.html"                        # isn't loaded by regular loader, but checked by test_500_page()
+    ".*",  # dot-files
+    "*~",  # tilde temp-files
+    "#*",  # files beginning with a hashmark
+    "500.html",  # isn't loaded by regular loader, but checked by test_500_page()
 ]
 
 TEST_COVERAGE_MAIN_FILE = os.path.join(BASE_DIR, "../release-coverage.json")
diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py
index bfe5a56597..d0a5496283 100644
--- a/ietf/utils/test_runner.py
+++ b/ietf/utils/test_runner.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2009-2020, All Rights Reserved
+# Copyright The IETF Trust 2009-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 #
 # Portion Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
@@ -54,7 +54,6 @@
 import urllib3
 import warnings
 
-from fnmatch import fnmatch
 from typing import Callable, Optional
 from urllib.parse import urlencode
 
@@ -414,8 +413,9 @@ def do_append(res, p0, p1, item):
             res.append((str(item.pattern), item))
     return res
 
+
 _all_templates = None
-def get_template_paths(apps=None):
+def get_template_paths(apps=None) -> list[str]:
     global _all_templates
     if not _all_templates:
         # TODO: Add app templates to the full list, if we are using
@@ -424,25 +424,26 @@ def get_template_paths(apps=None):
         templatepaths = settings.TEMPLATES[0]['DIRS']
         for templatepath in templatepaths:
             for dirpath, dirs, files in os.walk(templatepath):
-                if ".svn" in dirs:
-                    dirs.remove(".svn")
-                relative_path = dirpath[len(templatepath)+1:]
-                for file in files:
-                    ignore = False
-                    for pattern in settings.TEST_TEMPLATE_IGNORE:
-                        if fnmatch(file, pattern):
-                            ignore = True
-                            break
-                    if ignore:
-                        continue
-                    if relative_path != "":
-                        file = os.path.join(relative_path, file)
-                    templates.add(file)
-        if apps:
-            templates = [ t for t in templates if t.split(os.path.sep)[0] in apps ]
-        _all_templates = templates
+                # glob against path from PROJECT_DIR
+                project_path = pathlib.Path(
+                    dirpath.removeprefix(settings.PROJECT_DIR).lstrip("/") 
+                )
+                # label entries with name relative to templatepath
+                relative_path = pathlib.Path(
+                    dirpath.removeprefix(templatepath).lstrip("/")
+                )
+                if apps and relative_path.parts[0] not in apps:
+                    continue  # skip uninteresting apps
+                for filename in files:
+                    file_path = project_path / filename
+                    if not any(
+                        file_path.match(pat) for pat in settings.TEST_TEMPLATE_IGNORE
+                    ):
+                        templates.add(relative_path / filename)
+        _all_templates = [str(t) for t in templates]
     return _all_templates
 
+
 def save_test_results(failures, test_labels):
     # Record the test result in a file, in order to be able to check the
     # results and avoid re-running tests if we've already run them with OK
diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py
index 88484000f7..ce1842236d 100644
--- a/ietf/utils/tests.py
+++ b/ietf/utils/tests.py
@@ -19,7 +19,6 @@
 from email.mime.image import MIMEImage
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
-from fnmatch import fnmatch
 from importlib import import_module
 from textwrap import dedent
 from tempfile import mkdtemp
@@ -320,7 +319,7 @@ class TemplateChecksTestCase(TestCase):
     def setUp(self):
         super().setUp()
         set_coverage_checking(False)
-        self.paths = list(get_template_paths())
+        self.paths = get_template_paths()  # already filtered ignores
         self.paths.sort()
         for path in self.paths:
             try:
@@ -335,11 +334,7 @@ def tearDown(self):
     def test_parse_templates(self):
         errors = []
         for path in self.paths:
-            for pattern in settings.TEST_TEMPLATE_IGNORE:
-                if fnmatch(path, pattern):
-                    continue
-            if not path in self.templates:
-
+            if path not in self.templates:
                 try:
                     get_template(path)
                 except Exception as e:

From ae5080bb8d72d5e52534caf5b64d50dfda960730 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 23 Jul 2025 16:38:37 +0200
Subject: [PATCH 404/656] fix: better wrap of nomcom key in session (#9233)

---
 ietf/nomcom/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py
index 10494d323f..dd651c2941 100644
--- a/ietf/nomcom/utils.py
+++ b/ietf/nomcom/utils.py
@@ -184,7 +184,7 @@ def retrieve_nomcom_private_key(request, year):
     if not private_key:
         return private_key
 
-    command = "%s bf -d -in /dev/stdin -k \"%s\" -a"
+    command = "%s aes-128-ecb -d -in /dev/stdin -k \"%s\" -a -iter 1000"
     code, out, error = pipe(
         command % (
             settings.OPENSSL_COMMAND,
@@ -208,7 +208,7 @@ def store_nomcom_private_key(request, year, private_key):
     if not private_key:
         request.session['NOMCOM_PRIVATE_KEY_%s' % year] = ''
     else:
-        command = "%s bf -e -in /dev/stdin -k \"%s\" -a"
+        command = "%s aes-128-ecb -e -in /dev/stdin -k \"%s\" -a -iter 1000"
         code, out, error = pipe(
             command % (
                 settings.OPENSSL_COMMAND,

From 328ee129588c4b096808ab4e33afedd4b2cc5833 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 24 Jul 2025 13:04:27 +0200
Subject: [PATCH 405/656] fix: set charset for text/plain responses (#9120)

* fix: charset for text/plain

Use settings.DEFAULT_CHARSET

* fix: more charset for text/plain

ALL THE PLACES
---
 ietf/api/views.py        |  30 +++++++++--
 ietf/doc/views_ballot.py |  12 ++++-
 ietf/iesg/views.py       |  27 +++++++---
 ietf/meeting/views.py    | 112 +++++++++++++++++++++++++++++++--------
 ietf/nomcom/views.py     |   5 +-
 ietf/submit/views.py     |   5 +-
 ietf/utils/decorators.py |   2 +-
 7 files changed, 155 insertions(+), 38 deletions(-)

diff --git a/ietf/api/views.py b/ietf/api/views.py
index b4dd7f05d6..22523b2f17 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -114,7 +114,11 @@ class ApiV2PersonExportView(DetailView, JsonExportMixin):
     model = Person
 
     def err(self, code, text):
-        return HttpResponse(text, status=code, content_type='text/plain')
+        return HttpResponse(
+            text,
+            status=code,
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+        )
 
     def post(self, request):
         querydict = request.POST.copy()
@@ -152,7 +156,11 @@ def post(self, request):
 def api_new_meeting_registration_v2(request):
     '''REST API to notify the datatracker about a new meeting registration'''
     def _http_err(code, text):
-        return HttpResponse(text, status=code, content_type="text/plain")
+        return HttpResponse(
+            text,
+            status=code,
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+        )
 
     def _api_response(result):
         return JsonResponse(data={"result": result})
@@ -192,7 +200,11 @@ def _api_response(result):
 
     process_single_registration(reg_data, meeting)
 
-    return HttpResponse('Success', status=202, content_type='text/plain')
+    return HttpResponse(
+        'Success',
+        status=202,
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
 
 
 def version(request):
@@ -511,7 +523,11 @@ def related_email_list(request, email):
     to Datatracker, via Person object
     """
     def _http_err(code, text):
-        return HttpResponse(text, status=code, content_type="text/plain")
+        return HttpResponse(
+            text,
+            status=code,
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+        )
 
     if request.method == "GET":
         try:
@@ -637,7 +653,11 @@ def ingest_email_handler(request, test_mode=False):
     """
 
     def _http_err(code, text):
-        return HttpResponse(text, status=code, content_type="text/plain")
+        return HttpResponse(
+            text,
+            status=code,
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+        )
 
     def _api_response(result):
         return JsonResponse(data={"result": result})
diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py
index 9e2a417933..0ba340890d 100644
--- a/ietf/doc/views_ballot.py
+++ b/ietf/doc/views_ballot.py
@@ -257,7 +257,11 @@ def edit_position(request, name, ballot_id):
 @csrf_exempt
 def api_set_position(request):
     def err(code, text):
-        return HttpResponse(text, status=code, content_type='text/plain')
+        return HttpResponse(
+            text,
+            status=code,
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+        )
     if request.method == 'POST':
         ad = request.user.person
         name = request.POST.get('doc')
@@ -290,7 +294,11 @@ def err(code, text):
     addrs, frm, subject, body = build_position_email(ad, doc, pos)
     send_mail_text(request, addrs.to, frm, subject, body, cc=addrs.cc)
 
-    return HttpResponse("Done", status=200, content_type='text/plain')
+    return HttpResponse(
+        "Done",
+        status=200,
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
 
 
 def build_position_email(balloter, doc, pos):
diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py
index f1fe77f763..7b9f489b44 100644
--- a/ietf/iesg/views.py
+++ b/ietf/iesg/views.py
@@ -221,7 +221,7 @@ def agenda_txt(request, date=None):
             "date": data["date"],
             "sections": sorted(data["sections"].items(), key=lambda x:[int(p) for p in x[0].split('.')]),
             "domain": Site.objects.get_current().domain,
-            }, content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET)
+            }, content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}")
 
 @role_required('Area Director', 'Secretariat')
 def agenda_moderator_package(request, date=None):
@@ -277,14 +277,23 @@ def leaf_section(num, section):
 @role_required('Area Director', 'Secretariat')
 def agenda_package(request, date=None):
     data = agenda_data(date)
-    return render(request, "iesg/agenda_package.txt", {
+    return render(
+        request,
+        "iesg/agenda_package.txt",
+        {
             "date": data["date"],
             "sections": sorted(data["sections"].items()),
             "roll_call": data["sections"]["1.1"]["text"],
             "minutes": data["sections"]["1.3"]["text"],
-            "management_items": [(num, section) for num, section in data["sections"].items() if "6" < num < "7"],
+            "management_items": [
+                (num, section)
+                for num, section in data["sections"].items()
+                if "6" < num < "7"
+            ],
             "domain": Site.objects.get_current().domain,
-            }, content_type='text/plain')
+        },
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
 
 
 def agenda_documents_txt(request):
@@ -315,7 +324,10 @@ def agenda_documents_txt(request):
             d.rev,
             )
         rows.append("\t".join(row))
-    return HttpResponse("\n".join(rows), content_type='text/plain')
+    return HttpResponse(
+        "\n".join(rows),
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
 
 class RescheduleForm(forms.Form):
     telechat_date = forms.TypedChoiceField(coerce=lambda x: datetime.datetime.strptime(x, '%Y-%m-%d').date(), empty_value=None, required=False)
@@ -610,4 +622,7 @@ def telechat_agenda_content_manage(request):
 @role_required("Secretariat", "IAB Chair", "Area Director")
 def telechat_agenda_content_view(request, section):
     content = get_object_or_404(TelechatAgendaContent, section__slug=section, section__used=True)
-    return HttpResponse(content=content.text, content_type="text/plain; charset=utf-8")
+    return HttpResponse(
+        content=content.text,
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 8bd70a3733..85eda5a8f4 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -2760,7 +2760,10 @@ def upload_session_bluesheets(request, session_id, num):
             ota = session.official_timeslotassignment()
             sess_time = ota and ota.timeslot.time
             if not sess_time:
-                return HttpResponseGone("Cannot receive uploads for an unscheduled session.  Please check the session ID.", content_type="text/plain")
+                return HttpResponseGone(
+                    "Cannot receive uploads for an unscheduled session.  Please check the session ID.",
+                    content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+                )
 
 
             save_error = save_bluesheet(request, session, file, encoding=form.file_encoding[file.name])
@@ -2821,7 +2824,7 @@ def upload_session_minutes(request, session_id, num):
             except SessionNotScheduledError:
                 return HttpResponseGone(
                     "Cannot receive uploads for an unscheduled session. Please check the session ID.",
-                    content_type="text/plain",
+                    content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
                 )
             except SaveMaterialsError as err:
                 form.add_error(None, str(err))
@@ -2880,7 +2883,7 @@ def upload_session_narrativeminutes(request, session_id, num):
             except SessionNotScheduledError:
                 return HttpResponseGone(
                     "Cannot receive uploads for an unscheduled session. Please check the session ID.",
-                    content_type="text/plain",
+                    content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
                 )
             except SaveMaterialsError as err:
                 form.add_error(None, str(err))
@@ -2978,7 +2981,10 @@ def upload_session_agenda(request, session_id, num):
                 ota = session.official_timeslotassignment()
                 sess_time = ota and ota.timeslot.time
                 if not sess_time:
-                    return HttpResponseGone("Cannot receive uploads for an unscheduled session.  Please check the session ID.", content_type="text/plain")
+                    return HttpResponseGone(
+                        "Cannot receive uploads for an unscheduled session.  Please check the session ID.",
+                        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+                    )
                 if session.meeting.type_id=='ietf':
                     name = 'agenda-%s-%s' % (session.meeting.number, 
                                                  session.group.acronym) 
@@ -4282,11 +4288,17 @@ def api_set_meetecho_recording_name(request):
         name: the name to use for the recording at meetecho player
     """
     def err(code, text):
-        return HttpResponse(text, status=code, content_type='text/plain')
+        return HttpResponse(
+            text,
+            status=code,
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+        )
 
     if request.method != "POST":
         return HttpResponseNotAllowed(
-            content="Method not allowed", content_type="text/plain", permitted_methods=('POST',)
+            content="Method not allowed",
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+            permitted_methods=('POST',),
         )
 
     session_id = request.POST.get('session_id', None)
@@ -4306,7 +4318,11 @@ def err(code, text):
     session.meetecho_recording_name = name
     session.save()
 
-    return HttpResponse("Done", status=200, content_type='text/plain')
+    return HttpResponse(
+        "Done",
+        status=200,
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
 
 @require_api_key
 @role_required('Recording Manager')
@@ -4320,11 +4336,17 @@ def api_set_session_video_url(request):
       url: The recording url (on YouTube, or whatever)
     """
     def err(code, text):
-        return HttpResponse(text, status=code, content_type='text/plain')
+        return HttpResponse(
+            text,
+            status=code,
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+        )
 
     if request.method != 'POST':
         return HttpResponseNotAllowed(
-            content="Method not allowed", content_type="text/plain", permitted_methods=('POST',)
+            content="Method not allowed",
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+            permitted_methods=('POST',),
         )
 
     # Temporary: fall back to deprecated interface if we have old-style parameters.
@@ -4363,7 +4385,11 @@ def err(code, text):
         time = session.official_timeslotassignment().timeslot.time
         title = 'Video recording for %s on %s at %s' % (session.group.acronym, time.date(), time.time())
         create_recording(session, incoming_url, title=title, user=request.user.person)
-    return HttpResponse("Done", status=200, content_type='text/plain')
+    return HttpResponse(
+        "Done",
+        status=200,
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
 
 
 def deprecated_api_set_session_video_url(request):
@@ -4372,7 +4398,11 @@ def deprecated_api_set_session_video_url(request):
     Uses meeting/group/item to identify session.
     """
     def err(code, text):
-        return HttpResponse(text, status=code, content_type='text/plain')
+        return HttpResponse(
+            text,
+            status=code,
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+        )
     if request.method == 'POST':
         # parameters:
         #   apikey: the poster's personal API key
@@ -4426,7 +4456,11 @@ def err(code, text):
     else:
         return err(405, "Method not allowed")
 
-    return HttpResponse("Done", status=200, content_type='text/plain')
+    return HttpResponse(
+        "Done",
+        status=200,
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
 
 
 @require_api_key
@@ -4478,7 +4512,11 @@ def api_add_session_attendees(request):
     )
 
     def err(code, text):
-        return HttpResponse(text, status=code, content_type="text/plain")
+        return HttpResponse(
+            text,
+            status=code,
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+        )
 
     if request.method != "POST":
         return err(405, "Method not allowed")
@@ -4531,7 +4569,11 @@ def err(code, text):
         if save_error:
             return err(400, save_error)
 
-    return HttpResponse("Done", status=200, content_type="text/plain")
+    return HttpResponse(
+        "Done",
+        status=200,
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
 
 
 @require_api_key
@@ -4539,7 +4581,11 @@ def err(code, text):
 @csrf_exempt
 def api_upload_chatlog(request):
     def err(code, text):
-        return HttpResponse(text, status=code, content_type='text/plain')
+        return HttpResponse(
+            text,
+            status=code,
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+        )
     if request.method != 'POST':
         return err(405, "Method not allowed")
     apidata_post = request.POST.get('apidata')
@@ -4572,14 +4618,22 @@ def err(code, text):
     write_doc_for_session(session, 'chatlog', filename, json.dumps(apidata['chatlog']))
     e = NewRevisionDocEvent.objects.create(doc=doc, rev=doc.rev, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev)
     doc.save_with_history([e])
-    return HttpResponse("Done", status=200, content_type='text/plain')
+    return HttpResponse(
+        "Done",
+        status=200,
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
 
 @require_api_key
 @role_required('Recording Manager')
 @csrf_exempt
 def api_upload_polls(request):
     def err(code, text):
-        return HttpResponse(text, status=code, content_type='text/plain')
+        return HttpResponse(
+            text,
+            status=code,
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+        )
     if request.method != 'POST':
         return err(405, "Method not allowed")
     apidata_post = request.POST.get('apidata')
@@ -4612,7 +4666,11 @@ def err(code, text):
     write_doc_for_session(session, 'polls', filename, json.dumps(apidata['polls']))
     e = NewRevisionDocEvent.objects.create(doc=doc, rev=doc.rev, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev)
     doc.save_with_history([e])
-    return HttpResponse("Done", status=200, content_type='text/plain')
+    return HttpResponse(
+        "Done",
+        status=200,
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
 
 @require_api_key
 @role_required('Recording Manager', 'Secretariat')
@@ -4627,11 +4685,17 @@ def api_upload_bluesheet(request):
           [{'name': 'Name', 'affiliation': 'Organization', }, ...]
     """
     def err(code, text):
-        return HttpResponse(text, status=code, content_type='text/plain')
+        return HttpResponse(
+            text,
+            status=code,
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+        )
 
     if request.method != 'POST':
         return HttpResponseNotAllowed(
-            content="Method not allowed", content_type="text/plain", permitted_methods=('POST',)
+            content="Method not allowed",
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+            permitted_methods=('POST',),
         )
 
     session_id = request.POST.get('session_id', None)
@@ -4666,7 +4730,11 @@ def err(code, text):
         save_err = save_bluesheet(request, session, file)
     if save_err:
         return err(400, save_err)
-    return HttpResponse("Done", status=200, content_type='text/plain')
+    return HttpResponse(
+        "Done",
+        status=200,
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
 
 
 def important_dates(request, num=None, output_format=None):
@@ -5090,7 +5158,7 @@ def import_session_minutes(request, session_id, num):
             except SessionNotScheduledError:
                 return HttpResponseGone(
                     "Cannot import minutes for an unscheduled session. Please check the session ID.",
-                    content_type="text/plain",
+                    content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
                 )
             except SaveMaterialsError as err:
                 form.add_error(None, str(err))
diff --git a/ietf/nomcom/views.py b/ietf/nomcom/views.py
index d34126b1e7..c04e13f92b 100644
--- a/ietf/nomcom/views.py
+++ b/ietf/nomcom/views.py
@@ -1017,7 +1017,10 @@ def view_feedback_nominee(request, year, nominee_id):
                                          'positions': ','.join([str(p) for p in feedback.positions.all()]),
                                          },
                                         request=request)
-            response = HttpResponse(response, content_type='text/plain')
+            response = HttpResponse(
+                response,
+                content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+            )
             response['Content-Disposition'] = f'attachment; filename="{fn}"'
             return response
         elif submit == 'reclassify':
diff --git a/ietf/submit/views.py b/ietf/submit/views.py
index 2b9b55c00e..043b613016 100644
--- a/ietf/submit/views.py
+++ b/ietf/submit/views.py
@@ -755,4 +755,7 @@ def get_submission_or_404(submission_id, access_token=None):
 
 def async_poke_test(request):
     result = poke.delay()
-    return HttpResponse(f'Poked {result}', content_type='text/plain')
+    return HttpResponse(
+        f'Poked {result}',
+        content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+    )
diff --git a/ietf/utils/decorators.py b/ietf/utils/decorators.py
index 56c28c4b19..5e94dda91d 100644
--- a/ietf/utils/decorators.py
+++ b/ietf/utils/decorators.py
@@ -48,7 +48,7 @@ def require_api_key(f):
     @wraps(f)
     def _wrapper(request, *args, **kwargs):
         def err(code, text):
-            return HttpResponse(text, status=code, content_type='text/plain')
+            return HttpResponse(text, status=code, content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}")
         # Check method and get hash
         if request.method == 'POST':
             hash = request.POST.get('apikey')

From b766eaf54450cb494598474eeb9389ac7cb176de Mon Sep 17 00:00:00 2001
From: Rich Salz <rsalz@akamai.com>
Date: Thu, 24 Jul 2025 08:22:26 -0400
Subject: [PATCH 406/656] fix: Include text to set/update photo (#9186)

* fix: Include text to set/update photo

* fixup! fix: Include text to set/update photo

* fixup! fixup! fix: Include text to set/update photo

* fix: only show photo instructions to users with roles

* fix: only show Photo section to those with photos or roles

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/templates/registration/edit_profile.html | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/ietf/templates/registration/edit_profile.html b/ietf/templates/registration/edit_profile.html
index 1837016b15..1e4ab169e1 100644
--- a/ietf/templates/registration/edit_profile.html
+++ b/ietf/templates/registration/edit_profile.html
@@ -32,12 +32,19 @@ <h1>Your account</h1>
                 <a class="btn btn-primary" href="{% url 'ietf.ietfauth.views.change_password' %}">Change password</a>
             </div>
         </div>
-        {% if person.photo %}
+        {% if person.photo or person.role_set.exists %}
             <div class="row mb-3">
                 <label class="col-sm-2 col-form-label fw-bold">
                     Photo
                 </label>
-                <div class="col-sm-10">{% include "person/photo.html" with person=person %}</div>
+                <div class="col-sm-10">
+                    {% if person.photo %}
+                    {% include "person/photo.html" with person=person %}
+                    {% endif %}
+                    {% if person.role_set.exists %}
+                    <p>Email <a href="support@ietf.org">support@ietf.org</a>
+                    to update your photo.</p></div>
+                    {% endif %}
             </div>
         {% endif %}
         <div class="row mb-3">

From e55162360c198765ae16f2baa8413ed1b502fa9c Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Thu, 24 Jul 2025 14:25:49 +0200
Subject: [PATCH 407/656] fix: Improvements to registration admin views (#9185)

* fix: add tickets to Registration admin

* fix: more improvements to registration admin views

* fix: add ignore for mypy

* fix: remove unused import

* fix: add new admin templates to TEST_TEMPLATE_IGNORE

* fix: use full path for TEST_TEMPLATE_IGNORE
---
 ietf/meeting/admin.py                         | 56 ++++++++++++++++---
 ietf/settings.py                              |  2 +
 .../meeting/Registration/change_list.html     | 10 ++++
 .../RegistrationTicket/change_list.html       | 10 ++++
 4 files changed, 70 insertions(+), 8 deletions(-)
 create mode 100644 ietf/templates/admin/meeting/Registration/change_list.html
 create mode 100644 ietf/templates/admin/meeting/RegistrationTicket/change_list.html

diff --git a/ietf/meeting/admin.py b/ietf/meeting/admin.py
index 1c5d5c67b5..d886a9a4b6 100644
--- a/ietf/meeting/admin.py
+++ b/ietf/meeting/admin.py
@@ -3,11 +3,13 @@
 
 
 from django.contrib import admin
+from django.db.models import Count
 
 from ietf.meeting.models import (Attended, Meeting, Room, Session, TimeSlot, Constraint, Schedule,
     SchedTimeSessAssignment, ResourceAssociation, FloorPlan, UrlResource,
     SessionPresentation, ImportantDate, SlideSubmission, SchedulingEvent, BusinessConstraint,
-    ProceedingsMaterial, MeetingHost, Registration, RegistrationTicket)
+    ProceedingsMaterial, MeetingHost, Registration, RegistrationTicket,
+    AttendanceTypeName)
 
 
 class UrlResourceAdmin(admin.ModelAdmin):
@@ -219,8 +221,9 @@ class MeetingFilter(admin.SimpleListFilter):
     parameter_name = 'meeting_id'
 
     def lookups(self, request, model_admin):
-        # Your queryset to limit choices
-        choices = Meeting.objects.filter(type='ietf').values_list('id', 'number')
+        # only include meetings with registration records
+        meetings = Meeting.objects.filter(type='ietf').annotate(reg_count=Count('registration')).filter(reg_count__gt=0).order_by('-date')
+        choices = meetings.values_list('id', 'number')
         return choices
 
     def queryset(self, request, queryset):
@@ -228,23 +231,60 @@ def queryset(self, request, queryset):
             return queryset.filter(meeting__id=self.value())
         return queryset
 
+class AttendanceFilter(admin.SimpleListFilter):
+    title = 'Attendance Type'
+    parameter_name = 'attendance_type'
+
+    def lookups(self, request, model_admin):
+        choices = AttendanceTypeName.objects.all().values_list('slug', 'name')
+        return choices
+
+    def queryset(self, request, queryset):
+        if self.value():
+            return queryset.filter(tickets__attendance_type__slug=self.value()).distinct()
+        return queryset
+
 class RegistrationTicketInline(admin.TabularInline):
     model = RegistrationTicket
 
 class RegistrationAdmin(admin.ModelAdmin):
     model = Registration
-    # list_filter = [('meeting', Meeting.objects.filter(type='ietf')), ]
-    list_filter = [MeetingFilter, ]
-    list_display = ['meeting', 'first_name', 'last_name', 'affiliation', 'country_code', 'person', 'email', ]
-    search_fields = ['meeting__number', 'first_name', 'last_name', 'affiliation', 'country_code', 'email', ]
+    list_filter = [AttendanceFilter, MeetingFilter]
+    list_display = ['meeting', 'first_name', 'last_name', 'display_attendance', 'affiliation', 'country_code', 'email', ]
+    search_fields = ['first_name', 'last_name', 'affiliation', 'country_code', 'email', ]
     raw_id_fields = ['person']
     inlines = [RegistrationTicketInline, ]
+    ordering = ['-meeting__date', 'last_name']
+
+    def display_attendance(self, instance):
+        '''Only display the most significant ticket in the list.
+        To see all the tickets inspect the individual instance
+        '''
+        if instance.tickets.filter(attendance_type__slug='onsite').exists():
+            return 'onsite'
+        elif instance.tickets.filter(attendance_type__slug='remote').exists():
+            return 'remote'
+        elif instance.tickets.filter(attendance_type__slug='hackathon_onsite').exists():
+            return 'hackathon onsite'
+        elif instance.tickets.filter(attendance_type__slug='hackathon_remote').exists():
+            return 'hackathon remote'
+    display_attendance.short_description = "Attendance"  # type: ignore # https://github.com/python/mypy/issues/2087
+
 admin.site.register(Registration, RegistrationAdmin)
 
 class RegistrationTicketAdmin(admin.ModelAdmin):
     model = RegistrationTicket
     list_filter = ['attendance_type', ]
-    list_display = ['registration', 'attendance_type', 'ticket_type']
+    # not available until Django 5.2, the name of a related field, using the __ notation
+    # list_display = ['registration__meeting', 'registration', 'attendance_type', 'ticket_type', 'registration__email']
+    # list_select_related = ('registration',)
+    list_display = ['registration', 'attendance_type', 'ticket_type', 'display_meeting']
     search_fields = ['registration__first_name', 'registration__last_name', 'registration__email']
     raw_id_fields = ['registration']
+    ordering = ['-registration__meeting__date', 'registration__last_name']
+
+    def display_meeting(self, instance):
+        return instance.registration.meeting.number
+    display_meeting.short_description = "Meeting"  # type: ignore # https://github.com/python/mypy/issues/2087
+
 admin.site.register(RegistrationTicket, RegistrationTicketAdmin)
diff --git a/ietf/settings.py b/ietf/settings.py
index 64679ca1d8..3af01d76e6 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -729,6 +729,8 @@ def skip_unreadable_post(record):
     "*~",  # tilde temp-files
     "#*",  # files beginning with a hashmark
     "500.html",  # isn't loaded by regular loader, but checked by test_500_page()
+    "ietf/templates/admin/meeting/RegistrationTicket/change_list.html",
+    "ietf/templates/admin/meeting/Registration/change_list.html",
 ]
 
 TEST_COVERAGE_MAIN_FILE = os.path.join(BASE_DIR, "../release-coverage.json")
diff --git a/ietf/templates/admin/meeting/Registration/change_list.html b/ietf/templates/admin/meeting/Registration/change_list.html
new file mode 100644
index 0000000000..62784b2cb6
--- /dev/null
+++ b/ietf/templates/admin/meeting/Registration/change_list.html
@@ -0,0 +1,10 @@
+{% extends "admin/change_list.html" %}
+
+{% block search %}
+  {{ block.super }} {# This includes the original search form #}
+  {% if cl.search_fields %} {# Only show if search is enabled for the model #}
+    <p class="search-help-text" style="font-size: 0.85em; color: #666; margin-top: 5px;">
+      Hint: Search by: {{ cl.search_fields|join:", "|capfirst }}.
+    </p>
+  {% endif %}
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/admin/meeting/RegistrationTicket/change_list.html b/ietf/templates/admin/meeting/RegistrationTicket/change_list.html
new file mode 100644
index 0000000000..62784b2cb6
--- /dev/null
+++ b/ietf/templates/admin/meeting/RegistrationTicket/change_list.html
@@ -0,0 +1,10 @@
+{% extends "admin/change_list.html" %}
+
+{% block search %}
+  {{ block.super }} {# This includes the original search form #}
+  {% if cl.search_fields %} {# Only show if search is enabled for the model #}
+    <p class="search-help-text" style="font-size: 0.85em; color: #666; margin-top: 5px;">
+      Hint: Search by: {{ cl.search_fields|join:", "|capfirst }}.
+    </p>
+  {% endif %}
+{% endblock %}
\ No newline at end of file

From ea8377db0d4ba8bc990c3f29386fba942f55c84f Mon Sep 17 00:00:00 2001
From: Tero Kivinen <kivinen@iki.fi>
Date: Mon, 28 Jul 2025 16:18:13 +0300
Subject: [PATCH 408/656] fix: Add Reviews history page to review teams group
 page. (fixes #9190) (#9191)

* Add Reviews history page to review teams group page.

* fix: Added test cases.

Fixed html to validate, moved sort text to data-text from comments.
Added test cases. fixes #9190
---
 ietf/group/tests_review.py                    | 128 ++++++++++++++++++
 ietf/group/urls.py                            |   1 +
 ietf/group/utils.py                           |   1 +
 ietf/group/views.py                           |  50 +++++++
 .../group/review_requests_history.html        |  90 ++++++++++++
 5 files changed, 270 insertions(+)
 create mode 100644 ietf/templates/group/review_requests_history.html

diff --git a/ietf/group/tests_review.py b/ietf/group/tests_review.py
index a03b806f8f..d671228953 100644
--- a/ietf/group/tests_review.py
+++ b/ietf/group/tests_review.py
@@ -815,3 +815,131 @@ def test_reset_next_reviewer(self):
             self.assertEqual(NextReviewerInTeam.objects.get(team=group).next_reviewer, reviewers[target_index].person)
             self.client.logout()
             target_index += 2
+
+class RequestsHistoryTests(TestCase):
+    def test_requests_history_overview_page(self):
+        # Make assigned assignment
+        review_req = ReviewRequestFactory(state_id='assigned')
+        assignment = ReviewAssignmentFactory(review_request=review_req,
+                                             state_id='assigned',
+                                             reviewer=EmailFactory(),
+                                             assigned_on = review_req.time)
+        group = review_req.team
+
+        for url in [urlreverse(ietf.group.views.review_requests_history,
+                               kwargs={ 'acronym': group.acronym }),
+                    urlreverse(ietf.group.views.review_requests_history,
+                               kwargs={ 'acronym': group.acronym ,
+                                        'group_type': group.type_id}),
+                    urlreverse(ietf.group.views.review_requests_history,
+                               kwargs={ 'acronym': group.acronym }) +
+                    '?since=3m',
+                    urlreverse(ietf.group.views.review_requests_history,
+                               kwargs={ 'acronym': group.acronym ,
+                                        'group_type': group.type_id }) +
+                    '?since=3m']:
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            self.assertContains(r, review_req.doc.name)
+            self.assertContains(r, 'Assigned')
+            self.assertContains(r, escape(assignment.reviewer.person.name))
+
+        url = urlreverse(ietf.group.views.review_requests_history,
+                         kwargs={ 'acronym': group.acronym })
+
+        assignment.state = ReviewAssignmentStateName.objects.get(slug="completed")
+        assignment.result = ReviewResultName.objects.get(slug="ready")
+        assignment.save()
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, review_req.doc.name)
+        self.assertContains(r, 'Assigned')
+        self.assertContains(r, 'Completed')
+
+    def test_requests_history_filter_page(self):
+        # First assignment as assigned
+        review_req = ReviewRequestFactory(state_id = 'assigned',
+                                          doc = DocumentFactory())
+        assignment = ReviewAssignmentFactory(review_request = review_req,
+                                             state_id = 'assigned',
+                                             reviewer = EmailFactory(),
+                                             assigned_on = review_req.time)
+        group = review_req.team
+
+        # Second assignment in same group as accepted
+        review_req2 = ReviewRequestFactory(state_id = 'assigned',
+                                           team = review_req.team,
+                                           doc = DocumentFactory())
+        assignment2 = ReviewAssignmentFactory(review_request = review_req2,
+                                              state_id='accepted',
+                                              reviewer = EmailFactory(),
+                                              assigned_on = review_req2.time)
+
+        # Modify the assignment to be completed, and mark it ready
+        assignment2.state = ReviewAssignmentStateName.objects.get(slug="completed")
+        assignment2.result = ReviewResultName.objects.get(slug="ready")
+        assignment2.save()
+
+        # Check that we have all information when we do not filter
+        url = urlreverse(ietf.group.views.review_requests_history,
+                         kwargs={ 'acronym': group.acronym })
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, review_req.doc.name)
+        self.assertContains(r, review_req2.doc.name)
+        self.assertContains(r, 'Assigned')
+        self.assertContains(r, 'Accepted')
+        self.assertContains(r, 'Completed')
+        self.assertContains(r, 'Ready')
+        self.assertContains(r, escape(assignment.reviewer.person.name))
+        self.assertContains(r, escape(assignment2.reviewer.person.name))
+
+        # Check first reviewer history
+        for url in [urlreverse(ietf.group.views.review_requests_history,
+                               kwargs={ 'acronym': group.acronym }) +
+                    '?reviewer_email=' + str(assignment.reviewer),
+                    urlreverse(ietf.group.views.review_requests_history,
+                               kwargs={ 'acronym': group.acronym ,
+                                        'group_type': group.type_id}) +
+                    '?reviewer_email=' + str(assignment.reviewer)]:
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            self.assertContains(r, review_req.doc.name)
+            self.assertNotContains(r, review_req2.doc.name)
+            self.assertContains(r, 'Assigned')
+            self.assertNotContains(r, 'Accepted')
+            self.assertNotContains(r, 'Completed')
+            self.assertNotContains(r, 'Ready')
+            self.assertContains(r, escape(assignment.reviewer.person.name))
+            self.assertNotContains(r, escape(assignment2.reviewer.person.name))
+
+        # Check second reviewer history
+        for url in [urlreverse(ietf.group.views.review_requests_history,
+                               kwargs={ 'acronym': group.acronym }) +
+                    '?reviewer_email=' + str(assignment2.reviewer),
+                    urlreverse(ietf.group.views.review_requests_history,
+                               kwargs={ 'acronym': group.acronym ,
+                                        'group_type': group.type_id}) +
+                    '?reviewer_email=' + str(assignment2.reviewer)]:
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            self.assertNotContains(r, review_req.doc.name)
+            self.assertContains(r, review_req2.doc.name)
+            self.assertNotContains(r, 'Assigned')
+            self.assertContains(r, 'Accepted')
+            self.assertContains(r, 'Completed')
+            self.assertContains(r, 'Ready')
+            self.assertNotContains(r, escape(assignment.reviewer.person.name))
+            self.assertContains(r, escape(assignment2.reviewer.person.name))
+
+        # Check for reviewer that does not have anything
+        url = urlreverse(ietf.group.views.review_requests_history,
+                         kwargs={ 'acronym': group.acronym }) + '?reviewer_email=nobody@nowhere.example.org'
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertNotContains(r, review_req.doc.name)
+        self.assertNotContains(r, 'Assigned')
+        self.assertNotContains(r, 'Accepted')
+        self.assertNotContains(r, 'Completed')
diff --git a/ietf/group/urls.py b/ietf/group/urls.py
index 1824564c4d..8354aba063 100644
--- a/ietf/group/urls.py
+++ b/ietf/group/urls.py
@@ -24,6 +24,7 @@
     url(r'^about/status/edit/$', views.group_about_status_edit),
     url(r'^about/status/meeting/(?P<num>\d+)/$', views.group_about_status_meeting),
     url(r'^history/$',views.history),
+    url(r'^requestshistory/$',views.review_requests_history),
     url(r'^history/addcomment/$',views.add_comment),
     url(r'^email/$', views.email),
     url(r'^deps\.json$', views.dependencies),
diff --git a/ietf/group/utils.py b/ietf/group/utils.py
index dcf9d83e6f..29cfff2c2d 100644
--- a/ietf/group/utils.py
+++ b/ietf/group/utils.py
@@ -236,6 +236,7 @@ def construct_group_menu_context(request, group, selected, group_type, others):
         import ietf.group.views
         entries.append(("Review requests", urlreverse(ietf.group.views.review_requests, kwargs=kwargs)))
         entries.append(("Reviewers", urlreverse(ietf.group.views.reviewer_overview, kwargs=kwargs)))
+        entries.append(("Reviews History", urlreverse(ietf.group.views.review_requests_history, kwargs=kwargs)))
 
     if group.features.has_meetings:
         entries.append(("Meetings", urlreverse("ietf.group.views.meetings", kwargs=kwargs)))
diff --git a/ietf/group/views.py b/ietf/group/views.py
index 0c89302c6a..bc79599722 100644
--- a/ietf/group/views.py
+++ b/ietf/group/views.py
@@ -686,6 +686,56 @@ def history(request, acronym, group_type=None):
                       "can_add_comment": can_add_comment,
                   }))
 
+def review_requests_history(request, acronym, group_type=None):
+    group = get_group_or_404(acronym, group_type)
+    if not group.features.has_reviews:
+        raise Http404
+
+    reviewer_email = request.GET.get("reviewer_email", None)
+
+    if reviewer_email:
+        history = ReviewAssignment.history.model.objects.filter(
+            review_request__team__acronym=acronym,
+            reviewer=reviewer_email)
+    else:
+        history = ReviewAssignment.history.model.objects.filter(
+            review_request__team__acronym=acronym)
+        reviewer_email = ''
+        
+    since_choices = [
+        (None, "1 month"),
+        ("3m", "3 months"),
+        ("6m", "6 months"),
+        ("1y", "1 year"),
+        ("2y", "2 years"),
+        ("all", "All"),
+    ]
+    since = request.GET.get("since", None)
+    
+    if since not in [key for key, label in since_choices]:
+        since = None
+
+    if since != "all":
+        date_limit = {
+            None: datetime.timedelta(days=31),
+            "3m": datetime.timedelta(days=31 * 3),
+            "6m": datetime.timedelta(days=180),
+            "1y": datetime.timedelta(days=365),
+            "2y": datetime.timedelta(days=2 * 365),
+        }[since]
+
+        history = history.filter(review_request__time__gte=datetime_today(DEADLINE_TZINFO) - date_limit)
+
+    return render(request, 'group/review_requests_history.html',
+                  construct_group_menu_context(request, group, "reviews history", group_type, {
+                      "group": group,
+                      "acronym": acronym,
+                      "history": history,
+                      "since_choices": since_choices,
+                      "since": since,
+                      "reviewer_email": reviewer_email
+                  }))
+
 def materials(request, acronym, group_type=None):
     group = get_group_or_404(acronym, group_type)
     if not group.features.has_nonsession_materials:
diff --git a/ietf/templates/group/review_requests_history.html b/ietf/templates/group/review_requests_history.html
new file mode 100644
index 0000000000..1b1fb4d263
--- /dev/null
+++ b/ietf/templates/group/review_requests_history.html
@@ -0,0 +1,90 @@
+{% extends "group/group_base.html" %}
+{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{% load origin %}
+{% load tz %}
+{% load ietf_filters person_filters textfilters %}
+{% load static %}
+{% block pagehead %}
+    <link rel="stylesheet" href="{% static "ietf/css/list.css" %}">
+{% endblock %}
+{% block group_content %}
+    {% origin %}
+    {% if reviewer_email %}
+    <h2 class="my-3">Review requests history of {{ reviewer_email }}</h2>
+    {% else %}
+    <h2 class="my-3">Review requests history</h2>
+    {% endif %}
+    <form class="reviews-history-filter" action="#reviewshistory">
+      <div>
+	<label for="reviewer_email"
+	       class="query-input-label">Reviewer filter:</label>
+	<input type="text" class="query-input" id="reviewer_email"
+	       value="{{ reviewer_email }}" name="reviewer_email">
+	<button class="btn btn-primary" type="submit">Filter</button>
+      </div>
+      <div>
+	Past:
+	<div class="btn-group btn-group-sm" role="group">
+          {% for key, label in since_choices %}
+          <button class="btn btn-outline-primary {% if since == key %}active{% endif %}"
+                  {% if key %}name="since" value="{{ key }}"{% endif %}
+                  type="submit">
+            {{ label }}
+          </button>
+          {% endfor %}
+	</div>
+      </div>
+    </form>
+
+    <table class="table table-sm table-striped tablesorter">
+        <thead>
+            <tr>
+                <th scope="col" data-sort="date">Date (UTC)</th>
+                <th scope="col" data-sort="by">By</th>
+                <th scope="col" data-sort="doc">Document</th>
+                <th scope="col" data-sort="state">State</th>
+                <th scope="col" data-sort="review">Reviewer</th>
+                <th scope="col" data-sort="result">Result</th>
+                <th scope="col" data-sort="description">Description</th>
+            </tr>
+        </thead>
+        {% if history %}
+            <tbody>
+                {% for h in history %}
+                    <tr>
+                        <td>{{ h.history_date|utc|date:"Y-m-d H:i:s" }}</td>
+                        <td>{% person_link h.history_user.person %}</td>
+                        <td>{% if h.reviewed_rev %}
+			  <a href="{% url "ietf.doc.views_doc.document_main" name=h.review_request.doc.name rev=h.reviewed_rev %}">
+                        {{ h.review_request.doc.name }}-{{ h.reviewed_rev }}
+			  </a>
+			  {% else %}
+			  <a href="{% url "ietf.doc.views_doc.document_main" name=h.review_request.doc.name %}">{{ h.review_request.doc.name }}</a>
+			  {% endif %}
+			</td>
+                        <td data-text="{{ h.state }}">
+			  <a href="{% url "ietf.doc.views_review.review_request" name=h.review_request.doc.name request_id=h.review_request.pk %}">{{ h.state }} </a>
+			</td>
+                        <td data-text="{{ h.reviewer }}">
+			  {% person_link h.reviewer.person %}
+			  <a href="?reviewer_email={{h.reviewer.email_address}}#reviewshistory">
+			    (set&nbsp;as&nbsp;filter)
+			  </a>
+			</td>
+                        <td data-text="{{ h.result }}">
+			  {% if h.review %}<a href="{{ h.review.get_absolute_url }}">
+			    {{ h.result }}</a>
+			  {% else %}
+			  {{ h.result }}
+			  {% endif %}
+			</td>
+                        <td>{{ h.history_change_reason }}</td>
+                    </tr>
+                {% endfor %}
+            </tbody>
+        {% endif %}
+    </table>
+{% endblock %}
+{% block js %}
+    <script src="{% static "ietf/js/list.js" %}"></script>
+{% endblock %}

From 4762e252552587bfc80c91244c9acb9cc59d820b Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 4 Aug 2025 15:02:44 -0300
Subject: [PATCH 409/656] test: replace mock with unittest.mock (#9286)

---
 ietf/api/tests.py               | 2 +-
 ietf/community/tests.py         | 2 +-
 ietf/doc/tests.py               | 2 +-
 ietf/doc/tests_ballot.py        | 2 +-
 ietf/doc/tests_draft.py         | 2 +-
 ietf/doc/tests_material.py      | 2 +-
 ietf/doc/tests_review.py        | 2 +-
 ietf/doc/tests_tasks.py         | 2 +-
 ietf/group/tests.py             | 2 +-
 ietf/group/tests_info.py        | 2 +-
 ietf/idindex/tests.py           | 2 +-
 ietf/ipr/management/tests.py    | 2 +-
 ietf/ipr/tests.py               | 2 +-
 ietf/meeting/tests_models.py    | 2 +-
 ietf/meeting/tests_tasks.py     | 2 +-
 ietf/meeting/tests_utils.py     | 2 +-
 ietf/meeting/tests_views.py     | 2 +-
 ietf/message/tests.py           | 2 +-
 ietf/nomcom/management/tests.py | 2 +-
 ietf/nomcom/tests.py            | 2 +-
 ietf/person/tests.py            | 2 +-
 ietf/review/tests.py            | 2 +-
 ietf/submit/tests.py            | 2 +-
 ietf/sync/tests.py              | 2 +-
 ietf/utils/management/tests.py  | 2 +-
 ietf/utils/tests.py             | 2 +-
 requirements.txt                | 2 --
 27 files changed, 26 insertions(+), 28 deletions(-)

diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 93a2195467..93515dd0cb 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -5,7 +5,7 @@
 import datetime
 import json
 import html
-import mock
+from unittest import mock
 import os
 import sys
 
diff --git a/ietf/community/tests.py b/ietf/community/tests.py
index 1255ba46eb..04f1433d61 100644
--- a/ietf/community/tests.py
+++ b/ietf/community/tests.py
@@ -1,7 +1,7 @@
 # Copyright The IETF Trust 2016-2023, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-import mock
+from unittest import mock
 from pyquery import PyQuery
 
 from django.test.utils import override_settings
diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index 1229df46c5..d3fba03bcc 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -10,7 +10,7 @@
 from django.http import HttpRequest
 import lxml
 import bibtexparser
-import mock
+from unittest import mock
 import json
 import copy
 import random
diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py
index ec23f3d491..810ee598f6 100644
--- a/ietf/doc/tests_ballot.py
+++ b/ietf/doc/tests_ballot.py
@@ -3,7 +3,7 @@
 
 
 import datetime
-import mock
+from unittest import mock
 
 from pyquery import PyQuery
 
diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py
index 4753c4ff0c..576feb0582 100644
--- a/ietf/doc/tests_draft.py
+++ b/ietf/doc/tests_draft.py
@@ -5,7 +5,7 @@
 import os
 import datetime
 import io
-import mock
+from unittest import mock
 
 from collections import Counter
 from pathlib import Path
diff --git a/ietf/doc/tests_material.py b/ietf/doc/tests_material.py
index c87341c95b..04779bdaf1 100644
--- a/ietf/doc/tests_material.py
+++ b/ietf/doc/tests_material.py
@@ -6,7 +6,7 @@
 import shutil
 import io
 
-from mock import call, patch
+from unittest.mock import call, patch
 from pathlib import Path
 from pyquery import PyQuery
 
diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py
index 9850beca75..8c1fc99ffe 100644
--- a/ietf/doc/tests_review.py
+++ b/ietf/doc/tests_review.py
@@ -8,7 +8,7 @@
 import os
 import shutil
 
-from mock import patch, Mock
+from unittest.mock import patch, Mock
 from requests import Response
 
 from django.apps import apps
diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py
index 8a6ffa8be1..29689cd596 100644
--- a/ietf/doc/tests_tasks.py
+++ b/ietf/doc/tests_tasks.py
@@ -2,7 +2,7 @@
 
 import debug    # pyflakes:ignore
 import datetime
-import mock
+from unittest import mock
 
 from pathlib import Path
 
diff --git a/ietf/group/tests.py b/ietf/group/tests.py
index 31f8cc45b5..229744388c 100644
--- a/ietf/group/tests.py
+++ b/ietf/group/tests.py
@@ -3,7 +3,7 @@
 
 import datetime
 import json
-import mock
+from unittest import mock
 
 from django.urls import reverse as urlreverse
 from django.db.models import Q
diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py
index aaf937ee43..eb85860ece 100644
--- a/ietf/group/tests_info.py
+++ b/ietf/group/tests_info.py
@@ -6,7 +6,7 @@
 import datetime
 import io
 import bleach
-import mock
+from unittest import mock
 
 from unittest.mock import call, patch
 from pathlib import Path
diff --git a/ietf/idindex/tests.py b/ietf/idindex/tests.py
index 5cc7a7b3bb..ba6100550d 100644
--- a/ietf/idindex/tests.py
+++ b/ietf/idindex/tests.py
@@ -3,7 +3,7 @@
 
 
 import datetime
-import mock
+from unittest import mock
 
 from pathlib import Path
 from tempfile import TemporaryDirectory
diff --git a/ietf/ipr/management/tests.py b/ietf/ipr/management/tests.py
index d84b0cfef4..d7acd65042 100644
--- a/ietf/ipr/management/tests.py
+++ b/ietf/ipr/management/tests.py
@@ -1,7 +1,7 @@
 # Copyright The IETF Trust 2021, All Rights Reserved
 # -*- coding: utf-8 -*-
 """Tests of ipr management commands"""
-import mock
+from unittest import mock
 import sys
 
 from django.core.management import call_command
diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py
index e0d00b5d1a..74fa540126 100644
--- a/ietf/ipr/tests.py
+++ b/ietf/ipr/tests.py
@@ -3,7 +3,7 @@
 
 
 import datetime
-import mock
+from unittest import mock
 import re
 
 from pyquery import PyQuery
diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py
index e333ddad9a..869d9ec814 100644
--- a/ietf/meeting/tests_models.py
+++ b/ietf/meeting/tests_models.py
@@ -3,7 +3,7 @@
 """Tests of models in the Meeting application"""
 import datetime
 
-from mock import patch
+from unittest.mock import patch
 
 from django.conf import settings
 from django.test import override_settings
diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py
index 66de212899..0c442c4bf7 100644
--- a/ietf/meeting/tests_tasks.py
+++ b/ietf/meeting/tests_tasks.py
@@ -1,7 +1,7 @@
 # Copyright The IETF Trust 2025, All Rights Reserved
 
 import datetime
-from mock import patch, call
+from unittest.mock import patch, call
 from ietf.utils.test_utils import TestCase
 from ietf.utils.timezone import date_today
 from .factories import MeetingFactory
diff --git a/ietf/meeting/tests_utils.py b/ietf/meeting/tests_utils.py
index 391e017d68..7dd8f435e1 100644
--- a/ietf/meeting/tests_utils.py
+++ b/ietf/meeting/tests_utils.py
@@ -7,7 +7,7 @@
 import json
 import jsonschema
 from json import JSONDecodeError
-from mock import patch, Mock
+from unittest.mock import patch, Mock
 
 from django.http import HttpResponse, JsonResponse
 from ietf.meeting.factories import MeetingFactory, RegistrationFactory, RegistrationTicketFactory
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 1aac2a6523..96a29c2297 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -12,7 +12,7 @@
 import requests_mock
 
 from unittest import skipIf
-from mock import call, patch, PropertyMock
+from unittest.mock import call, patch, PropertyMock
 from pyquery import PyQuery
 from lxml.etree import tostring
 from io import StringIO, BytesIO
diff --git a/ietf/message/tests.py b/ietf/message/tests.py
index a677d5477e..e1bad9a1e6 100644
--- a/ietf/message/tests.py
+++ b/ietf/message/tests.py
@@ -1,7 +1,7 @@
 # Copyright The IETF Trust 2013-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
 import datetime
-import mock
+from unittest import mock
 
 from smtplib import SMTPException
 
diff --git a/ietf/nomcom/management/tests.py b/ietf/nomcom/management/tests.py
index 7bda2b5aa5..08c0e1fe32 100644
--- a/ietf/nomcom/management/tests.py
+++ b/ietf/nomcom/management/tests.py
@@ -1,7 +1,7 @@
 # Copyright The IETF Trust 2021, All Rights Reserved
 # -*- coding: utf-8 -*-
 """Tests of nomcom management commands"""
-import mock
+from unittest import mock
 import sys
 
 from collections import namedtuple
diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py
index ea17da6707..cc2e0826d3 100644
--- a/ietf/nomcom/tests.py
+++ b/ietf/nomcom/tests.py
@@ -4,7 +4,7 @@
 
 import datetime
 import io
-import mock
+from unittest import mock
 import random
 import shutil
 
diff --git a/ietf/person/tests.py b/ietf/person/tests.py
index 61d9b0ed70..6326362fd8 100644
--- a/ietf/person/tests.py
+++ b/ietf/person/tests.py
@@ -4,7 +4,7 @@
 
 import datetime
 import json
-import mock
+from unittest import mock
 
 from io import StringIO, BytesIO
 from PIL import Image
diff --git a/ietf/review/tests.py b/ietf/review/tests.py
index e9ddbd47af..5dc8f11e8e 100644
--- a/ietf/review/tests.py
+++ b/ietf/review/tests.py
@@ -1,7 +1,7 @@
 # Copyright The IETF Trust 2019-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
 import datetime
-import mock
+from unittest import mock
 import debug # pyflakes:ignore
 
 from pyquery import PyQuery
diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index 7e70c55965..6b9002502b 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -5,7 +5,7 @@
 import datetime
 import email
 import io
-import mock
+from unittest import mock
 import os
 import re
 import sys
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index 14d65de0b2..182b6e24c4 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -6,7 +6,7 @@
 import io
 import json
 import datetime
-import mock
+from unittest import mock
 import quopri
 import requests
 
diff --git a/ietf/utils/management/tests.py b/ietf/utils/management/tests.py
index d704999cd1..38be464c7f 100644
--- a/ietf/utils/management/tests.py
+++ b/ietf/utils/management/tests.py
@@ -1,7 +1,7 @@
 # Copyright The IETF Trust 2013-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-import mock
+from unittest import mock
 
 from django.core.management import call_command, CommandError
 from django.test import override_settings
diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py
index ce1842236d..01433888fe 100644
--- a/ietf/utils/tests.py
+++ b/ietf/utils/tests.py
@@ -11,7 +11,7 @@
 import shutil
 import types
 
-from mock import call, patch
+from unittest.mock import call, patch
 from pyquery import PyQuery
 from typing import Dict, List       # pyflakes:ignore
 
diff --git a/requirements.txt b/requirements.txt
index 4eb573ce36..8ed354192a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -51,8 +51,6 @@ logging_tree>=1.9    # Used only by the showloggers management command
 lxml>=5.3.0
 markdown>=3.3.6
 types-markdown>=3.3.6
-mock>=4.0.3    # Used only by tests, of course
-types-mock>=4.0.3
 mypy~=1.7.0    # Version requirements determined by django-stubs.
 oic>=1.3    # Used only by tests
 Pillow>=9.1.0

From e0546b1543565c0a293d198db9b15f1dd5121600 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 4 Aug 2025 15:04:09 -0300
Subject: [PATCH 410/656] fix: blank=True for xml_version (#9285)

---
 .../0002_alter_submission_xml_version.py       | 18 ++++++++++++++++++
 ietf/submit/models.py                          |  2 +-
 2 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 ietf/submit/migrations/0002_alter_submission_xml_version.py

diff --git a/ietf/submit/migrations/0002_alter_submission_xml_version.py b/ietf/submit/migrations/0002_alter_submission_xml_version.py
new file mode 100644
index 0000000000..275e6efd95
--- /dev/null
+++ b/ietf/submit/migrations/0002_alter_submission_xml_version.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.23 on 2025-08-01 19:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("submit", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="submission",
+            name="xml_version",
+            field=models.CharField(blank=True, default=None, max_length=4, null=True),
+        ),
+    ]
diff --git a/ietf/submit/models.py b/ietf/submit/models.py
index 51f7541e31..1145f761b4 100644
--- a/ietf/submit/models.py
+++ b/ietf/submit/models.py
@@ -55,7 +55,7 @@ class Submission(models.Model):
     file_size = models.IntegerField(null=True, blank=True)
     document_date = models.DateField(null=True, blank=True)
     submission_date = models.DateField(default=date_today)
-    xml_version = models.CharField(null=True, max_length=4, default=None)
+    xml_version = models.CharField(null=True, blank=True, max_length=4, default=None)
 
     submitter = models.CharField(max_length=255, blank=True, help_text="Name and email of submitter, e.g. \"John Doe &lt;john@example.org&gt;\".")
 

From 827f4e74a1b9a8e872634f31e9484a3dc8cd0842 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 4 Aug 2025 15:09:15 -0300
Subject: [PATCH 411/656] fix: escape nulls in XML api responses (#9283)

* fix: escape nulls in XML api responses

* refactor: use \u2400 instead of \0

Less likely to lead to null injection down the road

* test: modern naming/python

* test: test null char handling

* test: remove unused vars
---
 ietf/api/__init__.py | 21 +++++++++++++++++++++
 ietf/api/tests.py    | 20 ++++++++++++++++++--
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py
index 9fadab8e6f..d70866083e 100644
--- a/ietf/api/__init__.py
+++ b/ietf/api/__init__.py
@@ -145,5 +145,26 @@ def dehydrate(self, bundle, for_list=True):
 
 
 class Serializer(tastypie.serializers.Serializer):
+    OPTION_ESCAPE_NULLS = "datatracker-escape-nulls"
+
     def format_datetime(self, data):
         return data.astimezone(datetime.timezone.utc).replace(tzinfo=None).isoformat(timespec="seconds") + "Z"
+
+    def to_simple(self, data, options):
+        options = options or {}
+        simple_data = super().to_simple(data, options)
+        if (
+            options.get(self.OPTION_ESCAPE_NULLS, False) 
+            and isinstance(simple_data, str)
+        ):
+            # replace nulls with unicode "symbol for null character", \u2400
+            simple_data = simple_data.replace("\x00", "\u2400")
+        return simple_data
+
+    def to_etree(self, data, options=None, name=None, depth=0):
+        # lxml does not escape nulls on its own, so ask to_simple() to do it.
+        # This is mostly (only?) an issue when generating errors responses for
+        # fuzzers.
+        options = options or {}
+        options[self.OPTION_ESCAPE_NULLS] = True
+        return super().to_etree(data, options, name, depth)
diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 93515dd0cb..865f877bfb 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -41,6 +41,7 @@
 from ietf.utils.models import DumpInfo
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, reload_db_objects
 
+from . import Serializer
 from .ietf_utils import is_valid_token, requires_api_token
 from .views import EmailIngestionError
 
@@ -1496,7 +1497,7 @@ def test_good_password(self):
         data = self.response_data(r)
         self.assertEqual(data["result"], "success")
 
-class TastypieApiTestCase(ResourceTestCaseMixin, TestCase):
+class TastypieApiTests(ResourceTestCaseMixin, TestCase):
     def __init__(self, *args, **kwargs):
         self.apps = {}
         for app_name in settings.INSTALLED_APPS:
@@ -1506,7 +1507,7 @@ def __init__(self, *args, **kwargs):
                 models_path = os.path.join(os.path.dirname(app.__file__), "models.py")
                 if os.path.exists(models_path):
                     self.apps[name] = app_name
-        super(TastypieApiTestCase, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
     def test_api_top_level(self):
         client = Client(Accept='application/json')
@@ -1541,6 +1542,21 @@ def test_all_model_resources_exist(self):
                     self.assertIn(model._meta.model_name, list(app_resources.keys()),
                         "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,))
 
+    def test_serializer_to_etree_handles_nulls(self):
+        """Serializer to_etree() should handle a null character"""
+        serializer = Serializer()
+        try:
+            serializer.to_etree("string with no nulls in it")
+        except ValueError:
+            self.fail("serializer.to_etree raised ValueError on an ordinary string")
+        try:
+            serializer.to_etree("string with a \x00 in it")
+        except ValueError:
+            self.fail(
+                "serializer.to_etree raised ValueError on a string "
+                "containing a null character"
+            )
+
 
 class RfcdiffSupportTests(TestCase):
 

From 8c4bff875398fb7fc7de624c8155c8377276cec0 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 4 Aug 2025 15:11:12 -0300
Subject: [PATCH 412/656] test: guard against empty path (#9282)

---
 ietf/utils/test_runner.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py
index d0a5496283..a9b2e5d572 100644
--- a/ietf/utils/test_runner.py
+++ b/ietf/utils/test_runner.py
@@ -432,7 +432,11 @@ def get_template_paths(apps=None) -> list[str]:
                 relative_path = pathlib.Path(
                     dirpath.removeprefix(templatepath).lstrip("/")
                 )
-                if apps and relative_path.parts[0] not in apps:
+                if (
+                    apps 
+                    and len(relative_path.parts) > 0
+                    and relative_path.parts[0] not in apps
+                ):
                     continue  # skip uninteresting apps
                 for filename in files:
                     file_path = project_path / filename

From 649de73cab6c3aed3e909280e80e02272bc362a8 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 4 Aug 2025 15:13:46 -0300
Subject: [PATCH 413/656] fix: validate review_requests_history params (#9281)

* test: test null chars in GET params

* fix: validate GET params
---
 ietf/group/tests_review.py | 39 +++++++++++++++++++++++++++++++++
 ietf/group/views.py        | 45 +++++++++++++++++++++++---------------
 2 files changed, 66 insertions(+), 18 deletions(-)

diff --git a/ietf/group/tests_review.py b/ietf/group/tests_review.py
index d671228953..89c755bb26 100644
--- a/ietf/group/tests_review.py
+++ b/ietf/group/tests_review.py
@@ -943,3 +943,42 @@ def test_requests_history_filter_page(self):
         self.assertNotContains(r, 'Assigned')
         self.assertNotContains(r, 'Accepted')
         self.assertNotContains(r, 'Completed')
+
+    def test_requests_history_invalid_filter_parameters(self):
+        # First assignment as assigned
+        review_req = ReviewRequestFactory(state_id="assigned", doc=DocumentFactory())
+        group = review_req.team
+        url = urlreverse(
+            "ietf.group.views.review_requests_history",
+            kwargs={"acronym": group.acronym},
+        )
+        invalid_reviewer_emails = [
+            "%00null@example.com",  # urlencoded null character
+            "null@exa%00mple.com",  # urlencoded null character
+            "\x00null@example.com",  # literal null character
+            "null@ex\x00ample.com",  # literal null character
+        ]
+        for invalid_email in invalid_reviewer_emails:
+            r = self.client.get(
+                url + f"?reviewer_email={invalid_email}"
+            )
+            self.assertEqual(
+                r.status_code, 
+                400,
+                f"should return a 400 response for reviewer_email={repr(invalid_email)}"
+            )
+
+        invalid_since_choices = [
+            "forever",  # not an option
+            "all\x00",  # literal null character
+            "a%00ll",  # urlencoded null character
+        ]
+        for invalid_since in invalid_since_choices:
+            r = self.client.get(
+                url + f"?since={invalid_since}"
+            )
+            self.assertEqual(
+                r.status_code, 
+                400,
+                f"should return a 400 response for since={repr(invalid_since)}"
+            )
diff --git a/ietf/group/views.py b/ietf/group/views.py
index bc79599722..3529b31f68 100644
--- a/ietf/group/views.py
+++ b/ietf/group/views.py
@@ -51,7 +51,13 @@
 from django.contrib.auth.decorators import login_required
 from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, TextField, Value
 from django.db.models.functions import Coalesce
-from django.http import HttpResponse, HttpResponseRedirect, Http404, JsonResponse
+from django.http import (
+    HttpResponse,
+    HttpResponseRedirect,
+    Http404,
+    JsonResponse,
+    HttpResponseBadRequest,
+)
 from django.shortcuts import render, redirect, get_object_or_404
 from django.template.loader import render_to_string
 from django.urls import reverse as urlreverse
@@ -96,11 +102,9 @@
 from ietf.review.policies import get_reviewer_queue_policy
 from ietf.review.utils import (can_manage_review_requests_for_team,
                                can_access_review_stats_for_team,
-
                                extract_revision_ordered_review_requests_for_documents_and_replaced,
                                assign_review_request_to_reviewer,
                                close_review_request,
-
                                suggested_review_requests_for_team,
                                unavailable_periods_to_list,
                                current_unavailable_periods_for_reviewers,
@@ -686,13 +690,30 @@ def history(request, acronym, group_type=None):
                       "can_add_comment": can_add_comment,
                   }))
 
+
+class RequestsHistoryParamsForm(forms.Form):
+    SINCE_CHOICES = (
+        (None, "1 month"),
+        ("3m", "3 months"),
+        ("6m", "6 months"),
+        ("1y", "1 year"),
+        ("2y", "2 years"),
+        ("all", "All"),
+    )
+
+    reviewer_email = forms.EmailField(required=False)
+    since = forms.ChoiceField(choices=SINCE_CHOICES, required=False)
+
 def review_requests_history(request, acronym, group_type=None):
     group = get_group_or_404(acronym, group_type)
     if not group.features.has_reviews:
         raise Http404
 
-    reviewer_email = request.GET.get("reviewer_email", None)
+    params = RequestsHistoryParamsForm(request.GET)
+    if not params.is_valid():
+        return HttpResponseBadRequest("Invalid parameters")
 
+    reviewer_email = params.cleaned_data["reviewer_email"] or None
     if reviewer_email:
         history = ReviewAssignment.history.model.objects.filter(
             review_request__team__acronym=acronym,
@@ -702,19 +723,7 @@ def review_requests_history(request, acronym, group_type=None):
             review_request__team__acronym=acronym)
         reviewer_email = ''
         
-    since_choices = [
-        (None, "1 month"),
-        ("3m", "3 months"),
-        ("6m", "6 months"),
-        ("1y", "1 year"),
-        ("2y", "2 years"),
-        ("all", "All"),
-    ]
-    since = request.GET.get("since", None)
-    
-    if since not in [key for key, label in since_choices]:
-        since = None
-
+    since = params.cleaned_data["since"] or None
     if since != "all":
         date_limit = {
             None: datetime.timedelta(days=31),
@@ -731,7 +740,7 @@ def review_requests_history(request, acronym, group_type=None):
                       "group": group,
                       "acronym": acronym,
                       "history": history,
-                      "since_choices": since_choices,
+                      "since_choices": params.SINCE_CHOICES,
                       "since": since,
                       "reviewer_email": reviewer_email
                   }))

From 39165a0b5d079459da601ea82f87c9981f6508b7 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 4 Aug 2025 15:25:54 -0300
Subject: [PATCH 414/656] fix: serve materials w/mixed-case exts (#9273)

* fix: serve materials w/ mixed-case exts

* fix: another endpoint+reorder regex
---
 ietf/meeting/urls.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py
index 5da24ddb6f..18b123b4d8 100644
--- a/ietf/meeting/urls.py
+++ b/ietf/meeting/urls.py
@@ -64,7 +64,7 @@ def get_redirect_url(self, *args, **kwargs):
 type_interim_patterns = [
     url(r'^agenda/(?P<acronym>[A-Za-z0-9-]+)-drafts.pdf$', views.session_draft_pdf),
     url(r'^agenda/(?P<acronym>[A-Za-z0-9-]+)-drafts.tgz$', views.session_draft_tarfile),
-    url(r'^materials/%(document)s(?P<ext>\.[a-z0-9]+)$' % settings.URL_REGEXPS, views.materials_document),
+    url(r'^materials/%(document)s(?P<ext>\.[A-Za-z0-9]+)$' % settings.URL_REGEXPS, views.materials_document),
     url(r'^materials/%(document)s/?$' % settings.URL_REGEXPS, views.materials_document),
     url(r'^agenda.json$', views.agenda_json)
 ]
@@ -85,7 +85,7 @@ def get_redirect_url(self, *args, **kwargs):
     url(r'^week-view(?:.html)?/?$', AgendaRedirectView.as_view(pattern_name='agenda', permanent=True)),
     url(r'^materials(?:.html)?/?$', views.materials),
     url(r'^request_minutes/?$', views.request_minutes),
-    url(r'^materials/%(document)s(?P<ext>\.[a-z0-9]+)?/?$' % settings.URL_REGEXPS, views.materials_document),
+    url(r'^materials/%(document)s(?P<ext>\.[A-Za-z0-9]+)?/?$' % settings.URL_REGEXPS, views.materials_document),
     url(r'^session/?$', views.materials_editable_groups),
     url(r'^proceedings(?:.html)?/?$', views.proceedings),
     url(r'^proceedings(?:.html)?/finalize/?$', views.finalize_proceedings),

From afb0d2d245a11384d73e8f0cc0d31150dd91f80a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 4 Aug 2025 17:16:47 -0300
Subject: [PATCH 415/656] chore(deps): pin jsonfield version (#9267)

At least nominally, 3.2.0 requires py3.10. Package is deprecated.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 8ed354192a..1b00cf81a2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -44,7 +44,7 @@ hashids>=1.3.1
 html2text>=2020.1.16    # Used only to clean comment field of secr/sreq
 html5lib>=1.1    # Only used in tests
 inflect>= 6.0.2
-jsonfield>=3.1.0    # for SubmissionCheck.  This is https://github.com/bradjasper/django-jsonfield/.
+jsonfield>=3.1.0,<3.2.0  # 3.2.0 needs py3.10; deprecated-replace with Django JSONField
 jsonschema[format]>=4.2.1
 jwcrypto>=1.2    # for signed notifications - this is aspirational, and is not really used.
 logging_tree>=1.9    # Used only by the showloggers management command

From 6494ce880631ce798424a3a57db88f50f6ebf370 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Mon, 4 Aug 2025 20:29:21 +0000
Subject: [PATCH 416/656] ci: update base image target version to 20250804T2017

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 8317195446..662aee950f 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250624T1543
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250804T2017
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 1f2e39a0a2..d2c3dd6fc9 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250624T1543
+20250804T2017

From e220bc89b464aab54e11c6698f074ed51982c715 Mon Sep 17 00:00:00 2001
From: Eric Vyncke <eric@vyncke.org>
Date: Mon, 4 Aug 2025 22:43:44 +0200
Subject: [PATCH 417/656] Add link to reviewers's reviews (#9272)

---
 ietf/templates/person/profile.html | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ietf/templates/person/profile.html b/ietf/templates/person/profile.html
index 1424f037a1..a78a90412f 100644
--- a/ietf/templates/person/profile.html
+++ b/ietf/templates/person/profile.html
@@ -50,7 +50,11 @@ <h2 class="mt-5" id="roles-{{ forloop.counter }}">Roles</h2>
                 <tbody>
                     {% for role in person.role_set.all|active_roles %}
                         <tr>
-                            <td>{{ role.name.name }}</td>
+                            <td>{{ role.name.name }}
+                                {% if role.name.name == 'Reviewer' %}
+                                 (<a href="{% url 'ietf.group.views.review_requests_history' acronym=role.group.acronym %}?reviewer_email={{ role.email.address }}&since=1y">See reviews</a>)
+                                {% endif %}
+                            </td>
                             <td>
                                 <a href="{% url 'ietf.group.views.group_about' acronym=role.group.acronym %}">{{ role.group.name }}</a>
                                 (<a href="{% url 'ietf.group.views.group_about' acronym=role.group.acronym %}">{{ role.group.acronym }}</a>)

From e3b87d9459c597731d828482bd13041931fe0a2d Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 4 Aug 2025 22:28:31 -0300
Subject: [PATCH 418/656] chore: different celery path for sandboxes (#9300)

* chore: different celery path for sandboxes

* chore: typo
---
 docker/scripts/app-init-celery.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docker/scripts/app-init-celery.sh b/docker/scripts/app-init-celery.sh
index 5788b943da..17925633d2 100755
--- a/docker/scripts/app-init-celery.sh
+++ b/docker/scripts/app-init-celery.sh
@@ -92,7 +92,14 @@ fi
 
 USER_BIN_PATH="/home/dev/.local/bin"
 WATCHMEDO="$USER_BIN_PATH/watchmedo"
-CELERY="$USER_BIN_PATH/celery"
+# Find a celery that works
+if [[ -x "$USER_BIN_PATH/celery" ]]; then
+    # This branch is used for dev
+    CELERY="$USER_BIN_PATH/celery"
+else
+    # This branch is used for sandbox instances
+    CELERY="/usr/local/bin/celery"
+fi
 trap 'trap "" TERM; cleanup' TERM
 # start celery in the background so we can trap the TERM signal
 if [[ -n "${DEV_MODE}" && -x "${WATCHMEDO}" ]]; then

From b8e135b928f9d67c83e6ef6fda6c273fdb106748 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 5 Aug 2025 14:38:30 -0300
Subject: [PATCH 419/656] chore: remove find_github_backup_info.py (#9307)

---
 .../commands/find_github_backup_info.py       | 74 -------------------
 requirements.txt                              |  1 -
 2 files changed, 75 deletions(-)
 delete mode 100644 ietf/doc/management/commands/find_github_backup_info.py

diff --git a/ietf/doc/management/commands/find_github_backup_info.py b/ietf/doc/management/commands/find_github_backup_info.py
deleted file mode 100644
index f1f71452df..0000000000
--- a/ietf/doc/management/commands/find_github_backup_info.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# Copyright The IETF Trust 2020, All Rights Reserved
-
-
-import github3
-
-from collections import Counter
-from urllib.parse import urlparse
-
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
-
-from ietf.doc.models import DocExtResource
-from ietf.group.models import GroupExtResource
-from ietf.person.models import PersonExtResource
-
-# TODO: Think more about submodules. This currently will only take top level repos, with the assumption that the clone will include arguments to grab all the submodules. 
-# As a consequence, we might end up pulling more than we need (or that the org or user expected)
-# Make sure this is what we want.
-
-class Command(BaseCommand):
-    help = ('Locate information about github repositories to backup')
-
-    def add_arguments(self, parser):
-        parser.add_argument('--verbose', dest='verbose', action='store_true', help='Show counts of types of repositories')
-
-    def handle(self, *args, **options):
-
-        if not (hasattr(settings,'GITHUB_BACKUP_API_KEY') and settings.GITHUB_BACKUP_API_KEY):
-            raise CommandError("ERROR: can't find GITHUB_BACKUP_API_KEY") # TODO: at >= py3.1, use returncode
-
-        github = github3.login(token = settings.GITHUB_BACKUP_API_KEY)
-        owners = dict()
-        repos = set()
-
-        for cls in (DocExtResource, GroupExtResource, PersonExtResource):
-            for res in cls.objects.filter(name_id__in=('github_repo','github_org')):
-                path_parts = urlparse(res.value).path.strip('/').split('/')
-                if not path_parts or not path_parts[0]:
-                    continue
-
-                owner = path_parts[0]
-
-                if owner not in owners:
-                    try:
-                        gh_owner = github.user(username=owner)
-                        owners[owner] = gh_owner
-                    except github3.exceptions.NotFoundError:
-                        continue 
-
-                if gh_owner.type in ('User', 'Organization'):
-                    if len(path_parts) > 1:
-                        repo = path_parts[1]
-                        if (owner, repo) not in repos:
-                            try:
-                                github.repository(owner,repo)
-                                repos.add( (owner, repo) )
-                            except github3.exceptions.NotFoundError:
-                                continue
-                    else:
-                        for repo in github.repositories_by(owner):
-                            repos.add( (owner, repo.name) )
-
-        owner_types = Counter([owners[owner].type for owner in owners])
-        if options['verbose']:
-            self.stdout.write("Owners:")
-            for key in owner_types:
-                self.stdout.write("    %s: %s"%(key,owner_types[key]))
-            self.stdout.write("Repositories: %d" % len(repos))
-            for repo in sorted(repos):
-                self.stdout.write("    https://github.com/%s/%s" % repo )
-        else:
-            for repo in sorted(repos):
-                self.stdout.write("%s/%s" % repo )
-
diff --git a/requirements.txt b/requirements.txt
index 1b00cf81a2..f82bfc4101 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -38,7 +38,6 @@ drf-spectacular>=0.27
 drf-standardized-errors[openapi] >= 0.14
 types-docutils>=0.18.1
 factory-boy>=3.3
-github3.py>=3.2.0
 gunicorn>=20.1.0
 hashids>=1.3.1
 html2text>=2020.1.16    # Used only to clean comment field of secr/sreq

From beb873efc8a98cc5fe144304ebc050faeb814371 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Tue, 5 Aug 2025 17:51:24 +0000
Subject: [PATCH 420/656] ci: update base image target version to 20250805T1738

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 662aee950f..3d5520babe 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250804T2017
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250805T1738
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index d2c3dd6fc9..90d83abf03 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250804T2017
+20250805T1738

From ebe6fbf046590c9b6f08560075b760f2164f1f2a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 7 Aug 2025 12:13:41 -0300
Subject: [PATCH 421/656] feat: personless User deletion via admin (#9312)

* feat: admin to allow user deletion

* fix: permissions + drop dangerous action

* chore: minor style lint

* fix: avoid limit on a queryset delete

* feat: User age filter

* feat: show useful fields on User admin

* chore: fix lint

* fix: reverse direction of age filter
---
 ietf/ietfauth/admin.py | 136 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 136 insertions(+)
 create mode 100644 ietf/ietfauth/admin.py

diff --git a/ietf/ietfauth/admin.py b/ietf/ietfauth/admin.py
new file mode 100644
index 0000000000..c2914f9efa
--- /dev/null
+++ b/ietf/ietfauth/admin.py
@@ -0,0 +1,136 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+import datetime
+
+from django.conf import settings
+from django.contrib import admin, messages
+from django.contrib.admin import action
+from django.contrib.admin.actions import delete_selected as default_delete_selected
+from django.contrib.auth.admin import UserAdmin
+from django.contrib.auth.models import User
+from django.utils import timezone
+
+
+# Replace default UserAdmin with our custom one
+admin.site.unregister(User)
+
+
+class AgeListFilter(admin.SimpleListFilter):
+    title = "account age"
+    parameter_name = "age"
+    
+    def lookups(self, request, model_admin):
+        return [
+            ("1day", "> 1 day"),
+            ("3days", "> 3 days"),
+            ("1week", "> 1 week"),
+            ("1month", "> 1 month"),
+            ("1year", "> 1 year"),
+        ]
+
+    def queryset(self, request, queryset):
+        deltas = {
+            "1day": datetime.timedelta(days=1),
+            "3days": datetime.timedelta(days=3),
+            "1week": datetime.timedelta(weeks=1),
+            "1month": datetime.timedelta(days=30),
+            "1year": datetime.timedelta(days=365),
+        }
+        if self.value():
+            return queryset.filter(date_joined__lt=timezone.now()-deltas[self.value()])
+        return queryset
+
+
+@admin.register(User)
+class CustomUserAdmin(UserAdmin):
+    list_display = (
+        "username",
+        "person",
+        "date_joined",
+        "last_login",
+        "is_staff",
+    )
+    list_filter = list(UserAdmin.list_filter) + [
+        AgeListFilter,
+        ("person", admin.EmptyFieldListFilter),
+    ]
+    actions = ["delete_selected"]
+
+    @action(
+        permissions=["delete"], description="Delete personless %(verbose_name_plural)s"
+    )
+    def delete_selected(self, request, queryset):
+        """Delete selected action restricted to Users with a null Person field
+
+        This displaces the default delete_selected action with a safer one that will
+        only delete personless Users. It is done this way instead of by introducing
+        a new action so that we can simply hand off to the default action (imported
+        as default_delete_selected()) without having to adjust its template (and maybe
+        other things) to make it work with a different action name.
+        """
+        already_confirmed = bool(request.POST.get("post"))
+        personless_queryset = queryset.filter(person__isnull=True)
+        original_count = queryset.count()
+        personless_count = personless_queryset.count()
+        if personless_count > original_count:
+            # Refuse to act if the count increased!
+            self.message_user(
+                request,
+                (
+                    "Limiting the selection to Users without a Person INCREASED the "
+                    "count from {} to {}. This should not happen and probably means a "
+                    "concurrent change to the database affected this request. Please "
+                    "try again.".format(original_count, personless_count)
+                ),
+                level=messages.ERROR,
+            )
+            return None  # return to changelist
+
+        # Display warning/info if this is showing the confirmation page
+        if not already_confirmed:
+            if personless_count < original_count:
+                self.message_user(
+                    request,
+                    (
+                        "Limiting the selection to Users without a Person reduced the "
+                        "count from {} to {}. Only {} will be deleted.".format(
+                            original_count, personless_count, personless_count
+                        )
+                    ),
+                    level=messages.WARNING,
+                )
+            else:
+                self.message_user(
+                    request,
+                    "Confirmed that all selected Users had no Persons.",
+                )
+
+        # Django limits the number of fields in a request. The delete form itself
+        # includes a few metadata fields, so give it a little padding. The default
+        # limit is 1000 and everything will break if it's a small number, so not
+        # bothering to check that it's > 10.
+        max_count = settings.DATA_UPLOAD_MAX_NUMBER_FIELDS - 10
+        if personless_count > max_count:
+            self.message_user(
+                request,
+                (
+                    f"Only {max_count} Users can be deleted at once. Will only delete "
+                    f"the first {max_count} selected Personless Users."
+                ),
+                level=messages.WARNING,
+            )
+            # delete() doesn't like a queryset limited via [:max_count], so do an
+            # equivalent filter.
+            last_to_delete = personless_queryset.order_by("pk")[max_count]
+            personless_queryset = personless_queryset.filter(pk__lt=last_to_delete.pk)
+
+        if already_confirmed and personless_count != original_count:
+            # After confirmation, none of the above filtering should change anything.
+            # Refuse to delete if the DB moved underneath us.
+            self.message_user(
+                request,
+                "Queryset count changed, nothing deleted. Please try again.",
+                level=messages.ERROR,
+            )
+            return None
+
+        return default_delete_selected(self, request, personless_queryset)

From 86bce86731048f2dde04ace47af6425b775e23e9 Mon Sep 17 00:00:00 2001
From: Rudi Matz <rudi@staff.ietf.org>
Date: Thu, 7 Aug 2025 11:14:20 -0400
Subject: [PATCH 422/656] feat: use icalendar instead manual template (#9187)

* feat: use icalendar instead manual template

* avoid code duplication

* code cleanup

* ruff ruff

* remove comments

* add custom field with meeting's local Time zone

* more code cleanup

* remove unused template for ical

* pyflakes: remove unused imports and vars

* improve tests and code coverage

* remove commented line

* change URL in ical to use session material page
---
 ietf/meeting/tests_views.py       |  53 +++++++----
 ietf/meeting/views.py             | 148 +++++++++++++++++++++++++++---
 ietf/templates/meeting/agenda.ics |  32 -------
 requirements.txt                  |   1 +
 4 files changed, 168 insertions(+), 66 deletions(-)
 delete mode 100644 ietf/templates/meeting/agenda.ics

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 96a29c2297..f382772485 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -15,6 +15,7 @@
 from unittest.mock import call, patch, PropertyMock
 from pyquery import PyQuery
 from lxml.etree import tostring
+from icalendar import Calendar
 from io import StringIO, BytesIO
 from bs4 import BeautifulSoup
 from urllib.parse import urlparse, urlsplit
@@ -384,9 +385,6 @@ def test_meeting_agenda(self):
         r = self.client.get(ical_url)
 
         assert_ical_response_is_valid(self, r)
-        self.assertContains(r, "BEGIN:VTIMEZONE")
-        self.assertContains(r, "END:VTIMEZONE")
-        self.assertContains(r, meeting.time_zone, msg_prefix="time_zone should appear in its original case")
         self.assertNotEqual(
             meeting.time_zone,
             meeting.time_zone.lower(),
@@ -405,21 +403,32 @@ def test_meeting_agenda(self):
         assert_ical_response_is_valid(self, r)
         self.assertContains(r, session.group.acronym)
         self.assertContains(r, session.group.name)
-        self.assertContains(r, session.remote_instructions)
-        self.assertContains(r, slot.location.name)
-        self.assertContains(r, 'https://onsite.example.com')
-        self.assertContains(r, 'https://meetecho.example.com')
-        self.assertContains(r, "BEGIN:VTIMEZONE")
-        self.assertContains(r, "END:VTIMEZONE")        
 
-        self.assertContains(r, session.agenda().get_href())
-        self.assertContains(
-            r,
+        cal = Calendar.from_ical(r.content)
+        events = [component for component in cal.walk() if component.name == "VEVENT"]
+
+        self.assertEqual(len(events), 2)
+        self.assertIn(session.remote_instructions, events[0].get('description'))
+        self.assertIn("Onsite tool: https://onsite.example.com", events[0].get('description'))
+        self.assertIn("Meetecho: https://meetecho.example.com", events[0].get('description'))
+        self.assertIn(f"Agenda {session.agenda().get_href()}", events[0].get('description'))
+        session_materials_url = settings.IDTRACKER_BASE_URL + urlreverse(
+            'ietf.meeting.views.session_details',
+            kwargs=dict(num=meeting.number, acronym=session.group.acronym)
+        )
+        self.assertIn(f"Session materials: {session_materials_url}", events[0].get('description'))
+        self.assertIn(
             urlreverse(
                 'ietf.meeting.views.session_details',
                 kwargs=dict(num=meeting.number, acronym=session.group.acronym)),
-            msg_prefix='ical should contain link to meeting materials page for session')
+            events[0].get('description'))
+        self.assertEqual(
+            session_materials_url,
+            events[0].get('url')
+        )
 
+        self.assertContains(r, f"LOCATION:{slot.location.name}")
+        
         # Floor Plan
         r = self.client.get(urlreverse('floor-plan', kwargs=dict(num=meeting.number)))
         self.assertEqual(r.status_code, 200)
@@ -1049,32 +1058,36 @@ def test_group_ical(self):
         s1 = Session.objects.filter(meeting=meeting, group__acronym="mars").first()
         a1 = s1.official_timeslotassignment()
         t1 = a1.timeslot
+
         # Create an extra session
         t2 = TimeSlotFactory.create(
             meeting=meeting,
-            time=meeting.tz().localize(
+            time=pytz.utc.localize(
                 datetime.datetime.combine(meeting.date, datetime.time(11, 30))
             )
         )
+     
         s2 = SessionFactory.create(meeting=meeting, group=s1.group, add_to_schedule=False)
         SchedTimeSessAssignment.objects.create(timeslot=t2, session=s2, schedule=meeting.schedule)
-        #
+        
         url = urlreverse('ietf.meeting.views.agenda_ical', kwargs={'num':meeting.number, 'acronym':s1.group.acronym, })
         r = self.client.get(url)
         assert_ical_response_is_valid(self,
                                       r,
                                       expected_event_summaries=['mars - Martian Special Interest Group'],
                                       expected_event_count=2)
-        self.assertContains(r, t1.local_start_time().strftime('%Y%m%dT%H%M%S'))
-        self.assertContains(r, t2.local_start_time().strftime('%Y%m%dT%H%M%S'))
-        #
+        self.assertContains(r, f"DTSTART:{t1.time.strftime('%Y%m%dT%H%M%SZ')}")
+        self.assertContains(r, f"DTEND:{(t1.time + t1.duration).strftime('%Y%m%dT%H%M%SZ')}")
+        self.assertContains(r, f"DTSTART:{t2.time.strftime('%Y%m%dT%H%M%SZ')}")
+        self.assertContains(r, f"DTEND:{(t2.time + t2.duration).strftime('%Y%m%dT%H%M%SZ')}")
+
         url = urlreverse('ietf.meeting.views.agenda_ical', kwargs={'num':meeting.number, 'session_id':s1.id, })
         r = self.client.get(url)
         assert_ical_response_is_valid(self, r,
                                       expected_event_summaries=['mars - Martian Special Interest Group'],
                                       expected_event_count=1)
-        self.assertContains(r, t1.local_start_time().strftime('%Y%m%dT%H%M%S'))
-        self.assertNotContains(r, t2.local_start_time().strftime('%Y%m%dT%H%M%S'))
+        self.assertContains(r, f"DTSTART:{t1.time.strftime('%Y%m%dT%H%M%SZ')}")
+        self.assertNotContains(r, f"DTSTART:{t2.time.strftime('%Y%m%dT%H%M%SZ')}")
 
     def test_parse_agenda_filter_params(self):
         def _r(show=(), hide=(), showtypes=(), hidetypes=()):
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 85eda5a8f4..7fa3d21259 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -118,6 +118,9 @@
     UploadAgendaForm, UploadBlueSheetForm, UploadMinutesForm, UploadSlidesForm,
     UploadNarrativeMinutesForm)
 
+from icalendar import Calendar, Event
+from ietf.doc.templatetags.ietf_filters import absurl
+
 request_summary_exclude_group_types = ['team']
 
     
@@ -137,6 +140,10 @@ def send_interim_change_notice(request, meeting):
     message.related_groups.add(group)
     send_mail_message(request, message)
 
+def parse_ical_line_endings(ical):
+    """Parse icalendar line endings to ensure they are RFC 5545 compliant"""
+    return re.sub(r'\r(?!\n)|(?<!\r)\n', '\r\n', ical)
+
 # -------------------------------------------------
 # View Functions
 # -------------------------------------------------
@@ -1982,8 +1989,10 @@ def agenda_by_type_ics(request,num=None,type=None):
     ).order_by('session__type__slug','timeslot__time')
     if type:
         assignments = assignments.filter(session__type__slug=type)
-    updated = meeting.updated()
-    return render(request,"meeting/agenda.ics",{"schedule":schedule,"updated":updated,"assignments":assignments},content_type="text/calendar")
+
+    return render_icalendar(schedule, assignments)
+
+
 
 def session_draft_list(num, acronym):
     try:
@@ -2103,6 +2112,125 @@ def ical_session_status(assignment):
     else:
         return "CONFIRMED"
 
+def render_icalendar(schedule, assignments):
+    ical_content = generate_agenda_ical(schedule, assignments)
+    return HttpResponse(ical_content, content_type="text/calendar")
+
+def generate_agenda_ical(schedule, assignments):
+    """Generate iCalendar using the icalendar library"""
+
+    cal = Calendar()
+    cal.add("prodid", "-//IETF//datatracker.ietf.org ical agenda//EN")
+    cal.add("version", "2.0")
+    cal.add("method", "PUBLISH")
+
+    for item in assignments:
+        event = Event()
+
+        uid = f"ietf-{schedule.meeting.number}-{item.timeslot.pk}-{item.session.group.acronym}"
+        event.add("uid", uid)
+
+        # add custom field with meeting's local TZ
+        event.add("x-meeting-tz", schedule.meeting.time_zone)
+
+        if item.session.name:
+            summary = item.session.name
+        else:
+            group = item.session.group_at_the_time()
+            summary = f"{group.acronym} - {group.name}"
+
+        if item.session.agenda_note:
+            summary += f" ({item.session.agenda_note})"
+
+        event.add("summary", summary)
+
+        if item.timeslot.show_location and item.timeslot.get_location():
+            event.add("location", item.timeslot.get_location())
+
+        if item.session and hasattr(item.session, "current_status"):
+            status = ical_session_status(item)
+        else:
+            status = ""
+        event.add("status", status)
+
+        event.add("class", "PUBLIC")
+
+        event.add("dtstart", item.timeslot.utc_start_time())
+        event.add("dtend", item.timeslot.utc_end_time())
+
+        # DTSTAMP: when the event was created or last modified (in UTC)
+        dtstamp = item.timeslot.modified.astimezone(pytz.UTC)
+        event.add("dtstamp", dtstamp)
+
+        description_parts = [item.timeslot.name]
+
+        if item.session.agenda_note:
+            description_parts.append(f"Note: {item.session.agenda_note}")
+
+        if hasattr(item.session, "onsite_tool_url") and callable(
+            item.session.onsite_tool_url
+        ):
+            onsite_url = item.session.onsite_tool_url()
+            if onsite_url:
+                description_parts.append(f"Onsite tool: {onsite_url}")
+
+        if hasattr(item.session, "video_stream_url") and callable(
+            item.session.video_stream_url
+        ):
+            video_url = item.session.video_stream_url()
+            if video_url:
+                description_parts.append(f"Meetecho: {video_url}")
+
+        if (
+            item.timeslot.location
+            and hasattr(item.timeslot.location, "webex_url")
+            and callable(item.timeslot.location.webex_url)
+            and item.timeslot.location.webex_url() is not None
+        ):
+            description_parts.append(f"Webex: {item.timeslot.location.webex_url()}")
+
+        if item.session.remote_instructions:
+            description_parts.append(
+                f"Remote instructions: {item.session.remote_instructions}"
+            )
+
+        try:
+            materials_url = absurl(
+                "ietf.meeting.views.session_details",
+                num=schedule.meeting.number,
+                acronym=item.session.group.acronym,
+            )
+            description_parts.append(f"Session materials: {materials_url}")
+            event.add("url", materials_url)
+        except:
+            pass
+
+        if (
+            hasattr(schedule.meeting, "get_number")
+            and schedule.meeting.get_number() is not None
+        ):
+            try:
+                agenda_url = absurl("agenda", num=schedule.meeting.number)
+                description_parts.append(
+                    f"See in schedule: {agenda_url}#row-{item.slug()}"
+                )
+            except:
+                pass
+
+        agenda = item.session.agenda()
+        if agenda and hasattr(agenda, "get_versionless_href"):
+            agenda_url = agenda.get_versionless_href()
+            description_parts.append(f"{agenda.type} {agenda_url}")
+
+        # Join all description parts with 2 newlines
+        description = "\n\n".join(description_parts)
+        event.add("description", description)
+
+        # Add event to calendar
+        cal.add_component(event)
+
+    return cal.to_ical().decode("utf-8")
+
 def parse_agenda_filter_params(querydict):
     """Parse agenda filter parameters from a request"""
     if len(querydict) == 0:
@@ -2154,7 +2282,6 @@ def agenda_ical(request, num=None, acronym=None, session_id=None):
     else:
         meeting = get_meeting(num, type_in=None)  # get requested meeting, whatever its type
     schedule = get_schedule(meeting)
-    updated = meeting.updated()
 
     if schedule is None and acronym is None and session_id is None:
         raise Http404
@@ -2180,15 +2307,7 @@ def agenda_ical(request, num=None, acronym=None, session_id=None):
     elif session_id:
         assignments = [ a for a in assignments if a.session_id == int(session_id) ]
 
-    for a in assignments:
-        if a.session:
-            a.session.ical_status = ical_session_status(a)
-
-    return render(request, "meeting/agenda.ics", {
-        "schedule": schedule,
-        "assignments": assignments,
-        "updated": updated
-    }, content_type="text/calendar")
+    return render_icalendar(schedule, assignments)
 
 @cache_page(15 * 60)
 def agenda_json(request, num=None):
@@ -4138,7 +4257,7 @@ def upcoming_ical(request):
         'assignments': assignments,
         'ietfs': ietfs,
     }, request=request)
-    response = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", response)
+    response = parse_ical_line_endings(response)
 
     response = HttpResponse(response, content_type='text/calendar')
     response['Content-Disposition'] = 'attachment; filename="upcoming.ics"'
@@ -4764,7 +4883,7 @@ def important_dates(request, num=None, output_format=None):
             'meetings': meetings,
         }, request=request)
         # icalendar response file should have '\r\n' line endings per RFC5545
-        response = HttpResponse(re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", ics), content_type='text/calendar')
+        response = HttpResponse(parse_ical_line_endings(ics), content_type='text/calendar')
         response['Content-Disposition'] = 'attachment; filename="important-dates.ics"'
         return response
 
@@ -5198,3 +5317,4 @@ def import_session_minutes(request, session_id, num):
             'contents_unchanged': not contents_changed,
         },
     )
+
diff --git a/ietf/templates/meeting/agenda.ics b/ietf/templates/meeting/agenda.ics
deleted file mode 100644
index eb83dd479a..0000000000
--- a/ietf/templates/meeting/agenda.ics
+++ /dev/null
@@ -1,32 +0,0 @@
-{% load humanize tz %}{% autoescape off %}{% timezone schedule.meeting.tz %}{% with tzname=schedule.meeting.time_zone %}{% load ietf_filters textfilters %}{% load cache %}{% cache 1800 ietf_meeting_agenda_ics schedule.meeting.number request.path request.GET %}BEGIN:VCALENDAR
-VERSION:2.0
-METHOD:PUBLISH
-PRODID:-//IETF//datatracker.ietf.org ical agenda//EN
-{% if tzname != "utc" and tzname != "gmt" %}{% firstof schedule.meeting.vtimezone "" %}{% endif %}{% for item in assignments %}BEGIN:VEVENT
-UID:ietf-{{schedule.meeting.number}}-{{item.timeslot.pk}}-{{item.session.group.acronym}}
-SUMMARY:{% if item.session.name %}{{item.session.name|ics_esc}}{% else %}{{item.session.group_at_the_time.acronym|lower}} - {{item.session.group_at_the_time.name}}{%endif%}{% if item.session.agenda_note %} ({{item.session.agenda_note}}){% endif %}
-{% if item.timeslot.show_location %}LOCATION:{{item.timeslot.get_location}}
-{% endif %}STATUS:{{item.session.ical_status}}
-CLASS:PUBLIC
-DTSTART{% ics_date_time item.timeslot.local_start_time tzname %}
-DTEND{% ics_date_time item.timeslot.local_end_time tzname %}
-DTSTAMP{% ics_date_time item.timeslot.modified|utc 'utc' %}{% if item.session.agenda %}
-URL:{{item.session.agenda.get_versionless_href}}{% endif %}
-DESCRIPTION:{{item.timeslot.name|ics_esc}}\n{% if item.session.agenda_note %}
- Note: {{item.session.agenda_note|ics_esc}}\n{% endif %}{% if item.session.onsite_tool_url %}
- \n
- Onsite tool: {{ item.session.onsite_tool_url }}\n{% endif %}{% if item.session.video_stream_url %}
- \n
- Meetecho: {{ item.session.video_stream_url }}\n{% endif %}{% if item.timeslot.location.webex_url %}
- \n
- Webex: {{ item.timeslot.location.webex_url }}\n{% endif %}{% if item.session.remote_instructions %}
- \n
- Remote instructions: {{ item.session.remote_instructions }}\n{% endif %}{% if item.session.agenda %}{% with agenda=item.session.agenda %}
- \n
- {{agenda.type}} {{agenda.get_versionless_href}}\n{% endwith %}{% endif %}
- \n
- Session materials: {% absurl 'ietf.meeting.views.session_details' num=schedule.meeting.number acronym=item.session.group.acronym %}\n{% if schedule.meeting.get_number is not None %}
- \n{# link agenda for ietf meetings #}
- See in schedule: {% absurl 'agenda' num=schedule.meeting.number %}#row-{{ item.slug }}\n{% endif %}
-END:VEVENT
-{% endfor %}END:VCALENDAR{% endcache %}{% endwith %}{% endtimezone %}{% endautoescape %}
diff --git a/requirements.txt b/requirements.txt
index f82bfc4101..7aadc7bf7f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -42,6 +42,7 @@ gunicorn>=20.1.0
 hashids>=1.3.1
 html2text>=2020.1.16    # Used only to clean comment field of secr/sreq
 html5lib>=1.1    # Only used in tests
+icalendar>=5.0.0 
 inflect>= 6.0.2
 jsonfield>=3.1.0,<3.2.0  # 3.2.0 needs py3.10; deprecated-replace with Django JSONField
 jsonschema[format]>=4.2.1

From 666e9c53b45bc2eeda80ccbea66abeaa01df9830 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Thu, 7 Aug 2025 15:27:02 +0000
Subject: [PATCH 423/656] ci: update base image target version to 20250807T1514

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 3d5520babe..0ecf9566ef 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250805T1738
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250807T1514
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 90d83abf03..327fb48da4 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250805T1738
+20250807T1514

From a1a19e5b6cef8ddc8b6443b2e2844ee20be988fa Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 11 Aug 2025 14:00:51 -0300
Subject: [PATCH 424/656] fix: tz-aware tastypie datetimes (#9330)

* fix: tz-aware tastypie datetimes

* chore: comment

* chore: clarify comment
---
 ietf/api/__init__.py | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py
index d70866083e..e236347975 100644
--- a/ietf/api/__init__.py
+++ b/ietf/api/__init__.py
@@ -4,6 +4,7 @@
 
 import datetime
 import re
+import sys
 
 from urllib.parse import urlencode
 
@@ -25,6 +26,9 @@
 
 OMITTED_APPS_APIS = ["ietf.status"]
 
+# Pre-py3.11, fromisoformat() does not handle Z or +HH tz offsets
+HAVE_BROKEN_FROMISOFORMAT = sys.version_info < (3, 11, 0, "", 0)
+
 def populate_api_list():
     _module_dict = globals()
     for app_config in django_apps.get_app_configs():
@@ -58,6 +62,35 @@ def generate_cache_key(self, *args, **kwargs):
         # Use a list plus a ``.join()`` because it's faster than concatenation.
         return "%s:%s:%s:%s" % (self._meta.api_name, self._meta.resource_name, ':'.join(args), smooshed)
 
+    def _z_aware_fromisoformat(self, value):
+        """datetime.datetie.fromisoformat replacement that works with python < 3.11"""
+        if HAVE_BROKEN_FROMISOFORMAT:
+            if value.upper().endswith("Z"):
+                value = value[:-1] + "+00:00"  # Z -> UTC
+            elif re.match(r"[+-][0-9][0-9]$", value[-3:]):
+                value = value + ":00"  # -04 -> -04:00
+        return value
+
+    def filter_value_to_python(
+        self, value, field_name, filters, filter_expr, filter_type
+    ):
+        py_value = super().filter_value_to_python(
+            value, field_name, filters, filter_expr, filter_type
+        )
+        if isinstance(
+            self.fields[field_name], tastypie.fields.DateTimeField
+        ) and isinstance(py_value, str):
+            # Ensure datetime values are TZ-aware, using UTC by default
+            try:
+                dt = self._z_aware_fromisoformat(py_value)
+            except ValueError:
+                pass  # let tastypie deal with the original value
+            else:
+                if dt.tzinfo is None:
+                    dt = dt.replace(tzinfo=datetime.timezone.utc)
+                py_value = dt.isoformat()
+        return py_value
+
 
 TIMEDELTA_REGEX = re.compile(r'^(?P<days>\d+d)?\s?(?P<hours>\d+h)?\s?(?P<minutes>\d+m)?\s?(?P<seconds>\d+s?)$')
 

From 369e38f0b2ffe564340db6b6bcd64124d3190aa8 Mon Sep 17 00:00:00 2001
From: Eric Vyncke <eric@vyncke.org>
Date: Mon, 11 Aug 2025 19:05:59 +0200
Subject: [PATCH 425/656] Less shaded dark theme colors (#9335)

---
 ietf/static/css/ietf.scss | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss
index 014213c3e5..df973863d5 100644
--- a/ietf/static/css/ietf.scss
+++ b/ietf/static/css/ietf.scss
@@ -510,23 +510,23 @@ td.position-empty {
     tr.position-notready-row,
     tr.position-discuss-row,
     tr.position-block-row {
-        background-color: shade-color($color-discuss, 85%);
+        background-color: shade-color($color-discuss, 65%);
     }
 
     tr.position-yes-row {
-        background-color: shade-color($color-yes, 75%);
+        background-color: shade-color($color-yes, 65%);
     }
 
     tr.position-noobj-row {
-        background-color: shade-color($color-noobj, 75%);
+        background-color: shade-color($color-noobj, 65%);
     }
 
     tr.position-abstain-row {
-        background-color: shade-color($color-abstain, 85%);
+        background-color: shade-color($color-abstain, 65%);
     }
 
     tr.position-recuse-row {
-        background-color: shade-color($color-recuse, 85%);
+        background-color: shade-color($color-recuse, 65%);
     }
 }
 

From 37e3ffc8c41ad5e15464af4d242b2d6ee09c3775 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 11 Aug 2025 18:37:43 -0500
Subject: [PATCH 426/656] fix: allow RFC Editor to add comments to RFC history
 (#9344)

---
 ietf/doc/tests.py     | 34 ++++++++++++++++++++++++----------
 ietf/doc/views_doc.py |  2 +-
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index d3fba03bcc..fa8c7fa4fc 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -66,7 +66,7 @@
 from ietf.name.models import SessionStatusName, BallotPositionName, DocTypeName, RoleName
 from ietf.person.models import Person
 from ietf.person.factories import PersonFactory, EmailFactory
-from ietf.utils.mail import outbox, empty_outbox
+from ietf.utils.mail import get_payload_text, outbox, empty_outbox
 from ietf.utils.test_utils import login_testing_unauthorized, unicontent
 from ietf.utils.test_utils import TestCase
 from ietf.utils.text import normalize_text
@@ -2172,20 +2172,19 @@ def test_trailing_hypen_digit_name_bibxml(self):
 
 class AddCommentTestCase(TestCase):
     def test_add_comment(self):
-        draft = WgDraftFactory(name='draft-ietf-mars-test',group__acronym='mars')
-        url = urlreverse('ietf.doc.views_doc.add_comment', kwargs=dict(name=draft.name))
+        draft = WgDraftFactory(name="draft-ietf-mars-test", group__acronym="mars")
+        url = urlreverse("ietf.doc.views_doc.add_comment", kwargs=dict(name=draft.name))
         login_testing_unauthorized(self, "secretary", url)
 
         # normal get
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(unicontent(r))
-        self.assertEqual(len(q('form textarea[name=comment]')), 1)
+        self.assertEqual(len(q("form textarea[name=comment]")), 1)
 
-        # request resurrect
         events_before = draft.docevent_set.count()
         mailbox_before = len(outbox)
-        
+
         r = self.client.post(url, dict(comment="This is a test."))
         self.assertEqual(r.status_code, 302)
 
@@ -2193,9 +2192,9 @@ def test_add_comment(self):
         self.assertEqual("This is a test.", draft.latest_event().desc)
         self.assertEqual("added_comment", draft.latest_event().type)
         self.assertEqual(len(outbox), mailbox_before + 1)
-        self.assertIn("Comment added", outbox[-1]['Subject'])
-        self.assertIn(draft.name, outbox[-1]['Subject'])
-        self.assertIn('draft-ietf-mars-test@', outbox[-1]['To'])
+        self.assertIn("Comment added", outbox[-1]["Subject"])
+        self.assertIn(draft.name, outbox[-1]["Subject"])
+        self.assertIn("draft-ietf-mars-test@", outbox[-1]["To"])
 
         # Make sure we can also do it as IANA
         self.client.login(username="iana", password="iana+password")
@@ -2204,7 +2203,22 @@ def test_add_comment(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(unicontent(r))
-        self.assertEqual(len(q('form textarea[name=comment]')), 1)
+        self.assertEqual(len(q("form textarea[name=comment]")), 1)
+
+        empty_outbox()
+        rfc = WgRfcFactory()
+        self.client.login(username="rfc", password="rfc+password")
+        url = urlreverse("ietf.doc.views_doc.add_comment", kwargs=dict(name=rfc.name))
+        r = self.client.post(
+            url, dict(comment="This is an RFC Editor comment on an RFC.")
+        )
+        self.assertEqual(r.status_code, 302)
+
+        self.assertEqual(
+            "This is an RFC Editor comment on an RFC.", rfc.latest_event().desc
+        )
+        self.assertEqual(len(outbox), 1)
+        self.assertIn("This is an RFC Editor comment on an RFC.", get_payload_text(outbox[0]))
 
 
 class TemplateTagTest(TestCase):
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 591a72d907..4a20db3c89 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -1687,7 +1687,7 @@ def add_comment(request, name):
                 group__acronym=doc.group.acronym,
                 person__user=request.user)))
     else:
-        can_add_comment = has_role(request.user, ("Area Director", "Secretariat", "IRTF Chair"))
+        can_add_comment = has_role(request.user, ("Area Director", "Secretariat", "IRTF Chair", "RFC Editor"))
     if not can_add_comment:
         # The user is a chair or secretary, but not for this WG or RG
         permission_denied(request, "You need to be a chair or secretary of this group to add a comment.")

From 00264a4cb0f43df07e39beda8f74e1aa36814e71 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 14 Aug 2025 11:18:56 -0300
Subject: [PATCH 427/656] ci: ensure blobdbs exist for sandboxen

---
 dev/deploy-to-container/start.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/dev/deploy-to-container/start.sh b/dev/deploy-to-container/start.sh
index 2c83d6970c..5d976f80ea 100644
--- a/dev/deploy-to-container/start.sh
+++ b/dev/deploy-to-container/start.sh
@@ -35,6 +35,18 @@ echo "Running Datatracker checks..."
 
 # Migrate, adjusting to what the current state of the underlying database might be:
 
+# On production, the blobdb tables are in a separate database. Manipulate migration
+# history to ensure that they're created for the sandbox environment that runs it
+# all from a single database.
+echo "Ensuring blobdb relations exist..."
+/usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local --fake blobdb zero
+if ! /usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local blobdb; then
+  # If we are restarting a sandbox, the migration may already have run and re-running
+  # it will fail. Assume that happened and fake it.
+  /usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local --fake blobdb
+fi
+
+# Now run the migrations for real
 echo "Running Datatracker migrations..."
 /usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local
 

From f13a1a55774c9605b79e2db6f41dbe0ceb0d400a Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 15 Aug 2025 15:55:08 -0400
Subject: [PATCH 428/656] ci: Update db.Dockerfile to postgres 17

---
 docker/db.Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/db.Dockerfile b/docker/db.Dockerfile
index 0b57a80b70..48ab298780 100644
--- a/docker/db.Dockerfile
+++ b/docker/db.Dockerfile
@@ -1,7 +1,7 @@
 # =====================
 # --- Builder Stage ---
 # =====================
-FROM postgres:16 AS builder
+FROM postgres:17 AS builder
 
 ENV POSTGRES_PASSWORD=hk2j22sfiv
 ENV POSTGRES_USER=django
@@ -19,7 +19,7 @@ RUN ["/usr/local/bin/docker-entrypoint.sh", "postgres"]
 # ===================
 # --- Final Image ---
 # ===================
-FROM postgres:16
+FROM postgres:17
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 COPY --from=builder /data $PGDATA

From 0694863763509369228849b876b99666aaa70092 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 15 Aug 2025 17:05:40 -0300
Subject: [PATCH 429/656] refactor: jsonfield pkg->Django JSONField (#9284)

* refactor: drop jsonfield pkg in code

It still exists in migrations

* refactor: callable JSONField defaults

* chore: migrations

* chore: unused import

* chore: fix nulls in authors; renumber migrations

* fix: rename IETFJSONField to fix migration

* chore: update names.json

JSONField representation has changed

* fix: missed renaming; hide dead code from coverage
---
 .../0008_alter_group_used_roles_and_more.py   |  107 ++
 ietf/group/models.py                          |   55 +-
 ietf/name/fixtures/names.json                 | 1202 +++++++++++++----
 ...alter_sessionpurposename_timeslot_types.py |   27 +
 ietf/name/models.py                           |    6 +-
 ...icalperson_pronouns_selectable_and_more.py |   34 +
 ietf/person/models.py                         |    3 +-
 ...ion_authors_alter_submissioncheck_items.py |   46 +
 ietf/submit/models.py                         |    7 +-
 ietf/utils/db.py                              |   26 +-
 ietf/utils/fields.py                          |   19 +-
 11 files changed, 1259 insertions(+), 273 deletions(-)
 create mode 100644 ietf/group/migrations/0008_alter_group_used_roles_and_more.py
 create mode 100644 ietf/name/migrations/0019_alter_sessionpurposename_timeslot_types.py
 create mode 100644 ietf/person/migrations/0005_alter_historicalperson_pronouns_selectable_and_more.py
 create mode 100644 ietf/submit/migrations/0003_alter_submission_authors_alter_submissioncheck_items.py

diff --git a/ietf/group/migrations/0008_alter_group_used_roles_and_more.py b/ietf/group/migrations/0008_alter_group_used_roles_and_more.py
new file mode 100644
index 0000000000..28f345df00
--- /dev/null
+++ b/ietf/group/migrations/0008_alter_group_used_roles_and_more.py
@@ -0,0 +1,107 @@
+# Generated by Django 4.2.23 on 2025-08-15 16:46
+
+from django.db import migrations, models
+import ietf.group.models
+import ietf.name.models
+import ietf.utils.db
+import ietf.utils.validators
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("group", "0007_used_roles"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="group",
+            name="used_roles",
+            field=models.JSONField(
+                blank=True,
+                default=list,
+                help_text="Leave an empty list to get the group_type's default used roles",
+                max_length=256,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="groupfeatures",
+            name="admin_roles",
+            field=ietf.utils.db.EmptyAwareJSONField(
+                default=ietf.group.models.default_admin_roles, max_length=64
+            ),
+        ),
+        migrations.AlterField(
+            model_name="groupfeatures",
+            name="default_used_roles",
+            field=ietf.utils.db.EmptyAwareJSONField(default=list, max_length=256),
+        ),
+        migrations.AlterField(
+            model_name="groupfeatures",
+            name="docman_roles",
+            field=ietf.utils.db.EmptyAwareJSONField(
+                default=ietf.group.models.default_docman_roles, max_length=128
+            ),
+        ),
+        migrations.AlterField(
+            model_name="groupfeatures",
+            name="groupman_authroles",
+            field=ietf.utils.db.EmptyAwareJSONField(
+                default=ietf.group.models.default_groupman_authroles, max_length=128
+            ),
+        ),
+        migrations.AlterField(
+            model_name="groupfeatures",
+            name="groupman_roles",
+            field=ietf.utils.db.EmptyAwareJSONField(
+                default=ietf.group.models.default_groupman_roles, max_length=128
+            ),
+        ),
+        migrations.AlterField(
+            model_name="groupfeatures",
+            name="material_types",
+            field=ietf.utils.db.EmptyAwareJSONField(
+                default=ietf.group.models.default_material_types, max_length=64
+            ),
+        ),
+        migrations.AlterField(
+            model_name="groupfeatures",
+            name="matman_roles",
+            field=ietf.utils.db.EmptyAwareJSONField(
+                default=ietf.group.models.default_matman_roles, max_length=128
+            ),
+        ),
+        migrations.AlterField(
+            model_name="groupfeatures",
+            name="role_order",
+            field=ietf.utils.db.EmptyAwareJSONField(
+                default=ietf.group.models.default_role_order,
+                help_text="The order in which roles are shown, for instance on photo pages.  Enter valid JSON.",
+                max_length=128,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="groupfeatures",
+            name="session_purposes",
+            field=ietf.utils.db.EmptyAwareJSONField(
+                default=list,
+                help_text="Allowed session purposes for this group type",
+                max_length=256,
+                validators=[
+                    ietf.utils.validators.JSONForeignKeyListValidator(
+                        ietf.name.models.SessionPurposeName
+                    )
+                ],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="grouphistory",
+            name="used_roles",
+            field=models.JSONField(
+                blank=True,
+                default=list,
+                help_text="Leave an empty list to get the group_type's default used roles",
+                max_length=256,
+            ),
+        ),
+    ]
diff --git a/ietf/group/models.py b/ietf/group/models.py
index 52549e8cc1..608dcc86b9 100644
--- a/ietf/group/models.py
+++ b/ietf/group/models.py
@@ -3,7 +3,6 @@
 
 
 import email.utils
-import jsonfield
 import os
 import re
 
@@ -21,7 +20,7 @@
                               AgendaTypeName, AgendaFilterTypeName, ExtResourceName, SessionPurposeName,
                               AppealArtifactTypeName )
 from ietf.person.models import Email, Person
-from ietf.utils.db import IETFJSONField
+from ietf.utils.db import EmptyAwareJSONField
 from ietf.utils.mail import formataddr, send_mail_text
 from ietf.utils import log
 from ietf.utils.models import ForeignKey, OneToOneField
@@ -46,7 +45,7 @@ class GroupInfo(models.Model):
     unused_states = models.ManyToManyField('doc.State', help_text="Document states that have been disabled for the group.", blank=True)
     unused_tags = models.ManyToManyField(DocTagName, help_text="Document tags that have been disabled for the group.", blank=True)
 
-    used_roles = jsonfield.JSONField(max_length=256, blank=True, default=[], help_text="Leave an empty list to get the group_type's default used roles")
+    used_roles = models.JSONField(max_length=256, blank=True, default=list, help_text="Leave an empty list to get the group_type's default used roles")
 
     uses_milestone_dates = models.BooleanField(default=True)
 
@@ -235,6 +234,36 @@ def chat_archive_url(self):
 )
 
 
+# JSONFields need callable defaults that work with migrations to avoid sharing
+# data structures between instances. These helpers provide that. 
+def default_material_types():
+    return ["slides"]
+
+
+def default_admin_roles():
+    return ["chair"]
+
+
+def default_docman_roles():
+    return ["ad", "chair", "delegate", "secr"]
+
+
+def default_groupman_roles():
+    return ["ad", "chair"]
+
+
+def default_groupman_authroles():
+    return ["Secretariat"]
+
+
+def default_matman_roles():
+    return ["ad", "chair", "delegate", "secr"]
+
+
+def default_role_order():
+    return ["chair", "secr", "member"]
+
+
 class GroupFeatures(models.Model):
     type = OneToOneField(GroupTypeName, primary_key=True, null=False, related_name='features')
     #history = HistoricalRecords()
@@ -268,16 +297,16 @@ class GroupFeatures(models.Model):
     agenda_type             = models.ForeignKey(AgendaTypeName, null=True, default="ietf", on_delete=CASCADE)
     about_page              = models.CharField(max_length=64, blank=False, default="ietf.group.views.group_about" )
     default_tab             = models.CharField(max_length=64, blank=False, default="ietf.group.views.group_about" )
-    material_types          = IETFJSONField(max_length=64, accepted_empty_values=[[], {}], blank=False, default=["slides"])
-    default_used_roles      = IETFJSONField(max_length=256, accepted_empty_values=[[], {}], blank=False, default=[])
-    admin_roles             = IETFJSONField(max_length=64, accepted_empty_values=[[], {}], blank=False, default=["chair"]) # Trac Admin
-    docman_roles            = IETFJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=["ad","chair","delegate","secr"])
-    groupman_roles          = IETFJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=["ad","chair",])
-    groupman_authroles      = IETFJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=["Secretariat",])
-    matman_roles            = IETFJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=["ad","chair","delegate","secr"])
-    role_order              = IETFJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=["chair","secr","member"],
-                                                help_text="The order in which roles are shown, for instance on photo pages.  Enter valid JSON.")
-    session_purposes        = IETFJSONField(max_length=256, accepted_empty_values=[[], {}], blank=False, default=[],
+    material_types          = EmptyAwareJSONField(max_length=64, accepted_empty_values=[[], {}], blank=False, default=default_material_types)
+    default_used_roles      = EmptyAwareJSONField(max_length=256, accepted_empty_values=[[], {}], blank=False, default=list)
+    admin_roles             = EmptyAwareJSONField(max_length=64, accepted_empty_values=[[], {}], blank=False, default=default_admin_roles)  # Trac Admin
+    docman_roles            = EmptyAwareJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=default_docman_roles)
+    groupman_roles          = EmptyAwareJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=default_groupman_roles)
+    groupman_authroles      = EmptyAwareJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=default_groupman_authroles)
+    matman_roles            = EmptyAwareJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=default_matman_roles)
+    role_order              = EmptyAwareJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=default_role_order,
+                                            help_text="The order in which roles are shown, for instance on photo pages.  Enter valid JSON.")
+    session_purposes        = EmptyAwareJSONField(max_length=256, accepted_empty_values=[[], {}], blank=False, default=list,
                                                   help_text="Allowed session purposes for this group type",
                                                   validators=[JSONForeignKeyListValidator(SessionPurposeName)])
 
diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json
index ebdda1a1fa..0724cbb4b5 100644
--- a/ietf/name/fixtures/names.json
+++ b/ietf/name/fixtures/names.json
@@ -2630,6 +2630,19 @@
     "model": "doc.state",
     "pk": 181
   },
+  {
+    "fields": {
+      "desc": "The statement has been marked historic",
+      "name": "Historic",
+      "next_states": [],
+      "order": 0,
+      "slug": "historic",
+      "type": "statement",
+      "used": true
+    },
+    "model": "doc.state",
+    "pk": 182
+  },
   {
     "fields": {
       "label": "State"
@@ -2872,7 +2885,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "special",
       "agenda_type": "ietf",
       "create_wiki": true,
@@ -2880,10 +2895,24 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"matman\",\n    \"ad\",\n    \"chair\",\n    \"lead\",\n    \"delegate\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"chair\",\n    \"lead\",\n    \"delegate\"\n]",
+      "default_used_roles": [
+        "matman",
+        "ad",
+        "chair",
+        "lead",
+        "delegate"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "chair",
+        "lead",
+        "delegate"
+      ],
       "has_chartering_process": false,
       "has_default_chat": true,
       "has_documents": false,
@@ -2893,15 +2922,29 @@
       "has_reviews": false,
       "has_session_materials": true,
       "is_schedulable": true,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\",\n    \"lead\",\n    \"delegate\",\n    \"matman\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair",
+        "lead",
+        "delegate",
+        "matman"
+      ],
       "need_parent": false,
       "parent_types": [
         "ietf"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"lead\",\n    \"delegate\",\n    \"matman\"\n]",
-      "session_purposes": "[\n    \"presentation\"\n]",
+      "role_order": [
+        "chair",
+        "lead",
+        "delegate",
+        "matman"
+      ],
+      "session_purposes": [
+        "presentation"
+      ],
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",
@@ -2911,7 +2954,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "heading",
       "agenda_type": "ietf",
       "create_wiki": false,
@@ -2919,10 +2964,19 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"member\",\n    \"chair\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"chair\"\n]",
+      "default_used_roles": [
+        "member",
+        "chair"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "chair"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -2932,13 +2986,22 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": true,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair"
+      ],
       "need_parent": false,
       "parent_types": [],
       "req_subm_approval": false,
-      "role_order": "[\n    \"chair\"\n]",
-      "session_purposes": "[\n    \"closed_meeting\",\n    \"officehours\"\n]",
+      "role_order": [
+        "chair"
+      ],
+      "session_purposes": [
+        "closed_meeting",
+        "officehours"
+      ],
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",
@@ -2948,7 +3011,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": true,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "normal",
       "agenda_type": "ietf",
       "create_wiki": true,
@@ -2956,10 +3021,26 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"ad\",\n    \"chair\",\n    \"secr\",\n    \"delegate\"\n]",
-      "docman_roles": "[\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\",\n    \"Area Director\"\n]",
-      "groupman_roles": "[\n    \"ad\",\n    \"chair\",\n    \"delegate\"\n]",
+      "default_used_roles": [
+        "ad",
+        "chair",
+        "secr",
+        "delegate"
+      ],
+      "docman_roles": [
+        "chair",
+        "delegate",
+        "secr"
+      ],
+      "groupman_authroles": [
+        "Secretariat",
+        "Area Director"
+      ],
+      "groupman_roles": [
+        "ad",
+        "chair",
+        "delegate"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": true,
@@ -2969,16 +3050,28 @@
       "has_reviews": false,
       "has_session_materials": true,
       "is_schedulable": true,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"ad\",\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "ad",
+        "chair",
+        "delegate",
+        "secr"
+      ],
       "need_parent": false,
       "parent_types": [
         "area",
         "ietf"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[\n    \"regular\"\n]",
+      "role_order": [
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [
+        "regular"
+      ],
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",
@@ -2988,7 +3081,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"ad\"\n]",
+      "admin_roles": [
+        "ad"
+      ],
       "agenda_filter_type": "heading",
       "agenda_type": "ietf",
       "create_wiki": true,
@@ -2996,10 +3091,22 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"ad\",\n    \"liaison_contact\",\n    \"liaison_cc_contact\"\n]",
-      "docman_roles": "[\n    \"ad\",\n    \"delegate\",\n    \"secr\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"ad\"\n]",
+      "default_used_roles": [
+        "ad",
+        "liaison_contact",
+        "liaison_cc_contact"
+      ],
+      "docman_roles": [
+        "ad",
+        "delegate",
+        "secr"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "ad"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3009,15 +3116,27 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"ad\",\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "ad",
+        "chair",
+        "delegate",
+        "secr"
+      ],
       "need_parent": true,
       "parent_types": [
         "ietf"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[\n    \"regular\"\n]",
+      "role_order": [
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [
+        "regular"
+      ],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3027,7 +3146,10 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\",\n    \"secr\"\n]",
+      "admin_roles": [
+        "chair",
+        "secr"
+      ],
       "agenda_filter_type": "special",
       "agenda_type": "ad",
       "create_wiki": true,
@@ -3035,10 +3157,25 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"ad\",\n    \"chair\",\n    \"reviewer\",\n    \"secr\",\n    \"delegate\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"ad\",\n    \"secr\",\n    \"delegate\",\n    \"chair\"\n]",
+      "default_used_roles": [
+        "ad",
+        "chair",
+        "reviewer",
+        "secr",
+        "delegate"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "ad",
+        "secr",
+        "delegate",
+        "chair"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3048,15 +3185,31 @@
       "has_reviews": false,
       "has_session_materials": true,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"ad\",\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "ad",
+        "chair",
+        "delegate",
+        "secr"
+      ],
       "need_parent": true,
       "parent_types": [
         "area"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[\n    \"open_meeting\",\n    \"presentation\",\n    \"regular\",\n    \"social\",\n    \"tutorial\"\n]",
+      "role_order": [
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [
+        "open_meeting",
+        "presentation",
+        "regular",
+        "social",
+        "tutorial"
+      ],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3066,7 +3219,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "normal",
       "agenda_type": "ietf",
       "create_wiki": false,
@@ -3074,10 +3229,19 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"chair\",\n    \"member\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"chair\"\n]",
+      "default_used_roles": [
+        "chair",
+        "member"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "chair"
+      ],
       "has_chartering_process": false,
       "has_default_chat": true,
       "has_documents": false,
@@ -3087,13 +3251,23 @@
       "has_reviews": false,
       "has_session_materials": true,
       "is_schedulable": true,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair"
+      ],
       "need_parent": false,
       "parent_types": [],
       "req_subm_approval": false,
-      "role_order": "[\n    \"chair\",\n    \"member\"\n]",
-      "session_purposes": "[\n    \"officehours\",\n    \"regular\"\n]",
+      "role_order": [
+        "chair",
+        "member"
+      ],
+      "session_purposes": [
+        "officehours",
+        "regular"
+      ],
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",
@@ -3103,7 +3277,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": true,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "normal",
       "agenda_type": "ietf",
       "create_wiki": false,
@@ -3111,10 +3287,18 @@
       "customize_workflow": true,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_documents",
-      "default_used_roles": "[\n    \"chair\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"chair\"\n]",
+      "default_used_roles": [
+        "chair"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "chair"
+      ],
       "has_chartering_process": false,
       "has_default_chat": true,
       "has_documents": true,
@@ -3124,15 +3308,23 @@
       "has_reviews": false,
       "has_session_materials": true,
       "is_schedulable": true,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair"
+      ],
       "need_parent": false,
       "parent_types": [
         "rfcedtyp"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\"\n]",
-      "session_purposes": "[\n    \"regular\"\n]",
+      "role_order": [
+        "chair"
+      ],
+      "session_purposes": [
+        "regular"
+      ],
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",
@@ -3142,7 +3334,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "normal",
       "agenda_type": "ietf",
       "create_wiki": false,
@@ -3150,10 +3344,16 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"chair\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[]",
+      "default_used_roles": [
+        "chair"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3163,15 +3363,26 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\",\n    \"delegate\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair",
+        "delegate"
+      ],
       "need_parent": false,
       "parent_types": [
         "ietf"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[\n    \"closed_meeting\",\n    \"regular\"\n]",
+      "role_order": [
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [
+        "closed_meeting",
+        "regular"
+      ],
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",
@@ -3181,7 +3392,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"lead\"\n]",
+      "admin_roles": [
+        "lead"
+      ],
       "agenda_filter_type": "none",
       "agenda_type": "ad",
       "create_wiki": false,
@@ -3189,10 +3402,27 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"member\",\n    \"chair\",\n    \"lead\",\n    \"delegate\"\n]",
-      "docman_roles": "[\n    \"lead\",\n    \"chair\",\n    \"secr\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\",\n    \"IAB\"\n]",
-      "groupman_roles": "[\n    \"lead\",\n    \"chair\",\n    \"secr\",\n    \"delegate\"\n]",
+      "default_used_roles": [
+        "member",
+        "chair",
+        "lead",
+        "delegate"
+      ],
+      "docman_roles": [
+        "lead",
+        "chair",
+        "secr"
+      ],
+      "groupman_authroles": [
+        "Secretariat",
+        "IAB"
+      ],
+      "groupman_roles": [
+        "lead",
+        "chair",
+        "secr",
+        "delegate"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": true,
@@ -3202,15 +3432,29 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"lead\",\n    \"chair\",\n    \"secr\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "lead",
+        "chair",
+        "secr"
+      ],
       "need_parent": false,
       "parent_types": [
         "ietf"
       ],
       "req_subm_approval": false,
-      "role_order": "[\n    \"lead\",\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[\n    \"closed_meeting\",\n    \"officehours\",\n    \"open_meeting\"\n]",
+      "role_order": [
+        "lead",
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [
+        "closed_meeting",
+        "officehours",
+        "open_meeting"
+      ],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3220,7 +3464,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "none",
       "agenda_type": "ietf",
       "create_wiki": false,
@@ -3228,10 +3474,20 @@
       "customize_workflow": false,
       "default_parent": "iab",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[]",
-      "docman_roles": "[\n    \"ad\",\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"ad\",\n    \"chair\"\n]",
+      "default_used_roles": [],
+      "docman_roles": [
+        "ad",
+        "chair",
+        "delegate",
+        "secr"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "ad",
+        "chair"
+      ],
       "has_chartering_process": false,
       "has_default_chat": true,
       "has_documents": true,
@@ -3241,15 +3497,26 @@
       "has_reviews": false,
       "has_session_materials": true,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"ad\",\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "ad",
+        "chair",
+        "delegate",
+        "secr"
+      ],
       "need_parent": true,
       "parent_types": [
         "ietf"
       ],
       "req_subm_approval": false,
-      "role_order": "[\n    \"chair\",\n    \"secr\",\n    \"member\"\n]",
-      "session_purposes": "\"[\\\"regular\\\"]\"",
+      "role_order": [
+        "chair",
+        "secr",
+        "member"
+      ],
+      "session_purposes": "[\"regular\"]",
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3259,7 +3526,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "none",
       "agenda_type": "ietf",
       "create_wiki": false,
@@ -3267,10 +3536,18 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"auth\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"chair\"\n]",
+      "default_used_roles": [
+        "auth"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "chair"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3280,13 +3557,21 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair"
+      ],
       "need_parent": false,
       "parent_types": [],
       "req_subm_approval": false,
-      "role_order": "[\n    \"chair\"\n]",
-      "session_purposes": "[\n    \"officehours\"\n]",
+      "role_order": [
+        "chair"
+      ],
+      "session_purposes": [
+        "officehours"
+      ],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3296,7 +3581,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "none",
       "agenda_type": "ad",
       "create_wiki": false,
@@ -3304,10 +3591,19 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"delegate\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"chair\",\n    \"delegate\"\n]",
+      "default_used_roles": [
+        "delegate"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "chair",
+        "delegate"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3317,13 +3613,24 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": false,
-      "material_types": "\"[]\"",
-      "matman_roles": "[\n    \"chair\",\n    \"delegate\",\n    \"member\"\n]",
+      "material_types": "[]",
+      "matman_roles": [
+        "chair",
+        "delegate",
+        "member"
+      ],
       "need_parent": false,
       "parent_types": [],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"delegate\",\n    \"member\"\n]",
-      "session_purposes": "[\n    \"closed_meeting\",\n    \"open_meeting\"\n]",
+      "role_order": [
+        "chair",
+        "delegate",
+        "member"
+      ],
+      "session_purposes": [
+        "closed_meeting",
+        "open_meeting"
+      ],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3333,7 +3640,10 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\",\n    \"lead\"\n]",
+      "admin_roles": [
+        "chair",
+        "lead"
+      ],
       "agenda_filter_type": "heading",
       "agenda_type": "ietf",
       "create_wiki": false,
@@ -3341,10 +3651,26 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"ad\",\n    \"member\",\n    \"comdir\",\n    \"delegate\",\n    \"execdir\",\n    \"recman\",\n    \"secr\",\n    \"chair\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"chair\",\n    \"delegate\"\n]",
+      "default_used_roles": [
+        "ad",
+        "member",
+        "comdir",
+        "delegate",
+        "execdir",
+        "recman",
+        "secr",
+        "chair"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "chair",
+        "delegate"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3354,15 +3680,29 @@
       "has_reviews": false,
       "has_session_materials": true,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\",\n    \"delegate\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair",
+        "delegate"
+      ],
       "need_parent": false,
       "parent_types": [
         "ietf"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[\n    \"admin\",\n    \"plenary\",\n    \"presentation\",\n    \"social\",\n    \"officehours\"\n]",
+      "role_order": [
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [
+        "admin",
+        "plenary",
+        "presentation",
+        "social",
+        "officehours"
+      ],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3372,7 +3712,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "none",
       "agenda_type": "ad",
       "create_wiki": false,
@@ -3380,10 +3722,16 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"ad\"\n]",
-      "docman_roles": "[\n    \"auth\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[]",
+      "default_used_roles": [
+        "ad"
+      ],
+      "docman_roles": [
+        "auth"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3393,15 +3741,20 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [],
       "need_parent": true,
       "parent_types": [
         "area"
       ],
       "req_subm_approval": false,
-      "role_order": "[\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[]",
+      "role_order": [
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3411,7 +3764,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "heading",
       "agenda_type": "ietf",
       "create_wiki": false,
@@ -3419,10 +3774,20 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"member\",\n    \"atlarge\",\n    \"chair\",\n    \"delegate\"\n]",
-      "docman_roles": "[]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"chair\",\n    \"delegate\"\n]",
+      "default_used_roles": [
+        "member",
+        "atlarge",
+        "chair",
+        "delegate"
+      ],
+      "docman_roles": [],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "chair",
+        "delegate"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3432,15 +3797,24 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair",
+        "delegate",
+        "secr"
+      ],
       "need_parent": false,
       "parent_types": [
         "irtf"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[]",
+      "role_order": [
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3450,7 +3824,10 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\",\n    \"lead\"\n]",
+      "admin_roles": [
+        "chair",
+        "lead"
+      ],
       "agenda_filter_type": "heading",
       "agenda_type": "ietf",
       "create_wiki": false,
@@ -3458,10 +3835,20 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"chair\",\n    \"delegate\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"chair\",\n    \"delegate\"\n]",
+      "default_used_roles": [
+        "chair",
+        "delegate"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "chair",
+        "delegate"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": true,
@@ -3471,13 +3858,24 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": true,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\",\n    \"delegate\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair",
+        "delegate"
+      ],
       "need_parent": false,
       "parent_types": [],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"delegate\"\n]",
-      "session_purposes": "[\n    \"officehours\",\n    \"regular\"\n]",
+      "role_order": [
+        "chair",
+        "delegate"
+      ],
+      "session_purposes": [
+        "officehours",
+        "regular"
+      ],
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",
@@ -3487,7 +3885,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "none",
       "agenda_type": null,
       "create_wiki": false,
@@ -3495,10 +3895,17 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"chair\",\n    \"ceo\"\n]",
-      "docman_roles": "[]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"chair\"\n]",
+      "default_used_roles": [
+        "chair",
+        "ceo"
+      ],
+      "docman_roles": [],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "chair"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3508,15 +3915,27 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\",\n    \"secr\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair",
+        "secr"
+      ],
       "need_parent": false,
       "parent_types": [
         "isoc"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[\n    \"officehours\",\n    \"open_meeting\",\n    \"presentation\"\n]",
+      "role_order": [
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [
+        "officehours",
+        "open_meeting",
+        "presentation"
+      ],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3526,7 +3945,10 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\",\n    \"advisor\"\n]",
+      "admin_roles": [
+        "chair",
+        "advisor"
+      ],
       "agenda_filter_type": "none",
       "agenda_type": "side",
       "create_wiki": true,
@@ -3534,10 +3956,23 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"member\",\n    \"advisor\",\n    \"liaison\",\n    \"chair\",\n    \"techadv\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"chair\",\n    \"advisor\"\n]",
+      "default_used_roles": [
+        "member",
+        "advisor",
+        "liaison",
+        "chair",
+        "techadv"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "chair",
+        "advisor"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3547,15 +3982,26 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair"
+      ],
       "need_parent": false,
       "parent_types": [
         "area"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"member\",\n    \"advisor\"\n]",
-      "session_purposes": "[\n    \"closed_meeting\",\n    \"officehours\"\n]",
+      "role_order": [
+        "chair",
+        "member",
+        "advisor"
+      ],
+      "session_purposes": [
+        "closed_meeting",
+        "officehours"
+      ],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3565,7 +4011,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"lead\"\n]",
+      "admin_roles": [
+        "lead"
+      ],
       "agenda_filter_type": "normal",
       "agenda_type": "ad",
       "create_wiki": false,
@@ -3573,10 +4021,27 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"member\",\n    \"chair\",\n    \"lead\",\n    \"delegate\"\n]",
-      "docman_roles": "[\n    \"lead\",\n    \"chair\",\n    \"secr\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\",\n    \"IAB\"\n]",
-      "groupman_roles": "[\n    \"lead\",\n    \"chair\",\n    \"secr\",\n    \"delegate\"\n]",
+      "default_used_roles": [
+        "member",
+        "chair",
+        "lead",
+        "delegate"
+      ],
+      "docman_roles": [
+        "lead",
+        "chair",
+        "secr"
+      ],
+      "groupman_authroles": [
+        "Secretariat",
+        "IAB"
+      ],
+      "groupman_roles": [
+        "lead",
+        "chair",
+        "secr",
+        "delegate"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": true,
@@ -3586,15 +4051,28 @@
       "has_reviews": false,
       "has_session_materials": true,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"lead\",\n    \"chair\",\n    \"secr\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "lead",
+        "chair",
+        "secr"
+      ],
       "need_parent": false,
       "parent_types": [
         "ietf"
       ],
       "req_subm_approval": false,
-      "role_order": "[\n    \"lead\",\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[\n    \"regular\",\n    \"tutorial\"\n]",
+      "role_order": [
+        "lead",
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [
+        "regular",
+        "tutorial"
+      ],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3604,7 +4082,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": true,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "normal",
       "agenda_type": "ietf",
       "create_wiki": true,
@@ -3612,10 +4092,24 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"chair\",\n    \"secr\",\n    \"delegate\"\n]",
-      "docman_roles": "[\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\",\n    \"IRTF Chair\"\n]",
-      "groupman_roles": "[\n    \"chair\",\n    \"delegate\"\n]",
+      "default_used_roles": [
+        "chair",
+        "secr",
+        "delegate"
+      ],
+      "docman_roles": [
+        "chair",
+        "delegate",
+        "secr"
+      ],
+      "groupman_authroles": [
+        "Secretariat",
+        "IRTF Chair"
+      ],
+      "groupman_roles": [
+        "chair",
+        "delegate"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": true,
@@ -3625,15 +4119,26 @@
       "has_reviews": false,
       "has_session_materials": true,
       "is_schedulable": true,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair",
+        "delegate",
+        "secr"
+      ],
       "need_parent": false,
       "parent_types": [
         "irtf"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[\n    \"regular\"\n]",
+      "role_order": [
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [
+        "regular"
+      ],
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",
@@ -3643,7 +4148,10 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\",\n    \"secr\"\n]",
+      "admin_roles": [
+        "chair",
+        "secr"
+      ],
       "agenda_filter_type": "normal",
       "agenda_type": "ietf",
       "create_wiki": true,
@@ -3651,10 +4159,24 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.review_requests",
-      "default_used_roles": "[\n    \"ad\",\n    \"chair\",\n    \"reviewer\",\n    \"secr\",\n    \"delegate\"\n]",
-      "docman_roles": "[\n    \"secr\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"ad\",\n    \"secr\",\n    \"delegate\"\n]",
+      "default_used_roles": [
+        "ad",
+        "chair",
+        "reviewer",
+        "secr",
+        "delegate"
+      ],
+      "docman_roles": [
+        "secr"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "ad",
+        "secr",
+        "delegate"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3664,15 +4186,26 @@
       "has_reviews": true,
       "has_session_materials": true,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"ad\",\n    \"secr\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "ad",
+        "secr"
+      ],
       "need_parent": true,
       "parent_types": [
         "area"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[\n    \"open_meeting\",\n    \"social\"\n]",
+      "role_order": [
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [
+        "open_meeting",
+        "social"
+      ],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3682,7 +4215,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "normal",
       "agenda_type": "ietf",
       "create_wiki": false,
@@ -3690,10 +4225,19 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"auth\",\n    \"chair\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[\n    \"chair\"\n]",
+      "default_used_roles": [
+        "auth",
+        "chair"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [
+        "chair"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3703,13 +4247,23 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair"
+      ],
       "need_parent": false,
       "parent_types": [],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"secr\"\n]",
-      "session_purposes": "[\n    \"officehours\",\n    \"regular\"\n]",
+      "role_order": [
+        "chair",
+        "secr"
+      ],
+      "session_purposes": [
+        "officehours",
+        "regular"
+      ],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3719,7 +4273,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": true,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "normal",
       "agenda_type": "ietf",
       "create_wiki": true,
@@ -3727,10 +4283,25 @@
       "customize_workflow": true,
       "default_parent": "irtf",
       "default_tab": "ietf.group.views.group_documents",
-      "default_used_roles": "[\n    \"chair\",\n    \"techadv\",\n    \"secr\",\n    \"delegate\"\n]",
-      "docman_roles": "[\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\",\n    \"IRTF Chair\"\n]",
-      "groupman_roles": "[\n    \"chair\",\n    \"delegate\"\n]",
+      "default_used_roles": [
+        "chair",
+        "techadv",
+        "secr",
+        "delegate"
+      ],
+      "docman_roles": [
+        "chair",
+        "delegate",
+        "secr"
+      ],
+      "groupman_authroles": [
+        "Secretariat",
+        "IRTF Chair"
+      ],
+      "groupman_roles": [
+        "chair",
+        "delegate"
+      ],
       "has_chartering_process": true,
       "has_default_chat": true,
       "has_documents": true,
@@ -3740,15 +4311,27 @@
       "has_reviews": false,
       "has_session_materials": true,
       "is_schedulable": true,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair",
+        "delegate",
+        "secr"
+      ],
       "need_parent": true,
       "parent_types": [
         "irtf"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
-      "session_purposes": "[\n    \"regular\"\n]",
+      "role_order": [
+        "chair",
+        "delegate",
+        "secr"
+      ],
+      "session_purposes": [
+        "regular"
+      ],
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",
@@ -3758,7 +4341,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "none",
       "agenda_type": null,
       "create_wiki": false,
@@ -3766,10 +4351,23 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"liaiman\",\n    \"ceo\",\n    \"coord\",\n    \"auth\",\n    \"chair\",\n    \"liaison_contact\",\n    \"liaison_cc_contact\"\n]",
-      "docman_roles": "[\n    \"liaiman\",\n    \"matman\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\"\n]",
-      "groupman_roles": "[]",
+      "default_used_roles": [
+        "liaiman",
+        "ceo",
+        "coord",
+        "auth",
+        "chair",
+        "liaison_contact",
+        "liaison_cc_contact"
+      ],
+      "docman_roles": [
+        "liaiman",
+        "matman"
+      ],
+      "groupman_authroles": [
+        "Secretariat"
+      ],
+      "groupman_roles": [],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3779,16 +4377,20 @@
       "has_reviews": false,
       "has_session_materials": false,
       "is_schedulable": false,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [],
       "need_parent": false,
       "parent_types": [
         "area",
         "sdo"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"liaiman\"\n]",
-      "session_purposes": "[]",
+      "role_order": [
+        "liaiman"
+      ],
+      "session_purposes": [],
       "show_on_agenda": false
     },
     "model": "group.groupfeatures",
@@ -3798,7 +4400,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": false,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "special",
       "agenda_type": "ietf",
       "create_wiki": true,
@@ -3806,10 +4410,28 @@
       "customize_workflow": false,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_about",
-      "default_used_roles": "[\n    \"ad\",\n    \"member\",\n    \"delegate\",\n    \"secr\",\n    \"liaison\",\n    \"atlarge\",\n    \"chair\",\n    \"matman\",\n    \"techadv\"\n]",
-      "docman_roles": "[\n    \"chair\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\",\n    \"Area Director\"\n]",
-      "groupman_roles": "[\n    \"chair\",\n    \"delegate\"\n]",
+      "default_used_roles": [
+        "ad",
+        "member",
+        "delegate",
+        "secr",
+        "liaison",
+        "atlarge",
+        "chair",
+        "matman",
+        "techadv"
+      ],
+      "docman_roles": [
+        "chair"
+      ],
+      "groupman_authroles": [
+        "Secretariat",
+        "Area Director"
+      ],
+      "groupman_roles": [
+        "chair",
+        "delegate"
+      ],
       "has_chartering_process": false,
       "has_default_chat": false,
       "has_documents": false,
@@ -3819,15 +4441,29 @@
       "has_reviews": false,
       "has_session_materials": true,
       "is_schedulable": true,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"chair\",\n    \"matman\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "chair",
+        "matman"
+      ],
       "need_parent": false,
       "parent_types": [
         "area"
       ],
       "req_subm_approval": false,
-      "role_order": "[\n    \"chair\",\n    \"member\",\n    \"matman\"\n]",
-      "session_purposes": "[\n    \"coding\",\n    \"presentation\",\n    \"social\",\n    \"tutorial\"\n]",
+      "role_order": [
+        "chair",
+        "member",
+        "matman"
+      ],
+      "session_purposes": [
+        "coding",
+        "presentation",
+        "social",
+        "tutorial"
+      ],
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",
@@ -3837,7 +4473,9 @@
     "fields": {
       "about_page": "ietf.group.views.group_about",
       "acts_like_wg": true,
-      "admin_roles": "[\n    \"chair\"\n]",
+      "admin_roles": [
+        "chair"
+      ],
       "agenda_filter_type": "normal",
       "agenda_type": "ietf",
       "create_wiki": true,
@@ -3845,10 +4483,32 @@
       "customize_workflow": true,
       "default_parent": "",
       "default_tab": "ietf.group.views.group_documents",
-      "default_used_roles": "[\n    \"ad\",\n    \"editor\",\n    \"delegate\",\n    \"secr\",\n    \"chair\",\n    \"matman\",\n    \"techadv\",\n    \"liaison_contact\",\n    \"liaison_cc_contact\"\n]",
-      "docman_roles": "[\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
-      "groupman_authroles": "[\n    \"Secretariat\",\n    \"Area Director\"\n]",
-      "groupman_roles": "[\n    \"ad\",\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
+      "default_used_roles": [
+        "ad",
+        "editor",
+        "delegate",
+        "secr",
+        "chair",
+        "matman",
+        "techadv",
+        "liaison_contact",
+        "liaison_cc_contact"
+      ],
+      "docman_roles": [
+        "chair",
+        "delegate",
+        "secr"
+      ],
+      "groupman_authroles": [
+        "Secretariat",
+        "Area Director"
+      ],
+      "groupman_roles": [
+        "ad",
+        "chair",
+        "delegate",
+        "secr"
+      ],
       "has_chartering_process": true,
       "has_default_chat": true,
       "has_documents": true,
@@ -3858,15 +4518,28 @@
       "has_reviews": false,
       "has_session_materials": true,
       "is_schedulable": true,
-      "material_types": "[\n    \"slides\"\n]",
-      "matman_roles": "[\n    \"ad\",\n    \"chair\",\n    \"delegate\",\n    \"secr\"\n]",
+      "material_types": [
+        "slides"
+      ],
+      "matman_roles": [
+        "ad",
+        "chair",
+        "delegate",
+        "secr"
+      ],
       "need_parent": false,
       "parent_types": [
         "area"
       ],
       "req_subm_approval": true,
-      "role_order": "[\n    \"chair\",\n    \"secr\",\n    \"delegate\"\n]",
-      "session_purposes": "[\n    \"regular\"\n]",
+      "role_order": [
+        "chair",
+        "secr",
+        "delegate"
+      ],
+      "session_purposes": [
+        "regular"
+      ],
       "show_on_agenda": true
     },
     "model": "group.groupfeatures",
@@ -5392,6 +6065,21 @@
     "model": "mailtrigger.mailtrigger",
     "pk": "review_completed_opsdir_telechat"
   },
+  {
+    "fields": {
+      "cc": [
+        "ietf_last_call",
+        "review_doc_all_parties",
+        "review_doc_group_mail_list"
+      ],
+      "desc": "Recipients when a perfmetrdir IETF Last Call review is completed",
+      "to": [
+        "review_team_mail_list"
+      ]
+    },
+    "model": "mailtrigger.mailtrigger",
+    "pk": "review_completed_perfmetrdir_lc"
+  },
   {
     "fields": {
       "cc": [
@@ -13849,7 +14537,10 @@
       "name": "Administrative",
       "on_agenda": true,
       "order": 5,
-      "timeslot_types": "[\n    \"other\",\n    \"reg\"\n]",
+      "timeslot_types": [
+        "other",
+        "reg"
+      ],
       "used": true
     },
     "model": "name.sessionpurposename",
@@ -13861,7 +14552,10 @@
       "name": "Closed meeting",
       "on_agenda": false,
       "order": 10,
-      "timeslot_types": "[\n    \"other\",\n    \"regular\"\n]",
+      "timeslot_types": [
+        "other",
+        "regular"
+      ],
       "used": true
     },
     "model": "name.sessionpurposename",
@@ -13873,7 +14567,9 @@
       "name": "Coding",
       "on_agenda": true,
       "order": 4,
-      "timeslot_types": "[\n    \"other\"\n]",
+      "timeslot_types": [
+        "other"
+      ],
       "used": true
     },
     "model": "name.sessionpurposename",
@@ -13885,7 +14581,7 @@
       "name": "None",
       "on_agenda": true,
       "order": 0,
-      "timeslot_types": "[]",
+      "timeslot_types": [],
       "used": false
     },
     "model": "name.sessionpurposename",
@@ -13897,7 +14593,9 @@
       "name": "Office hours",
       "on_agenda": true,
       "order": 3,
-      "timeslot_types": "[\n    \"other\"\n]",
+      "timeslot_types": [
+        "other"
+      ],
       "used": true
     },
     "model": "name.sessionpurposename",
@@ -13909,7 +14607,9 @@
       "name": "Open meeting",
       "on_agenda": true,
       "order": 9,
-      "timeslot_types": "[\n    \"other\"\n]",
+      "timeslot_types": [
+        "other"
+      ],
       "used": true
     },
     "model": "name.sessionpurposename",
@@ -13921,7 +14621,9 @@
       "name": "Plenary",
       "on_agenda": true,
       "order": 7,
-      "timeslot_types": "[\n    \"plenary\"\n]",
+      "timeslot_types": [
+        "plenary"
+      ],
       "used": true
     },
     "model": "name.sessionpurposename",
@@ -13933,7 +14635,10 @@
       "name": "Presentation",
       "on_agenda": true,
       "order": 8,
-      "timeslot_types": "[\n    \"other\",\n    \"regular\"\n]",
+      "timeslot_types": [
+        "other",
+        "regular"
+      ],
       "used": true
     },
     "model": "name.sessionpurposename",
@@ -13945,7 +14650,9 @@
       "name": "Regular",
       "on_agenda": true,
       "order": 1,
-      "timeslot_types": "[\n    \"regular\"\n]",
+      "timeslot_types": [
+        "regular"
+      ],
       "used": true
     },
     "model": "name.sessionpurposename",
@@ -13957,7 +14664,10 @@
       "name": "Social",
       "on_agenda": true,
       "order": 6,
-      "timeslot_types": "[\n    \"break\",\n    \"other\"\n]",
+      "timeslot_types": [
+        "break",
+        "other"
+      ],
       "used": true
     },
     "model": "name.sessionpurposename",
@@ -13969,7 +14679,9 @@
       "name": "Tutorial",
       "on_agenda": true,
       "order": 2,
-      "timeslot_types": "[\n    \"other\"\n]",
+      "timeslot_types": [
+        "other"
+      ],
       "used": true
     },
     "model": "name.sessionpurposename",
diff --git a/ietf/name/migrations/0019_alter_sessionpurposename_timeslot_types.py b/ietf/name/migrations/0019_alter_sessionpurposename_timeslot_types.py
new file mode 100644
index 0000000000..a0ca81836d
--- /dev/null
+++ b/ietf/name/migrations/0019_alter_sessionpurposename_timeslot_types.py
@@ -0,0 +1,27 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+import ietf.utils.validators
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("name", "0018_alter_rolenames"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="sessionpurposename",
+            name="timeslot_types",
+            field=models.JSONField(
+                default=list,
+                help_text="Allowed TimeSlotTypeNames",
+                max_length=256,
+                validators=[
+                    ietf.utils.validators.JSONForeignKeyListValidator(
+                        "name.TimeSlotTypeName"
+                    )
+                ],
+            ),
+        ),
+    ]
diff --git a/ietf/name/models.py b/ietf/name/models.py
index 0e87d43548..24104c5f45 100644
--- a/ietf/name/models.py
+++ b/ietf/name/models.py
@@ -1,8 +1,6 @@
 # Copyright The IETF Trust 2010-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-import jsonfield
-
 from django.db import models
 
 from ietf.utils.models import ForeignKey
@@ -73,8 +71,8 @@ class SessionStatusName(NameModel):
     """Waiting for Approval, Approved, Waiting for Scheduling, Scheduled, Cancelled, Disapproved"""
 class SessionPurposeName(NameModel):
     """Regular, Tutorial, Office Hours, Coding, Social, Admin"""
-    timeslot_types = jsonfield.JSONField(
-        max_length=256, blank=False, default=[],
+    timeslot_types = models.JSONField(
+        max_length=256, blank=False, default=list,
         help_text='Allowed TimeSlotTypeNames',
         validators=[JSONForeignKeyListValidator('name.TimeSlotTypeName')],
     )
diff --git a/ietf/person/migrations/0005_alter_historicalperson_pronouns_selectable_and_more.py b/ietf/person/migrations/0005_alter_historicalperson_pronouns_selectable_and_more.py
new file mode 100644
index 0000000000..2af874b1fa
--- /dev/null
+++ b/ietf/person/migrations/0005_alter_historicalperson_pronouns_selectable_and_more.py
@@ -0,0 +1,34 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("person", "0004_alter_person_photo_alter_person_photo_thumb"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="historicalperson",
+            name="pronouns_selectable",
+            field=models.JSONField(
+                blank=True,
+                default=list,
+                max_length=120,
+                null=True,
+                verbose_name="Pronouns",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="person",
+            name="pronouns_selectable",
+            field=models.JSONField(
+                blank=True,
+                default=list,
+                max_length=120,
+                null=True,
+                verbose_name="Pronouns",
+            ),
+        ),
+    ]
diff --git a/ietf/person/models.py b/ietf/person/models.py
index 93364478ae..03cf0c87fb 100644
--- a/ietf/person/models.py
+++ b/ietf/person/models.py
@@ -4,7 +4,6 @@
 
 import email.utils
 import email.header
-import jsonfield
 import uuid
 
 from hashids import Hashids
@@ -57,7 +56,7 @@ class Person(models.Model):
     ascii = models.CharField("Full Name (ASCII)", max_length=255, help_text="Name as rendered in ASCII (Latin, unaccented) characters.", validators=[name_character_validator])
     # The short ascii-form of the name.  Also in alias table if non-null
     ascii_short = models.CharField("Abbreviated Name (ASCII)", max_length=32, null=True, blank=True, help_text="Example: A. Nonymous.  Fill in this with initials and surname only if taking the initials and surname of the ASCII name above produces an incorrect initials-only form. (Blank is OK).", validators=[name_character_validator])
-    pronouns_selectable = jsonfield.JSONCharField("Pronouns", max_length=120, blank=True, null=True, default=list )
+    pronouns_selectable = models.JSONField("Pronouns", max_length=120, blank=True, null=True, default=list )
     pronouns_freetext = models.CharField(" ", max_length=30, null=True, blank=True, help_text="Optionally provide your personal pronouns. These will be displayed on your public profile page and alongside your name in Meetecho and, in future, other systems. Select any number of the checkboxes OR provide a custom string up to 30 characters.")
     biography = models.TextField(blank=True, help_text="Short biography for use on leadership pages. Use plain text or reStructuredText markup.")
     photo = models.ImageField(
diff --git a/ietf/submit/migrations/0003_alter_submission_authors_alter_submissioncheck_items.py b/ietf/submit/migrations/0003_alter_submission_authors_alter_submissioncheck_items.py
new file mode 100644
index 0000000000..2c51659204
--- /dev/null
+++ b/ietf/submit/migrations/0003_alter_submission_authors_alter_submissioncheck_items.py
@@ -0,0 +1,46 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+
+
+def forward(apps, schema_editor):
+    JSON_ENCODED_NULL = r"\u0000"
+    NULL = "\x00"
+    NUL_SYMBOL = "\u2400"  # Unicode single-char "NUL" symbol 
+    
+    Submission = apps.get_model("submit", "Submission")
+    # The qs filter sees the serialized JSON string...
+    null_in_authors = Submission.objects.filter(authors__contains=JSON_ENCODED_NULL)
+    for submission in null_in_authors:
+        # submission.authors is now deserialized into Python objects
+        for author in submission.authors:
+            for k in author:
+                author[k] = author[k].replace(NULL, NUL_SYMBOL)
+        submission.save()
+
+
+def reverse(apps, schema_editor):
+    pass  # don't restore invalid data
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("submit", "0002_alter_submission_xml_version"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+        migrations.AlterField(
+            model_name="submission",
+            name="authors",
+            field=models.JSONField(
+                default=list,
+                help_text="List of authors with name, email, affiliation and country.",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="submissioncheck",
+            name="items",
+            field=models.JSONField(blank=True, default=dict, null=True),
+        ),
+    ]
diff --git a/ietf/submit/models.py b/ietf/submit/models.py
index 1145f761b4..576ba3e114 100644
--- a/ietf/submit/models.py
+++ b/ietf/submit/models.py
@@ -3,7 +3,6 @@
 
 
 import email
-import jsonfield
 
 from django.db import models
 from django.utils import timezone
@@ -46,7 +45,9 @@ class Submission(models.Model):
     words = models.IntegerField(null=True, blank=True)
     formal_languages = models.ManyToManyField(FormalLanguageName, blank=True, help_text="Formal languages used in document")
 
-    authors = jsonfield.JSONField(default=list, help_text="List of authors with name, email, affiliation and country.")
+    authors = models.JSONField(default=list, help_text="List of authors with name, email, affiliation and country.")
+    # Schema note: authors is a list of authors. Each author is a JSON object with 
+    # "name", "email", "affiliation", and "country" keys. All values are strings.
     note = models.TextField(blank=True)
     replaces = models.CharField(max_length=1000, blank=True)
 
@@ -135,7 +136,7 @@ class SubmissionCheck(models.Model):
     message = models.TextField(null=True, blank=True)
     errors = models.IntegerField(null=True, blank=True, default=None)
     warnings = models.IntegerField(null=True, blank=True, default=None)
-    items = jsonfield.JSONField(null=True, blank=True, default='{}')
+    items = models.JSONField(null=True, blank=True, default=dict)
     symbol = models.CharField(max_length=64, default='')
     #
     def __str__(self):
diff --git a/ietf/utils/db.py b/ietf/utils/db.py
index d451f6cfd8..865c9b999f 100644
--- a/ietf/utils/db.py
+++ b/ietf/utils/db.py
@@ -6,12 +6,34 @@
 # 
 # JSONField should recognize {}, (), and [] as valid, non-empty JSON
 # values.  However, the base Field class excludes them
+
 import jsonfield
+from django.db import models
+
+from ietf.utils.fields import IETFJSONField as FormIETFJSONField, EmptyAwareJSONField as FormEmptyAwareJSONField
+
+
+class EmptyAwareJSONField(models.JSONField):
+    form_class = FormEmptyAwareJSONField
+
+    def __init__(self, *args, empty_values=FormEmptyAwareJSONField.empty_values, accepted_empty_values=None, **kwargs):
+        if accepted_empty_values is None:
+            accepted_empty_values = []
+        self.empty_values = [x
+                             for x in empty_values
+                             if x not in accepted_empty_values]
+        super().__init__(*args, **kwargs)
 
-from ietf.utils.fields import IETFJSONField as FormIETFJSONField
+    def formfield(self, **kwargs):
+        if 'form_class' not in kwargs or issubclass(kwargs['form_class'], FormEmptyAwareJSONField):
+            kwargs.setdefault('empty_values', self.empty_values)
+        return super().formfield(**{**kwargs})
 
 
-class IETFJSONField(jsonfield.JSONField):
+class IETFJSONField(jsonfield.JSONField):  # pragma: no cover
+    # Deprecated - use EmptyAwareJSONField instead (different base class requires a
+    # new field name)
+    # Remove this class when migrations are squashed and it is no longer referenced
     form_class = FormIETFJSONField
 
     def __init__(self, *args, empty_values=FormIETFJSONField.empty_values, accepted_empty_values=None, **kwargs):
diff --git a/ietf/utils/fields.py b/ietf/utils/fields.py
index 3e6f56d45e..ba3fecebc6 100644
--- a/ietf/utils/fields.py
+++ b/ietf/utils/fields.py
@@ -6,8 +6,6 @@
 import json
 import re
 
-import jsonfield
-
 import debug                            # pyflakes:ignore
 
 from typing import Optional, Type # pyflakes:ignore
@@ -328,8 +326,21 @@ def has_changed(self, initial, data):
         return super().has_changed(initial, data)
 
 
-class IETFJSONField(jsonfield.fields.forms.JSONField):
-    def __init__(self, *args, empty_values=jsonfield.fields.forms.JSONField.empty_values,
+class IETFJSONField(forms.JSONField):  # pragma: no cover
+    # Deprecated - use EmptyAwareJSONField instead
+    def __init__(self, *args, empty_values=forms.JSONField.empty_values,
+                 accepted_empty_values=None, **kwargs):
+        if accepted_empty_values is None:
+            accepted_empty_values = []
+        self.empty_values = [x
+                             for x in empty_values
+                             if x not in accepted_empty_values]
+
+        super().__init__(*args, **kwargs)
+
+
+class EmptyAwareJSONField(forms.JSONField):
+    def __init__(self, *args, empty_values=forms.JSONField.empty_values,
                  accepted_empty_values=None, **kwargs):
         if accepted_empty_values is None:
             accepted_empty_values = []

From 711313d983d1d5ec9afe8c014a76959e04c04632 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 19 Aug 2025 12:08:11 -0300
Subject: [PATCH 430/656] chore: postgresql-client-17 for base.Dockerfile
 (#9378)

---
 docker/base.Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile
index f364456c7a..57aac8ee56 100644
--- a/docker/base.Dockerfile
+++ b/docker/base.Dockerfile
@@ -69,7 +69,7 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends
 	nodejs \
 	pgloader \
 	pigz \
-	postgresql-client-14 \
+	postgresql-client-17 \
 	pv \
 	python3-ipython \
 	ripgrep \

From 811216838823507b911635967a012bc41f43edd5 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Tue, 19 Aug 2025 15:21:46 +0000
Subject: [PATCH 431/656] ci: update base image target version to 20250819T1508

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 0ecf9566ef..756b7021b7 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250807T1514
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250819T1508
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 327fb48da4..2e0f1519da 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250807T1514
+20250819T1508

From 566cf50b39fa4b0e7882a12b8ce977c29192009f Mon Sep 17 00:00:00 2001
From: "A. Nour" <nouraaproton.me@proton.me>
Date: Tue, 19 Aug 2025 17:52:56 +0100
Subject: [PATCH 432/656] feat: add auto-generated IPR email notifications on
 WG Call for Adoption or WG Last Call (#9322)

* feat(documents): add auto-generated IPR email notifications on WG Call for Adoption or WG Last Call

* fix: edit call durations in mails and call logic under new_state in views

* fix: calc  end_date as  7 * call_duration

* feat(mailtrigger): added new mailtrigger for wg-lc and rfc stream states

* test: add mailtrigger test fixtures and new tests

* fix: use two action-oriented mailtrigger names

The two actions have the same recipients to start with, but that
may change over time. Mailtrigger names should describe "what
happened to trigger this email?". Changed the utility names to
match the actions.

* fix: send from whomever issued the call

Using a list name as the From will not work - the mail infrastructure
blocks such mail when it is submitted.

* chore: revert ietf/doc/tests_draft.py

* fix: trigger call for adoption email from manage adoption view

* fix: changed template names to match functions

* fix: match the subject requested in the issue

* fix: Initial tests

* fix: pass duration to the email message generator

* fix: only issue the c-adopt and wg-lc email for ietf-stream docs

* chore: remove stray whitespace

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/doc/mails.py                             |  55 +++++
 ietf/doc/tests_draft.py                       | 205 +++++++++++++++++-
 ietf/doc/views_draft.py                       |  17 +-
 ..._call_for_adoption_and_last_call_issued.py |  43 ++++
 ietf/name/fixtures/names.json                 |  28 +++
 .../doc/mail/wg_call_for_adoption_issued.txt  |  21 ++
 .../doc/mail/wg_last_call_issued.txt          |  22 ++
 7 files changed, 385 insertions(+), 6 deletions(-)
 create mode 100644 ietf/mailtrigger/migrations/0006_call_for_adoption_and_last_call_issued.py
 create mode 100644 ietf/templates/doc/mail/wg_call_for_adoption_issued.txt
 create mode 100644 ietf/templates/doc/mail/wg_last_call_issued.txt

diff --git a/ietf/doc/mails.py b/ietf/doc/mails.py
index ddecbb6b54..f20d398c3c 100644
--- a/ietf/doc/mails.py
+++ b/ietf/doc/mails.py
@@ -103,6 +103,61 @@ def email_stream_changed(request, doc, old_stream, new_stream, text=""):
               dict(text=text,
                    url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url()),
               cc=cc)
+    
+def email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=None):
+    if cfa_duration_weeks is None:
+        cfa_duration_weeks=2
+    (to, cc) = gather_address_lists("doc_wg_call_for_adoption_issued", doc=doc)
+    frm = request.user.person.formatted_email()
+
+    end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=7 * cfa_duration_weeks)
+
+    subject = f"Call for adoption: {doc.name}-{doc.rev}  (Ends {end_date})"
+
+    send_mail(
+        request,
+        to,
+        frm,
+        subject,
+        "doc/mail/wg_call_for_adoption_issued.txt",
+        dict(
+            doc=doc,
+            subject=subject,
+            url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(),
+            end_date=end_date,
+            cfa_duration_weeks=cfa_duration_weeks,
+            wg_list=doc.group.list_email,
+        ),
+        cc=cc,
+    )
+
+
+def email_wg_last_call_issued(request, doc, wglc_duration_weeks=None):
+    if wglc_duration_weeks is None:
+        wglc_duration_weeks = 2
+    (to, cc) = gather_address_lists("doc_wg_last_call_issued", doc=doc)
+    frm = request.user.person.formatted_email()
+
+
+    end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=7 * wglc_duration_weeks)
+    subject =  f"WG Last Call: {doc.name}-{doc.rev} (Ends {end_date})"
+
+    send_mail(
+        request,
+        to,
+        frm,
+        subject,
+        "doc/mail/wg_last_call_issued.txt",
+        dict(
+            doc=doc,
+            subject=subject,
+            url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(),
+            end_date=end_date,
+            wglc_duration_weeks=wglc_duration_weeks,
+            wg_list=doc.group.list_email,
+        ),
+        cc=cc,
+    )
 
 def email_pulled_from_rfc_queue(request, doc, comment, prev_state, next_state):
     extra=extra_automation_headers(doc)
diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py
index 576feb0582..ab7eaba768 100644
--- a/ietf/doc/tests_draft.py
+++ b/ietf/doc/tests_draft.py
@@ -1707,11 +1707,12 @@ def test_adopt_document(self):
                 self.assertEqual(draft.group, chair_role.group)
                 self.assertEqual(draft.stream_id, stream_state_type_slug[type_id][13:]) # trim off "draft-stream-"
                 self.assertEqual(draft.docevent_set.count() - events_before, 5)
-                self.assertEqual(len(outbox), 1)
-                self.assertTrue("Call For Adoption" in outbox[-1]["Subject"])
-                self.assertTrue(f"{chair_role.group.acronym}-chairs@" in outbox[-1]['To'])
-                self.assertTrue(f"{draft.name}@" in outbox[-1]['To'])
-                self.assertTrue(f"{chair_role.group.acronym}@" in outbox[-1]['To'])
+                self.assertEqual(len(outbox), 2)
+                self.assertTrue("Call For Adoption" in outbox[0]["Subject"])
+                self.assertTrue(f"{chair_role.group.acronym}-chairs@" in outbox[0]['To'])
+                self.assertTrue(f"{draft.name}@" in outbox[0]['To'])
+                self.assertTrue(f"{chair_role.group.acronym}@" in outbox[0]['To'])
+                # contents of outbox[1] are tested elsewhere
 
             # adopt
             empty_outbox()
@@ -2001,6 +2002,200 @@ def test_set_state(self):
         self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string())
         self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string())
 
+    def test_wg_call_for_adoption_issued(self):
+        role = RoleFactory(
+            name_id="chair",
+            group__acronym="mars",
+            group__list_email="mars-wg@ietf.org",
+            person__user__username="marschairman",
+            person__name="WG Cháir Man",
+        )
+        # First test the usual workflow through the manage adoption view
+        draft = IndividualDraftFactory()
+        url = urlreverse(
+            "ietf.doc.views_draft.adopt_draft", kwargs=dict(name=draft.name)
+        )
+        login_testing_unauthorized(self, "marschairman", url)
+        empty_outbox()
+        call_issued = State.objects.get(type="draft-stream-ietf", slug="c-adopt")
+        r = self.client.post(
+            url,
+            dict(
+                comment="some comment",
+                group=role.group.pk,
+                newstate=call_issued.pk,
+                weeks="10",
+            ),
+        )
+        self.assertEqual(r.status_code, 302)
+        self.assertEqual(len(outbox), 2)
+        self.assertIn("mars-wg@ietf.org", outbox[1]["To"])
+        self.assertIn("Call for adoption", outbox[1]["Subject"])
+        body = get_payload_text(outbox[1])
+        self.assertIn("disclosure obligations", body)
+        self.assertIn("starts a 10-week", body)
+        # Test not entering a duration on the form
+        draft = IndividualDraftFactory()
+        url = urlreverse(
+            "ietf.doc.views_draft.adopt_draft", kwargs=dict(name=draft.name)
+        )
+        empty_outbox()
+        call_issued = State.objects.get(type="draft-stream-ietf", slug="c-adopt")
+        r = self.client.post(
+            url,
+            dict(
+                comment="some comment",
+                group=role.group.pk,
+                newstate=call_issued.pk,
+            ),
+        )
+        self.assertEqual(r.status_code, 302)
+        self.assertEqual(len(outbox), 2)
+        self.assertIn("mars-wg@ietf.org", outbox[1]["To"])
+        self.assertIn("Call for adoption", outbox[1]["Subject"])
+        body = get_payload_text(outbox[1])
+        self.assertIn("disclosure obligations", body)
+        self.assertIn("starts a 2-week", body)
+
+        # Test the less usual workflow of issuing a call for adoption 
+        # of a document that's already in the ietf stream
+        draft = WgDraftFactory(group=role.group)
+        url = urlreverse(
+            "ietf.doc.views_draft.change_stream_state",
+            kwargs=dict(name=draft.name, state_type="draft-stream-ietf"),
+        )
+        old_state = draft.get_state("draft-stream-%s" % draft.stream_id)
+        new_state = State.objects.get(
+            used=True, type="draft-stream-%s" % draft.stream_id, slug="c-adopt"
+        )
+        self.assertNotEqual(old_state, new_state)
+        empty_outbox()
+        r = self.client.post(
+            url,
+            dict(
+                new_state=new_state.pk,
+                comment="some comment",
+                weeks="10",
+                tags=[
+                    t.pk
+                    for t in draft.tags.filter(
+                        slug__in=get_tags_for_stream_id(draft.stream_id)
+                    )
+                ],
+            ),
+        )
+        self.assertEqual(r.status_code, 302)
+        self.assertEqual(len(outbox), 2)
+        self.assertIn("mars-wg@ietf.org", outbox[1]["To"])
+        self.assertIn("Call for adoption", outbox[1]["Subject"])
+        body = get_payload_text(outbox[1])
+        self.assertIn("disclosure obligations", body)
+        self.assertIn("starts a 10-week", body)
+        draft = WgDraftFactory(group=role.group)
+        url = urlreverse(
+            "ietf.doc.views_draft.change_stream_state",
+            kwargs=dict(name=draft.name, state_type="draft-stream-ietf"),
+        )
+        old_state = draft.get_state("draft-stream-%s" % draft.stream_id)
+        new_state = State.objects.get(
+            used=True, type="draft-stream-%s" % draft.stream_id, slug="c-adopt"
+        )
+        self.assertNotEqual(old_state, new_state)
+        empty_outbox()
+        r = self.client.post(
+            url,
+            dict(
+                new_state=new_state.pk,
+                comment="some comment",
+                tags=[
+                    t.pk
+                    for t in draft.tags.filter(
+                        slug__in=get_tags_for_stream_id(draft.stream_id)
+                    )
+                ],
+            ),
+        )
+        self.assertEqual(r.status_code, 302)
+        self.assertEqual(len(outbox), 2)
+        self.assertIn("mars-wg@ietf.org", outbox[1]["To"])
+        self.assertIn("Call for adoption", outbox[1]["Subject"])
+        body = get_payload_text(outbox[1])
+        self.assertIn("disclosure obligations", body)
+        self.assertIn("starts a 2-week", body)
+
+    def test_wg_last_call_issued(self):
+        role = RoleFactory(
+            name_id="chair",
+            group__acronym="mars",
+            group__list_email="mars-wg@ietf.org",
+            person__user__username="marschairman",
+            person__name="WG Cháir Man",
+        )
+        draft = WgDraftFactory(group=role.group)
+        url = urlreverse(
+            "ietf.doc.views_draft.change_stream_state",
+            kwargs=dict(name=draft.name, state_type="draft-stream-ietf"),
+        )
+        login_testing_unauthorized(self, "marschairman", url)
+        old_state = draft.get_state("draft-stream-%s" % draft.stream_id)
+        new_state = State.objects.get(
+            used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-lc"
+        )
+        self.assertNotEqual(old_state, new_state)
+        empty_outbox()
+        r = self.client.post(
+            url,
+            dict(
+                new_state=new_state.pk,
+                comment="some comment",
+                weeks="10",
+                tags=[
+                    t.pk
+                    for t in draft.tags.filter(
+                        slug__in=get_tags_for_stream_id(draft.stream_id)
+                    )
+                ],
+            ),
+        )
+        self.assertEqual(r.status_code, 302)
+        self.assertEqual(len(outbox), 2)
+        self.assertIn("mars-wg@ietf.org", outbox[1]["To"])
+        self.assertIn("WG Last Call", outbox[1]["Subject"])
+        body = get_payload_text(outbox[1])
+        self.assertIn("disclosure obligations", body)
+        self.assertIn("starts a 10-week", body)
+        draft = WgDraftFactory(group=role.group)
+        url = urlreverse(
+            "ietf.doc.views_draft.change_stream_state",
+            kwargs=dict(name=draft.name, state_type="draft-stream-ietf"),
+        )
+        old_state = draft.get_state("draft-stream-%s" % draft.stream_id)
+        new_state = State.objects.get(
+            used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-lc"
+        )
+        self.assertNotEqual(old_state, new_state)
+        empty_outbox()
+        r = self.client.post(
+            url,
+            dict(
+                new_state=new_state.pk,
+                comment="some comment",
+                tags=[
+                    t.pk
+                    for t in draft.tags.filter(
+                        slug__in=get_tags_for_stream_id(draft.stream_id)
+                    )
+                ],
+            ),
+        )
+        self.assertEqual(r.status_code, 302)
+        self.assertEqual(len(outbox), 2)
+        self.assertIn("mars-wg@ietf.org", outbox[1]["To"])
+        self.assertIn("WG Last Call", outbox[1]["Subject"])
+        body = get_payload_text(outbox[1])
+        self.assertIn("disclosure obligations", body)
+        self.assertIn("starts a 2-week", body)
+
     def test_pubreq_validation(self):
         role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man')
         RoleFactory(name_id='delegate',group=role.group,person__user__email='marsdelegate@ietf.org')
diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py
index c80537afb3..16d04ee66a 100644
--- a/ietf/doc/views_draft.py
+++ b/ietf/doc/views_draft.py
@@ -28,6 +28,7 @@
     IanaExpertDocEvent, IESG_SUBSTATE_TAGS)
 from ietf.doc.mails import ( email_pulled_from_rfc_queue, email_resurrect_requested,
     email_resurrection_completed, email_state_changed, email_stream_changed,
+    email_wg_call_for_adoption_issued, email_wg_last_call_issued,
     email_stream_state_changed, email_stream_tags_changed, extra_automation_headers,
     generate_publication_request, email_adopted, email_intended_status_changed,
     email_iesg_processing_document, email_ad_approved_doc,
@@ -1568,8 +1569,15 @@ def adopt_draft(request, name):
 
                 update_reminder(doc, "stream-s", e, due_date)
 
+                # The following call name is very misleading - the view allows 
+                # setting states that are _not_ the adopted state.
                 email_adopted(request, doc, prev_state, new_state, by, comment)
 
+                # Currently only the IETF stream uses the c-adopt state - guard against other
+                # streams starting to use it asthe IPR rules for those streams will be different.
+                if doc.stream_id == "ietf" and new_state.slug == "c-adopt":
+                    email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=form.cleaned_data["weeks"])
+
             # comment
             if comment:
                 e = DocEvent(type="added_comment", doc=doc, rev=doc.rev, by=by)
@@ -1754,13 +1762,20 @@ def change_stream_state(request, name, state_type):
                 events.append(e)
 
                 due_date = None
-                if form.cleaned_data["weeks"] != None:
+                if form.cleaned_data["weeks"] is not None:
                     due_date = datetime_today(DEADLINE_TZINFO) + datetime.timedelta(weeks=form.cleaned_data["weeks"])
 
                 update_reminder(doc, "stream-s", e, due_date)
 
                 email_stream_state_changed(request, doc, prev_state, new_state, by, comment)
 
+                if doc.stream_id == "ietf":
+                    if new_state.slug == "c-adopt":
+                        email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=form.cleaned_data["weeks"])
+                    
+                    if new_state.slug == "wg-lc":
+                        email_wg_last_call_issued(request, doc, wglc_duration_weeks=form.cleaned_data["weeks"])
+
             # tags
             existing_tags = set(doc.tags.all())
             new_tags = set(form.cleaned_data["tags"])
diff --git a/ietf/mailtrigger/migrations/0006_call_for_adoption_and_last_call_issued.py b/ietf/mailtrigger/migrations/0006_call_for_adoption_and_last_call_issued.py
new file mode 100644
index 0000000000..7adad150eb
--- /dev/null
+++ b/ietf/mailtrigger/migrations/0006_call_for_adoption_and_last_call_issued.py
@@ -0,0 +1,43 @@
+# Copyright The IETF Trust 2023, All Rights Reserved
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    MailTrigger = apps.get_model("mailtrigger", "MailTrigger")
+    Recipient = apps.get_model("mailtrigger", "Recipient")
+    recipients = list(
+        Recipient.objects.filter(
+            slug__in=(
+                "doc_group_mail_list",
+                "doc_authors",
+                "doc_group_chairs",
+                "doc_shepherd",
+            )
+        )
+    )
+    call_for_adoption = MailTrigger.objects.create(
+        slug="doc_wg_call_for_adoption_issued",
+        desc="Recipients when a working group call for adoption is issued",
+    )
+    call_for_adoption.to.add(*recipients)
+    wg_last_call = MailTrigger.objects.create(
+        slug="doc_wg_last_call_issued",
+        desc="Recipients when a working group last call is issued",
+    )
+    wg_last_call.to.add(*recipients)
+
+
+def reverse(apps, schema_editor):
+    MailTrigger = apps.get_model("mailtrigger", "MailTrigger")
+    MailTrigger.objects.filter(
+        slug_in=("doc_wg_call_for_adoption_issued", "doc_wg_last_call_issued")
+    ).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("mailtrigger", "0005_rfc_recipients"),
+    ]
+
+    operations = [migrations.RunPython(forward, reverse)]
diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json
index 0724cbb4b5..c94e15a459 100644
--- a/ietf/name/fixtures/names.json
+++ b/ietf/name/fixtures/names.json
@@ -5116,6 +5116,34 @@
     "model": "mailtrigger.mailtrigger",
     "pk": "doc_telechat_details_changed"
   },
+  {
+    "fields": {
+      "cc": [],
+      "desc": "Recipients when a working group call for adoption is issued",
+      "to": [
+        "doc_authors",
+        "doc_group_chairs",
+        "doc_group_mail_list",
+        "doc_shepherd"
+      ]
+    },
+    "model": "mailtrigger.mailtrigger",
+    "pk": "doc_wg_call_for_adoption_issued"
+  },
+  {
+    "fields": {
+      "cc": [],
+      "desc": "Recipients when a working group last call is issued",
+      "to": [
+        "doc_authors",
+        "doc_group_chairs",
+        "doc_group_mail_list",
+        "doc_shepherd"
+      ]
+    },
+    "model": "mailtrigger.mailtrigger",
+    "pk": "doc_wg_last_call_issued"
+  },
   {
     "fields": {
       "cc": [],
diff --git a/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt b/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt
new file mode 100644
index 0000000000..c4a2401bc2
--- /dev/null
+++ b/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt
@@ -0,0 +1,21 @@
+{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %}
+Subject: {{ subject }}
+
+This message starts a {{ cfa_duration_weeks }}-week Call for Adoption for this document. 
+
+Abstract: 
+{{ doc.abstract }}
+
+File can be retrieved from:
+{{ url }}
+
+Please reply to this message keeping {{ wg_list }} in copy by indicating whether you support or not the adoption of this draft as a WG document. Comments to motivate your preference are highly appreciated. 
+
+Authors, and WG participants in general, are reminded of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [2]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3].
+
+Thank you.
+[1] https://datatracker.ietf.org/doc/bcp78/
+[2] https://datatracker.ietf.org/doc/bcp79/
+[3] https://datatracker.ietf.org/doc/rfc6701/
+{% endfilter %}
+{% endautoescape %}
diff --git a/ietf/templates/doc/mail/wg_last_call_issued.txt b/ietf/templates/doc/mail/wg_last_call_issued.txt
new file mode 100644
index 0000000000..35b1e149d7
--- /dev/null
+++ b/ietf/templates/doc/mail/wg_last_call_issued.txt
@@ -0,0 +1,22 @@
+{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %}
+Subject: {{ subject }}
+
+This message starts a {{ wglc_duration_weeks }}-week WG Last Call for this document. 
+
+Abstract: 
+{{ doc.abstract }}
+
+File can be retrieved from:
+{{ url }}
+
+Please review and indicate your support or objection to proceed with the publication of this document by replying to this email keeping {{ wg_list }} in copy. Objections should be motivated and suggestions to resolve them are highly appreciated.
+
+Authors, and WG participants in general, are reminded again of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [1]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3].
+
+Thank you.
+
+[1] https://datatracker.ietf.org/doc/bcp78/
+[2] https://datatracker.ietf.org/doc/bcp79/
+[3] https://datatracker.ietf.org/doc/rfc6701/
+{% endfilter %}
+{% endautoescape %}

From 1e451fbac105b18a3539c406b8380890ee701ef8 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 19 Aug 2025 18:12:38 -0300
Subject: [PATCH 433/656] ci: replace DOCKER_BUILD_NO_SUMMARY (#9383)

Option is deprecated; replace with DOCKER_BUILD_SUMMARY: false
---
 .github/workflows/build-base-app.yml          | 2 +-
 .github/workflows/build-devblobstore.yml      | 2 +-
 .github/workflows/build-mq-broker.yml         | 2 +-
 .github/workflows/dev-assets-sync-nightly.yml | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index 479cd7cadf..ef8a17f6b4 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -43,7 +43,7 @@ jobs:
     - name: Docker Build & Push
       uses: docker/build-push-action@v6
       env:
-        DOCKER_BUILD_NO_SUMMARY: true
+        DOCKER_BUILD_SUMMARY: false
       with:
         context: .
         file: docker/base.Dockerfile
diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml
index 2d8214b448..f49a11af19 100644
--- a/.github/workflows/build-devblobstore.yml
+++ b/.github/workflows/build-devblobstore.yml
@@ -35,7 +35,7 @@ jobs:
     - name: Docker Build & Push
       uses: docker/build-push-action@v6
       env:
-        DOCKER_BUILD_NO_SUMMARY: true
+        DOCKER_BUILD_SUMMARY: false
       with:
         context: .
         file: docker/devblobstore.Dockerfile
diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml
index 85c27c23cc..4de861dbcd 100644
--- a/.github/workflows/build-mq-broker.yml
+++ b/.github/workflows/build-mq-broker.yml
@@ -42,7 +42,7 @@ jobs:
     - name: Docker Build & Push
       uses: docker/build-push-action@v6
       env:
-        DOCKER_BUILD_NO_SUMMARY: true
+        DOCKER_BUILD_SUMMARY: false
       with:
         context: .
         file: dev/mq/Dockerfile
diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml
index a7fe67f012..19933bddfd 100644
--- a/.github/workflows/dev-assets-sync-nightly.yml
+++ b/.github/workflows/dev-assets-sync-nightly.yml
@@ -41,7 +41,7 @@ jobs:
       - name: Docker Build & Push
         uses: docker/build-push-action@v6
         env:
-          DOCKER_BUILD_NO_SUMMARY: true
+          DOCKER_BUILD_SUMMARY: false
         with:
           context: .
           file: dev/shared-assets-sync/Dockerfile

From b7da3d7a779f310f765f205119624229d9a860d7 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 21 Aug 2025 10:25:49 -0300
Subject: [PATCH 434/656] fix: escape linkify filter input (#9389)

* fix: escape linkify filter input

* test: exercise linkify

* chore: lint
---
 ietf/utils/templatetags/tests.py       | 66 ++++++++++++++++++++++++++
 ietf/utils/templatetags/textfilters.py | 10 ++--
 ietf/utils/text.py                     |  6 +++
 3 files changed, 79 insertions(+), 3 deletions(-)

diff --git a/ietf/utils/templatetags/tests.py b/ietf/utils/templatetags/tests.py
index a93bf2d94d..859319be3d 100644
--- a/ietf/utils/templatetags/tests.py
+++ b/ietf/utils/templatetags/tests.py
@@ -3,6 +3,7 @@
 from django.template import Context, Origin, Template
 from django.test import override_settings
 
+from ietf.utils.templatetags.textfilters import linkify
 from ietf.utils.test_utils import TestCase
 import debug  # pyflakes: ignore
 
@@ -39,3 +40,68 @@ def test_origin_outside_base_dir(self):
                 output = template.render(Context())
                 self.assertNotIn(component, output,
                                  'Full path components should not be revealed in html')
+
+
+class TextfiltersTests(TestCase):
+    def test_linkify(self):
+        # Cases with autoescape = True (the default)
+        self.assertEqual(
+            linkify("plain string"),
+            "plain string",
+        )
+        self.assertEqual(
+            linkify("https://www.ietf.org"),
+            '<a href="https://www.ietf.org">https://www.ietf.org</a>',
+        )
+        self.assertEqual(
+            linkify('<a href="https://www.ietf.org">IETF</a>'),
+            (
+                '&lt;a href=&quot;<a href="https://www.ietf.org">https://www.ietf.org</a>&quot;&gt;IETF&lt;/a&gt;'
+            ),
+        )
+        self.assertEqual(
+            linkify("somebody@example.com"),
+            '<a href="mailto:somebody@example.com">somebody@example.com</a>',
+        )
+        self.assertEqual(
+            linkify("Some Body <somebody@example.com>"),
+            (
+                'Some Body &lt;<a href="mailto:somebody@example.com">'
+                'somebody@example.com</a>&gt;'
+            ),
+        )
+        self.assertEqual(
+            linkify("<script>alert('h4x0r3d');</script>"),
+            "&lt;script&gt;alert(&#x27;h4x0r3d&#x27;);&lt;/script&gt;",
+        )
+
+        # Cases with autoescape = False (these are dangerous and assume the caller
+        # has sanitized already)
+        self.assertEqual(
+            linkify("plain string", autoescape=False),
+            "plain string",
+        )
+        self.assertEqual(
+            linkify("https://www.ietf.org", autoescape=False),
+            '<a href="https://www.ietf.org">https://www.ietf.org</a>',
+        )
+        self.assertEqual(
+            linkify('<a href="https://www.ietf.org">IETF</a>', autoescape=False),
+            '<a href="https://www.ietf.org">IETF</a>',
+        )
+        self.assertEqual(
+            linkify("somebody@example.com", autoescape=False),
+            '<a href="mailto:somebody@example.com">somebody@example.com</a>',
+        )
+        # bleach.Linkifier translates the < -> &lt; and > -> &gt; on this one
+        self.assertEqual(
+            linkify("Some Body <somebody@example.com>", autoescape=False),
+            (
+                'Some Body &lt;<a href="mailto:somebody@example.com">'
+                'somebody@example.com</a>&gt;'
+            ),
+        )
+        self.assertEqual(
+            linkify("<script>alert('friendly script');</script>", autoescape=False),
+            "<script>alert('friendly script');</script>",
+        )
diff --git a/ietf/utils/templatetags/textfilters.py b/ietf/utils/templatetags/textfilters.py
index 3b240740e0..e3bfbe0c56 100644
--- a/ietf/utils/templatetags/textfilters.py
+++ b/ietf/utils/templatetags/textfilters.py
@@ -7,6 +7,7 @@
 from django import template
 from django.conf import settings
 from django.template.defaultfilters import stringfilter
+from django.utils.html import conditional_escape
 from django.utils.safestring import mark_safe
 
 import debug                            # pyflakes:ignore
@@ -71,10 +72,13 @@ def texescape_filter(value):
     "A TeX escape filter"
     return texescape(value)
     
-@register.filter
+@register.filter(needs_autoescape=True)
 @stringfilter
-def linkify(value):
-    text = mark_safe(_linkify(value))
+def linkify(value, autoescape=True):
+    if autoescape:
+        # Escape unless the input was already a SafeString
+        value = conditional_escape(value)
+    text = mark_safe(_linkify(value))  # _linkify is a safe operation
     return text
 
 @register.filter
diff --git a/ietf/utils/text.py b/ietf/utils/text.py
index 4e5d5b6cd5..590ec3fd30 100644
--- a/ietf/utils/text.py
+++ b/ietf/utils/text.py
@@ -60,6 +60,12 @@ def check_url_validity(attrs, new=False):
 
 
 def linkify(text):
+    """Convert URL-ish substrings into HTML links
+    
+    This does no sanitization whatsoever. Caller must sanitize the input or output as
+    contextually appropriate. Do not call `mark_safe()` on the output if the input is
+    user-provided unless it has been sanitized or escaped.
+    """
     return _bleach_linker.linkify(text)
 
 

From 450ffd8e9e586fbfbc9c583a01ee4770de4f62d9 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 21 Aug 2025 10:26:38 -0300
Subject: [PATCH 435/656] chore(deps): bump pytz (#9385)

* chore(deps): bump pytz

* chore(deps): fix copy/paste error
---
 ...meeting_country_alter_meeting_time_zone.py | 693 ++++++++++++++++++
 requirements.txt                              |   4 +-
 2 files changed, 695 insertions(+), 2 deletions(-)
 create mode 100644 ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py

diff --git a/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py b/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py
new file mode 100644
index 0000000000..8f5db26112
--- /dev/null
+++ b/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py
@@ -0,0 +1,693 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("meeting", "0015_alter_meeting_time_zone"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="meeting",
+            name="country",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("", "---------"),
+                    ("AF", "Afghanistan"),
+                    ("AL", "Albania"),
+                    ("DZ", "Algeria"),
+                    ("AD", "Andorra"),
+                    ("AO", "Angola"),
+                    ("AI", "Anguilla"),
+                    ("AQ", "Antarctica"),
+                    ("AG", "Antigua & Barbuda"),
+                    ("AR", "Argentina"),
+                    ("AM", "Armenia"),
+                    ("AW", "Aruba"),
+                    ("AU", "Australia"),
+                    ("AT", "Austria"),
+                    ("AZ", "Azerbaijan"),
+                    ("BS", "Bahamas"),
+                    ("BH", "Bahrain"),
+                    ("BD", "Bangladesh"),
+                    ("BB", "Barbados"),
+                    ("BY", "Belarus"),
+                    ("BE", "Belgium"),
+                    ("BZ", "Belize"),
+                    ("BJ", "Benin"),
+                    ("BM", "Bermuda"),
+                    ("BT", "Bhutan"),
+                    ("BO", "Bolivia"),
+                    ("BA", "Bosnia & Herzegovina"),
+                    ("BW", "Botswana"),
+                    ("BV", "Bouvet Island"),
+                    ("BR", "Brazil"),
+                    ("GB", "Britain (UK)"),
+                    ("IO", "British Indian Ocean Territory"),
+                    ("BN", "Brunei"),
+                    ("BG", "Bulgaria"),
+                    ("BF", "Burkina Faso"),
+                    ("BI", "Burundi"),
+                    ("KH", "Cambodia"),
+                    ("CM", "Cameroon"),
+                    ("CA", "Canada"),
+                    ("CV", "Cape Verde"),
+                    ("BQ", "Caribbean NL"),
+                    ("KY", "Cayman Islands"),
+                    ("CF", "Central African Rep."),
+                    ("TD", "Chad"),
+                    ("CL", "Chile"),
+                    ("CN", "China"),
+                    ("CX", "Christmas Island"),
+                    ("CC", "Cocos (Keeling) Islands"),
+                    ("CO", "Colombia"),
+                    ("KM", "Comoros"),
+                    ("CD", "Congo (Dem. Rep.)"),
+                    ("CG", "Congo (Rep.)"),
+                    ("CK", "Cook Islands"),
+                    ("CR", "Costa Rica"),
+                    ("HR", "Croatia"),
+                    ("CU", "Cuba"),
+                    ("CW", "Curaçao"),
+                    ("CY", "Cyprus"),
+                    ("CZ", "Czech Republic"),
+                    ("CI", "Côte d'Ivoire"),
+                    ("DK", "Denmark"),
+                    ("DJ", "Djibouti"),
+                    ("DM", "Dominica"),
+                    ("DO", "Dominican Republic"),
+                    ("TL", "East Timor"),
+                    ("EC", "Ecuador"),
+                    ("EG", "Egypt"),
+                    ("SV", "El Salvador"),
+                    ("GQ", "Equatorial Guinea"),
+                    ("ER", "Eritrea"),
+                    ("EE", "Estonia"),
+                    ("SZ", "Eswatini (Swaziland)"),
+                    ("ET", "Ethiopia"),
+                    ("FK", "Falkland Islands"),
+                    ("FO", "Faroe Islands"),
+                    ("FJ", "Fiji"),
+                    ("FI", "Finland"),
+                    ("FR", "France"),
+                    ("GF", "French Guiana"),
+                    ("PF", "French Polynesia"),
+                    ("TF", "French S. Terr."),
+                    ("GA", "Gabon"),
+                    ("GM", "Gambia"),
+                    ("GE", "Georgia"),
+                    ("DE", "Germany"),
+                    ("GH", "Ghana"),
+                    ("GI", "Gibraltar"),
+                    ("GR", "Greece"),
+                    ("GL", "Greenland"),
+                    ("GD", "Grenada"),
+                    ("GP", "Guadeloupe"),
+                    ("GU", "Guam"),
+                    ("GT", "Guatemala"),
+                    ("GG", "Guernsey"),
+                    ("GN", "Guinea"),
+                    ("GW", "Guinea-Bissau"),
+                    ("GY", "Guyana"),
+                    ("HT", "Haiti"),
+                    ("HM", "Heard Island & McDonald Islands"),
+                    ("HN", "Honduras"),
+                    ("HK", "Hong Kong"),
+                    ("HU", "Hungary"),
+                    ("IS", "Iceland"),
+                    ("IN", "India"),
+                    ("ID", "Indonesia"),
+                    ("IR", "Iran"),
+                    ("IQ", "Iraq"),
+                    ("IE", "Ireland"),
+                    ("IM", "Isle of Man"),
+                    ("IL", "Israel"),
+                    ("IT", "Italy"),
+                    ("JM", "Jamaica"),
+                    ("JP", "Japan"),
+                    ("JE", "Jersey"),
+                    ("JO", "Jordan"),
+                    ("KZ", "Kazakhstan"),
+                    ("KE", "Kenya"),
+                    ("KI", "Kiribati"),
+                    ("KP", "Korea (North)"),
+                    ("KR", "Korea (South)"),
+                    ("KW", "Kuwait"),
+                    ("KG", "Kyrgyzstan"),
+                    ("LA", "Laos"),
+                    ("LV", "Latvia"),
+                    ("LB", "Lebanon"),
+                    ("LS", "Lesotho"),
+                    ("LR", "Liberia"),
+                    ("LY", "Libya"),
+                    ("LI", "Liechtenstein"),
+                    ("LT", "Lithuania"),
+                    ("LU", "Luxembourg"),
+                    ("MO", "Macau"),
+                    ("MG", "Madagascar"),
+                    ("MW", "Malawi"),
+                    ("MY", "Malaysia"),
+                    ("MV", "Maldives"),
+                    ("ML", "Mali"),
+                    ("MT", "Malta"),
+                    ("MH", "Marshall Islands"),
+                    ("MQ", "Martinique"),
+                    ("MR", "Mauritania"),
+                    ("MU", "Mauritius"),
+                    ("YT", "Mayotte"),
+                    ("MX", "Mexico"),
+                    ("FM", "Micronesia"),
+                    ("MD", "Moldova"),
+                    ("MC", "Monaco"),
+                    ("MN", "Mongolia"),
+                    ("ME", "Montenegro"),
+                    ("MS", "Montserrat"),
+                    ("MA", "Morocco"),
+                    ("MZ", "Mozambique"),
+                    ("MM", "Myanmar (Burma)"),
+                    ("NA", "Namibia"),
+                    ("NR", "Nauru"),
+                    ("NP", "Nepal"),
+                    ("NL", "Netherlands"),
+                    ("NC", "New Caledonia"),
+                    ("NZ", "New Zealand"),
+                    ("NI", "Nicaragua"),
+                    ("NE", "Niger"),
+                    ("NG", "Nigeria"),
+                    ("NU", "Niue"),
+                    ("NF", "Norfolk Island"),
+                    ("MK", "North Macedonia"),
+                    ("MP", "Northern Mariana Islands"),
+                    ("NO", "Norway"),
+                    ("OM", "Oman"),
+                    ("PK", "Pakistan"),
+                    ("PW", "Palau"),
+                    ("PS", "Palestine"),
+                    ("PA", "Panama"),
+                    ("PG", "Papua New Guinea"),
+                    ("PY", "Paraguay"),
+                    ("PE", "Peru"),
+                    ("PH", "Philippines"),
+                    ("PN", "Pitcairn"),
+                    ("PL", "Poland"),
+                    ("PT", "Portugal"),
+                    ("PR", "Puerto Rico"),
+                    ("QA", "Qatar"),
+                    ("RO", "Romania"),
+                    ("RU", "Russia"),
+                    ("RW", "Rwanda"),
+                    ("RE", "Réunion"),
+                    ("AS", "Samoa (American)"),
+                    ("WS", "Samoa (western)"),
+                    ("SM", "San Marino"),
+                    ("ST", "Sao Tome & Principe"),
+                    ("SA", "Saudi Arabia"),
+                    ("SN", "Senegal"),
+                    ("RS", "Serbia"),
+                    ("SC", "Seychelles"),
+                    ("SL", "Sierra Leone"),
+                    ("SG", "Singapore"),
+                    ("SK", "Slovakia"),
+                    ("SI", "Slovenia"),
+                    ("SB", "Solomon Islands"),
+                    ("SO", "Somalia"),
+                    ("ZA", "South Africa"),
+                    ("GS", "South Georgia & the South Sandwich Islands"),
+                    ("SS", "South Sudan"),
+                    ("ES", "Spain"),
+                    ("LK", "Sri Lanka"),
+                    ("BL", "St Barthelemy"),
+                    ("SH", "St Helena"),
+                    ("KN", "St Kitts & Nevis"),
+                    ("LC", "St Lucia"),
+                    ("SX", "St Maarten (Dutch)"),
+                    ("MF", "St Martin (French)"),
+                    ("PM", "St Pierre & Miquelon"),
+                    ("VC", "St Vincent"),
+                    ("SD", "Sudan"),
+                    ("SR", "Suriname"),
+                    ("SJ", "Svalbard & Jan Mayen"),
+                    ("SE", "Sweden"),
+                    ("CH", "Switzerland"),
+                    ("SY", "Syria"),
+                    ("TW", "Taiwan"),
+                    ("TJ", "Tajikistan"),
+                    ("TZ", "Tanzania"),
+                    ("TH", "Thailand"),
+                    ("TG", "Togo"),
+                    ("TK", "Tokelau"),
+                    ("TO", "Tonga"),
+                    ("TT", "Trinidad & Tobago"),
+                    ("TN", "Tunisia"),
+                    ("TR", "Turkey"),
+                    ("TM", "Turkmenistan"),
+                    ("TC", "Turks & Caicos Is"),
+                    ("TV", "Tuvalu"),
+                    ("UM", "US minor outlying islands"),
+                    ("UG", "Uganda"),
+                    ("UA", "Ukraine"),
+                    ("AE", "United Arab Emirates"),
+                    ("US", "United States"),
+                    ("UY", "Uruguay"),
+                    ("UZ", "Uzbekistan"),
+                    ("VU", "Vanuatu"),
+                    ("VA", "Vatican City"),
+                    ("VE", "Venezuela"),
+                    ("VN", "Vietnam"),
+                    ("VG", "Virgin Islands (UK)"),
+                    ("VI", "Virgin Islands (US)"),
+                    ("WF", "Wallis & Futuna"),
+                    ("EH", "Western Sahara"),
+                    ("YE", "Yemen"),
+                    ("ZM", "Zambia"),
+                    ("ZW", "Zimbabwe"),
+                    ("AX", "Åland Islands"),
+                ],
+                max_length=2,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="meeting",
+            name="time_zone",
+            field=models.CharField(
+                choices=[
+                    ("", "---------"),
+                    ("Africa/Abidjan", "Africa/Abidjan"),
+                    ("Africa/Accra", "Africa/Accra"),
+                    ("Africa/Addis_Ababa", "Africa/Addis_Ababa"),
+                    ("Africa/Algiers", "Africa/Algiers"),
+                    ("Africa/Asmara", "Africa/Asmara"),
+                    ("Africa/Bamako", "Africa/Bamako"),
+                    ("Africa/Bangui", "Africa/Bangui"),
+                    ("Africa/Banjul", "Africa/Banjul"),
+                    ("Africa/Bissau", "Africa/Bissau"),
+                    ("Africa/Blantyre", "Africa/Blantyre"),
+                    ("Africa/Brazzaville", "Africa/Brazzaville"),
+                    ("Africa/Bujumbura", "Africa/Bujumbura"),
+                    ("Africa/Cairo", "Africa/Cairo"),
+                    ("Africa/Casablanca", "Africa/Casablanca"),
+                    ("Africa/Ceuta", "Africa/Ceuta"),
+                    ("Africa/Conakry", "Africa/Conakry"),
+                    ("Africa/Dakar", "Africa/Dakar"),
+                    ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"),
+                    ("Africa/Djibouti", "Africa/Djibouti"),
+                    ("Africa/Douala", "Africa/Douala"),
+                    ("Africa/El_Aaiun", "Africa/El_Aaiun"),
+                    ("Africa/Freetown", "Africa/Freetown"),
+                    ("Africa/Gaborone", "Africa/Gaborone"),
+                    ("Africa/Harare", "Africa/Harare"),
+                    ("Africa/Johannesburg", "Africa/Johannesburg"),
+                    ("Africa/Juba", "Africa/Juba"),
+                    ("Africa/Kampala", "Africa/Kampala"),
+                    ("Africa/Khartoum", "Africa/Khartoum"),
+                    ("Africa/Kigali", "Africa/Kigali"),
+                    ("Africa/Kinshasa", "Africa/Kinshasa"),
+                    ("Africa/Lagos", "Africa/Lagos"),
+                    ("Africa/Libreville", "Africa/Libreville"),
+                    ("Africa/Lome", "Africa/Lome"),
+                    ("Africa/Luanda", "Africa/Luanda"),
+                    ("Africa/Lubumbashi", "Africa/Lubumbashi"),
+                    ("Africa/Lusaka", "Africa/Lusaka"),
+                    ("Africa/Malabo", "Africa/Malabo"),
+                    ("Africa/Maputo", "Africa/Maputo"),
+                    ("Africa/Maseru", "Africa/Maseru"),
+                    ("Africa/Mbabane", "Africa/Mbabane"),
+                    ("Africa/Mogadishu", "Africa/Mogadishu"),
+                    ("Africa/Monrovia", "Africa/Monrovia"),
+                    ("Africa/Nairobi", "Africa/Nairobi"),
+                    ("Africa/Ndjamena", "Africa/Ndjamena"),
+                    ("Africa/Niamey", "Africa/Niamey"),
+                    ("Africa/Nouakchott", "Africa/Nouakchott"),
+                    ("Africa/Ouagadougou", "Africa/Ouagadougou"),
+                    ("Africa/Porto-Novo", "Africa/Porto-Novo"),
+                    ("Africa/Sao_Tome", "Africa/Sao_Tome"),
+                    ("Africa/Tripoli", "Africa/Tripoli"),
+                    ("Africa/Tunis", "Africa/Tunis"),
+                    ("Africa/Windhoek", "Africa/Windhoek"),
+                    ("America/Adak", "America/Adak"),
+                    ("America/Anchorage", "America/Anchorage"),
+                    ("America/Anguilla", "America/Anguilla"),
+                    ("America/Antigua", "America/Antigua"),
+                    ("America/Araguaina", "America/Araguaina"),
+                    (
+                        "America/Argentina/Buenos_Aires",
+                        "America/Argentina/Buenos_Aires",
+                    ),
+                    ("America/Argentina/Catamarca", "America/Argentina/Catamarca"),
+                    ("America/Argentina/Cordoba", "America/Argentina/Cordoba"),
+                    ("America/Argentina/Jujuy", "America/Argentina/Jujuy"),
+                    ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"),
+                    ("America/Argentina/Mendoza", "America/Argentina/Mendoza"),
+                    (
+                        "America/Argentina/Rio_Gallegos",
+                        "America/Argentina/Rio_Gallegos",
+                    ),
+                    ("America/Argentina/Salta", "America/Argentina/Salta"),
+                    ("America/Argentina/San_Juan", "America/Argentina/San_Juan"),
+                    ("America/Argentina/San_Luis", "America/Argentina/San_Luis"),
+                    ("America/Argentina/Tucuman", "America/Argentina/Tucuman"),
+                    ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"),
+                    ("America/Aruba", "America/Aruba"),
+                    ("America/Asuncion", "America/Asuncion"),
+                    ("America/Atikokan", "America/Atikokan"),
+                    ("America/Bahia", "America/Bahia"),
+                    ("America/Bahia_Banderas", "America/Bahia_Banderas"),
+                    ("America/Barbados", "America/Barbados"),
+                    ("America/Belem", "America/Belem"),
+                    ("America/Belize", "America/Belize"),
+                    ("America/Blanc-Sablon", "America/Blanc-Sablon"),
+                    ("America/Boa_Vista", "America/Boa_Vista"),
+                    ("America/Bogota", "America/Bogota"),
+                    ("America/Boise", "America/Boise"),
+                    ("America/Cambridge_Bay", "America/Cambridge_Bay"),
+                    ("America/Campo_Grande", "America/Campo_Grande"),
+                    ("America/Cancun", "America/Cancun"),
+                    ("America/Caracas", "America/Caracas"),
+                    ("America/Cayenne", "America/Cayenne"),
+                    ("America/Cayman", "America/Cayman"),
+                    ("America/Chicago", "America/Chicago"),
+                    ("America/Chihuahua", "America/Chihuahua"),
+                    ("America/Ciudad_Juarez", "America/Ciudad_Juarez"),
+                    ("America/Costa_Rica", "America/Costa_Rica"),
+                    ("America/Coyhaique", "America/Coyhaique"),
+                    ("America/Creston", "America/Creston"),
+                    ("America/Cuiaba", "America/Cuiaba"),
+                    ("America/Curacao", "America/Curacao"),
+                    ("America/Danmarkshavn", "America/Danmarkshavn"),
+                    ("America/Dawson", "America/Dawson"),
+                    ("America/Dawson_Creek", "America/Dawson_Creek"),
+                    ("America/Denver", "America/Denver"),
+                    ("America/Detroit", "America/Detroit"),
+                    ("America/Dominica", "America/Dominica"),
+                    ("America/Edmonton", "America/Edmonton"),
+                    ("America/Eirunepe", "America/Eirunepe"),
+                    ("America/El_Salvador", "America/El_Salvador"),
+                    ("America/Fort_Nelson", "America/Fort_Nelson"),
+                    ("America/Fortaleza", "America/Fortaleza"),
+                    ("America/Glace_Bay", "America/Glace_Bay"),
+                    ("America/Goose_Bay", "America/Goose_Bay"),
+                    ("America/Grand_Turk", "America/Grand_Turk"),
+                    ("America/Grenada", "America/Grenada"),
+                    ("America/Guadeloupe", "America/Guadeloupe"),
+                    ("America/Guatemala", "America/Guatemala"),
+                    ("America/Guayaquil", "America/Guayaquil"),
+                    ("America/Guyana", "America/Guyana"),
+                    ("America/Halifax", "America/Halifax"),
+                    ("America/Havana", "America/Havana"),
+                    ("America/Hermosillo", "America/Hermosillo"),
+                    ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"),
+                    ("America/Indiana/Knox", "America/Indiana/Knox"),
+                    ("America/Indiana/Marengo", "America/Indiana/Marengo"),
+                    ("America/Indiana/Petersburg", "America/Indiana/Petersburg"),
+                    ("America/Indiana/Tell_City", "America/Indiana/Tell_City"),
+                    ("America/Indiana/Vevay", "America/Indiana/Vevay"),
+                    ("America/Indiana/Vincennes", "America/Indiana/Vincennes"),
+                    ("America/Indiana/Winamac", "America/Indiana/Winamac"),
+                    ("America/Inuvik", "America/Inuvik"),
+                    ("America/Iqaluit", "America/Iqaluit"),
+                    ("America/Jamaica", "America/Jamaica"),
+                    ("America/Juneau", "America/Juneau"),
+                    ("America/Kentucky/Louisville", "America/Kentucky/Louisville"),
+                    ("America/Kentucky/Monticello", "America/Kentucky/Monticello"),
+                    ("America/La_Paz", "America/La_Paz"),
+                    ("America/Lima", "America/Lima"),
+                    ("America/Los_Angeles", "America/Los_Angeles"),
+                    ("America/Maceio", "America/Maceio"),
+                    ("America/Managua", "America/Managua"),
+                    ("America/Manaus", "America/Manaus"),
+                    ("America/Martinique", "America/Martinique"),
+                    ("America/Matamoros", "America/Matamoros"),
+                    ("America/Mazatlan", "America/Mazatlan"),
+                    ("America/Menominee", "America/Menominee"),
+                    ("America/Merida", "America/Merida"),
+                    ("America/Metlakatla", "America/Metlakatla"),
+                    ("America/Mexico_City", "America/Mexico_City"),
+                    ("America/Miquelon", "America/Miquelon"),
+                    ("America/Moncton", "America/Moncton"),
+                    ("America/Monterrey", "America/Monterrey"),
+                    ("America/Montevideo", "America/Montevideo"),
+                    ("America/Montserrat", "America/Montserrat"),
+                    ("America/Nassau", "America/Nassau"),
+                    ("America/New_York", "America/New_York"),
+                    ("America/Nome", "America/Nome"),
+                    ("America/Noronha", "America/Noronha"),
+                    ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"),
+                    ("America/North_Dakota/Center", "America/North_Dakota/Center"),
+                    (
+                        "America/North_Dakota/New_Salem",
+                        "America/North_Dakota/New_Salem",
+                    ),
+                    ("America/Nuuk", "America/Nuuk"),
+                    ("America/Ojinaga", "America/Ojinaga"),
+                    ("America/Panama", "America/Panama"),
+                    ("America/Paramaribo", "America/Paramaribo"),
+                    ("America/Phoenix", "America/Phoenix"),
+                    ("America/Port-au-Prince", "America/Port-au-Prince"),
+                    ("America/Port_of_Spain", "America/Port_of_Spain"),
+                    ("America/Porto_Velho", "America/Porto_Velho"),
+                    ("America/Puerto_Rico", "America/Puerto_Rico"),
+                    ("America/Punta_Arenas", "America/Punta_Arenas"),
+                    ("America/Rankin_Inlet", "America/Rankin_Inlet"),
+                    ("America/Recife", "America/Recife"),
+                    ("America/Regina", "America/Regina"),
+                    ("America/Resolute", "America/Resolute"),
+                    ("America/Rio_Branco", "America/Rio_Branco"),
+                    ("America/Santarem", "America/Santarem"),
+                    ("America/Santiago", "America/Santiago"),
+                    ("America/Santo_Domingo", "America/Santo_Domingo"),
+                    ("America/Sao_Paulo", "America/Sao_Paulo"),
+                    ("America/Scoresbysund", "America/Scoresbysund"),
+                    ("America/Sitka", "America/Sitka"),
+                    ("America/St_Johns", "America/St_Johns"),
+                    ("America/St_Kitts", "America/St_Kitts"),
+                    ("America/St_Lucia", "America/St_Lucia"),
+                    ("America/St_Thomas", "America/St_Thomas"),
+                    ("America/St_Vincent", "America/St_Vincent"),
+                    ("America/Swift_Current", "America/Swift_Current"),
+                    ("America/Tegucigalpa", "America/Tegucigalpa"),
+                    ("America/Thule", "America/Thule"),
+                    ("America/Tijuana", "America/Tijuana"),
+                    ("America/Toronto", "America/Toronto"),
+                    ("America/Tortola", "America/Tortola"),
+                    ("America/Vancouver", "America/Vancouver"),
+                    ("America/Whitehorse", "America/Whitehorse"),
+                    ("America/Winnipeg", "America/Winnipeg"),
+                    ("America/Yakutat", "America/Yakutat"),
+                    ("Antarctica/Casey", "Antarctica/Casey"),
+                    ("Antarctica/Davis", "Antarctica/Davis"),
+                    ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"),
+                    ("Antarctica/Macquarie", "Antarctica/Macquarie"),
+                    ("Antarctica/Mawson", "Antarctica/Mawson"),
+                    ("Antarctica/McMurdo", "Antarctica/McMurdo"),
+                    ("Antarctica/Palmer", "Antarctica/Palmer"),
+                    ("Antarctica/Rothera", "Antarctica/Rothera"),
+                    ("Antarctica/Syowa", "Antarctica/Syowa"),
+                    ("Antarctica/Troll", "Antarctica/Troll"),
+                    ("Antarctica/Vostok", "Antarctica/Vostok"),
+                    ("Asia/Aden", "Asia/Aden"),
+                    ("Asia/Almaty", "Asia/Almaty"),
+                    ("Asia/Amman", "Asia/Amman"),
+                    ("Asia/Anadyr", "Asia/Anadyr"),
+                    ("Asia/Aqtau", "Asia/Aqtau"),
+                    ("Asia/Aqtobe", "Asia/Aqtobe"),
+                    ("Asia/Ashgabat", "Asia/Ashgabat"),
+                    ("Asia/Atyrau", "Asia/Atyrau"),
+                    ("Asia/Baghdad", "Asia/Baghdad"),
+                    ("Asia/Bahrain", "Asia/Bahrain"),
+                    ("Asia/Baku", "Asia/Baku"),
+                    ("Asia/Bangkok", "Asia/Bangkok"),
+                    ("Asia/Barnaul", "Asia/Barnaul"),
+                    ("Asia/Beirut", "Asia/Beirut"),
+                    ("Asia/Bishkek", "Asia/Bishkek"),
+                    ("Asia/Brunei", "Asia/Brunei"),
+                    ("Asia/Chita", "Asia/Chita"),
+                    ("Asia/Colombo", "Asia/Colombo"),
+                    ("Asia/Damascus", "Asia/Damascus"),
+                    ("Asia/Dhaka", "Asia/Dhaka"),
+                    ("Asia/Dili", "Asia/Dili"),
+                    ("Asia/Dubai", "Asia/Dubai"),
+                    ("Asia/Dushanbe", "Asia/Dushanbe"),
+                    ("Asia/Famagusta", "Asia/Famagusta"),
+                    ("Asia/Gaza", "Asia/Gaza"),
+                    ("Asia/Hebron", "Asia/Hebron"),
+                    ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"),
+                    ("Asia/Hong_Kong", "Asia/Hong_Kong"),
+                    ("Asia/Hovd", "Asia/Hovd"),
+                    ("Asia/Irkutsk", "Asia/Irkutsk"),
+                    ("Asia/Jakarta", "Asia/Jakarta"),
+                    ("Asia/Jayapura", "Asia/Jayapura"),
+                    ("Asia/Jerusalem", "Asia/Jerusalem"),
+                    ("Asia/Kabul", "Asia/Kabul"),
+                    ("Asia/Kamchatka", "Asia/Kamchatka"),
+                    ("Asia/Karachi", "Asia/Karachi"),
+                    ("Asia/Kathmandu", "Asia/Kathmandu"),
+                    ("Asia/Khandyga", "Asia/Khandyga"),
+                    ("Asia/Kolkata", "Asia/Kolkata"),
+                    ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"),
+                    ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"),
+                    ("Asia/Kuching", "Asia/Kuching"),
+                    ("Asia/Kuwait", "Asia/Kuwait"),
+                    ("Asia/Macau", "Asia/Macau"),
+                    ("Asia/Magadan", "Asia/Magadan"),
+                    ("Asia/Makassar", "Asia/Makassar"),
+                    ("Asia/Manila", "Asia/Manila"),
+                    ("Asia/Muscat", "Asia/Muscat"),
+                    ("Asia/Nicosia", "Asia/Nicosia"),
+                    ("Asia/Novokuznetsk", "Asia/Novokuznetsk"),
+                    ("Asia/Novosibirsk", "Asia/Novosibirsk"),
+                    ("Asia/Omsk", "Asia/Omsk"),
+                    ("Asia/Oral", "Asia/Oral"),
+                    ("Asia/Phnom_Penh", "Asia/Phnom_Penh"),
+                    ("Asia/Pontianak", "Asia/Pontianak"),
+                    ("Asia/Pyongyang", "Asia/Pyongyang"),
+                    ("Asia/Qatar", "Asia/Qatar"),
+                    ("Asia/Qostanay", "Asia/Qostanay"),
+                    ("Asia/Qyzylorda", "Asia/Qyzylorda"),
+                    ("Asia/Riyadh", "Asia/Riyadh"),
+                    ("Asia/Sakhalin", "Asia/Sakhalin"),
+                    ("Asia/Samarkand", "Asia/Samarkand"),
+                    ("Asia/Seoul", "Asia/Seoul"),
+                    ("Asia/Shanghai", "Asia/Shanghai"),
+                    ("Asia/Singapore", "Asia/Singapore"),
+                    ("Asia/Srednekolymsk", "Asia/Srednekolymsk"),
+                    ("Asia/Taipei", "Asia/Taipei"),
+                    ("Asia/Tashkent", "Asia/Tashkent"),
+                    ("Asia/Tbilisi", "Asia/Tbilisi"),
+                    ("Asia/Tehran", "Asia/Tehran"),
+                    ("Asia/Thimphu", "Asia/Thimphu"),
+                    ("Asia/Tokyo", "Asia/Tokyo"),
+                    ("Asia/Tomsk", "Asia/Tomsk"),
+                    ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"),
+                    ("Asia/Urumqi", "Asia/Urumqi"),
+                    ("Asia/Ust-Nera", "Asia/Ust-Nera"),
+                    ("Asia/Vientiane", "Asia/Vientiane"),
+                    ("Asia/Vladivostok", "Asia/Vladivostok"),
+                    ("Asia/Yakutsk", "Asia/Yakutsk"),
+                    ("Asia/Yangon", "Asia/Yangon"),
+                    ("Asia/Yekaterinburg", "Asia/Yekaterinburg"),
+                    ("Asia/Yerevan", "Asia/Yerevan"),
+                    ("Atlantic/Azores", "Atlantic/Azores"),
+                    ("Atlantic/Bermuda", "Atlantic/Bermuda"),
+                    ("Atlantic/Canary", "Atlantic/Canary"),
+                    ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"),
+                    ("Atlantic/Faroe", "Atlantic/Faroe"),
+                    ("Atlantic/Madeira", "Atlantic/Madeira"),
+                    ("Atlantic/Reykjavik", "Atlantic/Reykjavik"),
+                    ("Atlantic/South_Georgia", "Atlantic/South_Georgia"),
+                    ("Atlantic/St_Helena", "Atlantic/St_Helena"),
+                    ("Atlantic/Stanley", "Atlantic/Stanley"),
+                    ("Australia/Adelaide", "Australia/Adelaide"),
+                    ("Australia/Brisbane", "Australia/Brisbane"),
+                    ("Australia/Broken_Hill", "Australia/Broken_Hill"),
+                    ("Australia/Darwin", "Australia/Darwin"),
+                    ("Australia/Eucla", "Australia/Eucla"),
+                    ("Australia/Hobart", "Australia/Hobart"),
+                    ("Australia/Lindeman", "Australia/Lindeman"),
+                    ("Australia/Lord_Howe", "Australia/Lord_Howe"),
+                    ("Australia/Melbourne", "Australia/Melbourne"),
+                    ("Australia/Perth", "Australia/Perth"),
+                    ("Australia/Sydney", "Australia/Sydney"),
+                    ("Europe/Amsterdam", "Europe/Amsterdam"),
+                    ("Europe/Andorra", "Europe/Andorra"),
+                    ("Europe/Astrakhan", "Europe/Astrakhan"),
+                    ("Europe/Athens", "Europe/Athens"),
+                    ("Europe/Belgrade", "Europe/Belgrade"),
+                    ("Europe/Berlin", "Europe/Berlin"),
+                    ("Europe/Brussels", "Europe/Brussels"),
+                    ("Europe/Bucharest", "Europe/Bucharest"),
+                    ("Europe/Budapest", "Europe/Budapest"),
+                    ("Europe/Chisinau", "Europe/Chisinau"),
+                    ("Europe/Copenhagen", "Europe/Copenhagen"),
+                    ("Europe/Dublin", "Europe/Dublin"),
+                    ("Europe/Gibraltar", "Europe/Gibraltar"),
+                    ("Europe/Helsinki", "Europe/Helsinki"),
+                    ("Europe/Istanbul", "Europe/Istanbul"),
+                    ("Europe/Kaliningrad", "Europe/Kaliningrad"),
+                    ("Europe/Kirov", "Europe/Kirov"),
+                    ("Europe/Kyiv", "Europe/Kyiv"),
+                    ("Europe/Lisbon", "Europe/Lisbon"),
+                    ("Europe/London", "Europe/London"),
+                    ("Europe/Luxembourg", "Europe/Luxembourg"),
+                    ("Europe/Madrid", "Europe/Madrid"),
+                    ("Europe/Malta", "Europe/Malta"),
+                    ("Europe/Minsk", "Europe/Minsk"),
+                    ("Europe/Monaco", "Europe/Monaco"),
+                    ("Europe/Moscow", "Europe/Moscow"),
+                    ("Europe/Oslo", "Europe/Oslo"),
+                    ("Europe/Paris", "Europe/Paris"),
+                    ("Europe/Prague", "Europe/Prague"),
+                    ("Europe/Riga", "Europe/Riga"),
+                    ("Europe/Rome", "Europe/Rome"),
+                    ("Europe/Samara", "Europe/Samara"),
+                    ("Europe/Saratov", "Europe/Saratov"),
+                    ("Europe/Simferopol", "Europe/Simferopol"),
+                    ("Europe/Sofia", "Europe/Sofia"),
+                    ("Europe/Stockholm", "Europe/Stockholm"),
+                    ("Europe/Tallinn", "Europe/Tallinn"),
+                    ("Europe/Tirane", "Europe/Tirane"),
+                    ("Europe/Ulyanovsk", "Europe/Ulyanovsk"),
+                    ("Europe/Vaduz", "Europe/Vaduz"),
+                    ("Europe/Vienna", "Europe/Vienna"),
+                    ("Europe/Vilnius", "Europe/Vilnius"),
+                    ("Europe/Volgograd", "Europe/Volgograd"),
+                    ("Europe/Warsaw", "Europe/Warsaw"),
+                    ("Europe/Zurich", "Europe/Zurich"),
+                    ("Indian/Antananarivo", "Indian/Antananarivo"),
+                    ("Indian/Chagos", "Indian/Chagos"),
+                    ("Indian/Christmas", "Indian/Christmas"),
+                    ("Indian/Cocos", "Indian/Cocos"),
+                    ("Indian/Comoro", "Indian/Comoro"),
+                    ("Indian/Kerguelen", "Indian/Kerguelen"),
+                    ("Indian/Mahe", "Indian/Mahe"),
+                    ("Indian/Maldives", "Indian/Maldives"),
+                    ("Indian/Mauritius", "Indian/Mauritius"),
+                    ("Indian/Mayotte", "Indian/Mayotte"),
+                    ("Indian/Reunion", "Indian/Reunion"),
+                    ("Pacific/Apia", "Pacific/Apia"),
+                    ("Pacific/Auckland", "Pacific/Auckland"),
+                    ("Pacific/Bougainville", "Pacific/Bougainville"),
+                    ("Pacific/Chatham", "Pacific/Chatham"),
+                    ("Pacific/Chuuk", "Pacific/Chuuk"),
+                    ("Pacific/Easter", "Pacific/Easter"),
+                    ("Pacific/Efate", "Pacific/Efate"),
+                    ("Pacific/Fakaofo", "Pacific/Fakaofo"),
+                    ("Pacific/Fiji", "Pacific/Fiji"),
+                    ("Pacific/Funafuti", "Pacific/Funafuti"),
+                    ("Pacific/Galapagos", "Pacific/Galapagos"),
+                    ("Pacific/Gambier", "Pacific/Gambier"),
+                    ("Pacific/Guadalcanal", "Pacific/Guadalcanal"),
+                    ("Pacific/Guam", "Pacific/Guam"),
+                    ("Pacific/Honolulu", "Pacific/Honolulu"),
+                    ("Pacific/Kanton", "Pacific/Kanton"),
+                    ("Pacific/Kiritimati", "Pacific/Kiritimati"),
+                    ("Pacific/Kosrae", "Pacific/Kosrae"),
+                    ("Pacific/Kwajalein", "Pacific/Kwajalein"),
+                    ("Pacific/Majuro", "Pacific/Majuro"),
+                    ("Pacific/Marquesas", "Pacific/Marquesas"),
+                    ("Pacific/Midway", "Pacific/Midway"),
+                    ("Pacific/Nauru", "Pacific/Nauru"),
+                    ("Pacific/Niue", "Pacific/Niue"),
+                    ("Pacific/Norfolk", "Pacific/Norfolk"),
+                    ("Pacific/Noumea", "Pacific/Noumea"),
+                    ("Pacific/Pago_Pago", "Pacific/Pago_Pago"),
+                    ("Pacific/Palau", "Pacific/Palau"),
+                    ("Pacific/Pitcairn", "Pacific/Pitcairn"),
+                    ("Pacific/Pohnpei", "Pacific/Pohnpei"),
+                    ("Pacific/Port_Moresby", "Pacific/Port_Moresby"),
+                    ("Pacific/Rarotonga", "Pacific/Rarotonga"),
+                    ("Pacific/Saipan", "Pacific/Saipan"),
+                    ("Pacific/Tahiti", "Pacific/Tahiti"),
+                    ("Pacific/Tarawa", "Pacific/Tarawa"),
+                    ("Pacific/Tongatapu", "Pacific/Tongatapu"),
+                    ("Pacific/Wake", "Pacific/Wake"),
+                    ("Pacific/Wallis", "Pacific/Wallis"),
+                    ("UTC", "UTC"),
+                ],
+                default="UTC",
+                max_length=255,
+            ),
+        ),
+    ]
diff --git a/requirements.txt b/requirements.txt
index 7aadc7bf7f..b3226d24b1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -66,8 +66,8 @@ python-json-logger>=3.1.0
 python-magic==0.4.18    # Versions beyond the yanked .19 and .20 introduce form failures
 pymemcache>=4.0.0  # for django.core.cache.backends.memcached.PyMemcacheCache
 python-mimeparse>=1.6    # from TastyPie
-pytz==2022.2.1   # Pinned as changes need to be vetted for their effect on Meeting fields
-types-pytz==2022.2.1   # match pytz version
+pytz==2025.2  # Pinned as changes need to be vetted for their effect on Meeting fields
+types-pytz==2025.2.0.20250516  # match pytz versionrequests>=2.31.0
 requests>=2.31.0
 types-requests>=2.27.1
 requests-mock>=1.9.3

From 891049aa28c12b1afab4e16561bd31316e3b508e Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Thu, 21 Aug 2025 13:39:47 +0000
Subject: [PATCH 436/656] ci: update base image target version to 20250821T1326

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 756b7021b7..9019dbb626 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250819T1508
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250821T1326
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 2e0f1519da..c5c7fb37b9 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250819T1508
+20250821T1326

From 86f2cfa29b8958633bf9dd23b624cfb4246f5693 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 21 Aug 2025 08:59:40 -0500
Subject: [PATCH 437/656] chore(deps): bump types-pytz from 2022.2.1 to
 2025.2.0.20250809 (#9339)

Bumps [types-pytz](https://github.com/typeshed-internal/stub_uploader) from 2022.2.1 to 2025.2.0.20250809.
- [Commits](https://github.com/typeshed-internal/stub_uploader/commits)

---
updated-dependencies:
- dependency-name: types-pytz
  dependency-version: 2025.2.0.20250809
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b3226d24b1..60d3d8152e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -67,7 +67,7 @@ python-magic==0.4.18    # Versions beyond the yanked .19 and .20 introduce form
 pymemcache>=4.0.0  # for django.core.cache.backends.memcached.PyMemcacheCache
 python-mimeparse>=1.6    # from TastyPie
 pytz==2025.2  # Pinned as changes need to be vetted for their effect on Meeting fields
-types-pytz==2025.2.0.20250516  # match pytz versionrequests>=2.31.0
+types-pytz==2025.2.0.20250809  # match pytz versionrequests>=2.31.0
 requests>=2.31.0
 types-requests>=2.27.1
 requests-mock>=1.9.3

From f84f2e5b4ade1bff90feb845707f70f5a317253c Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Thu, 21 Aug 2025 14:13:02 +0000
Subject: [PATCH 438/656] ci: update base image target version to 20250821T1359

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 9019dbb626..d619ee99ee 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250821T1326
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250821T1359
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index c5c7fb37b9..b6fc12e128 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250821T1326
+20250821T1359

From 64884e78ad3169a0d488ae3d046c1d65acf376b4 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 21 Aug 2025 15:03:05 -0300
Subject: [PATCH 439/656] fix: actually call fromisoformat() (#9399)

---
 ietf/api/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py
index e236347975..230f8339bd 100644
--- a/ietf/api/__init__.py
+++ b/ietf/api/__init__.py
@@ -62,14 +62,14 @@ def generate_cache_key(self, *args, **kwargs):
         # Use a list plus a ``.join()`` because it's faster than concatenation.
         return "%s:%s:%s:%s" % (self._meta.api_name, self._meta.resource_name, ':'.join(args), smooshed)
 
-    def _z_aware_fromisoformat(self, value):
-        """datetime.datetie.fromisoformat replacement that works with python < 3.11"""
+    def _z_aware_fromisoformat(self, value: str) -> datetime.datetime:
+        """datetime.datetime.fromisoformat replacement that works with python < 3.11"""
         if HAVE_BROKEN_FROMISOFORMAT:
             if value.upper().endswith("Z"):
                 value = value[:-1] + "+00:00"  # Z -> UTC
             elif re.match(r"[+-][0-9][0-9]$", value[-3:]):
                 value = value + ":00"  # -04 -> -04:00
-        return value
+        return datetime.datetime.fromisoformat(value)
 
     def filter_value_to_python(
         self, value, field_name, filters, filter_expr, filter_type

From 3b6b4770d67a8c88aee642db7129c35be930da12 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 22 Aug 2025 03:58:37 -0400
Subject: [PATCH 440/656] ci: Fix build.yml for staging db recreate

---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 9c24380764..79ef750b5d 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -448,11 +448,11 @@ jobs:
       - name: Refresh Staging DB
         uses: the-actions-org/workflow-dispatch@v4
         with:
-          workflow: update-staging-db.yml
+          workflow: deploy-db.yml
           repo: ietf-tools/infra-k8s
           ref: main
           token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
-          inputs: '{ "sourceDb":"datatracker" }'
+          inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "waitClusterReady":true }'
           wait-for-completion: true
           wait-for-completion-timeout: 10m
           wait-for-completion-interval: 20s

From 82a241084954146152196d612a13074bdc1b5de9 Mon Sep 17 00:00:00 2001
From: Eric Vyncke <eric@vyncke.org>
Date: Fri, 22 Aug 2025 17:46:45 +0200
Subject: [PATCH 441/656] feat: updated text for BoF request (per IESG demand)
 (#9406)

---
 ietf/templates/doc/bofreq/bofreq_template.md | 10 +++----
 ietf/templates/doc/bofreq/new_bofreq.html    | 30 ++++++++++++++++++--
 2 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/ietf/templates/doc/bofreq/bofreq_template.md b/ietf/templates/doc/bofreq/bofreq_template.md
index 78949ba7bd..49c5e511a5 100644
--- a/ietf/templates/doc/bofreq/bofreq_template.md
+++ b/ietf/templates/doc/bofreq/bofreq_template.md
@@ -1,15 +1,15 @@
-# Name: Exact MPLS Edges (EXAMPLE) (There's an acronym for anything if you really want one ;-)
+# Name: EXAct MPLs Edges (EXAMPLE) (There's an acronym for anything if you really want one ;-)
 ## Description 
 Replace this with a few paragraphs describing the BOF request.
 
 Fill in the details below. Keep items in the order they appear here.
 
 ## Required Details
-- Status: (not) WG Forming
-- Responsible AD: name
+- Status: "not WG Forming" or "WG forming"
+- Responsible AD: name (or at least area(s) if you know)
 - BOF proponents: name <email>, name <email> (1-3 people - who are requesting and coordinating discussion for proposal) 
 - Number of people expected to attend: 100
-- Length of session (1 or 2 hours): 2 hours
+- Length of session (1 or usually 2 hours): 2 hours
 - Conflicts (whole Areas and/or WGs)
    - Chair Conflicts: TBD
    - Technology Overlap: TBD
@@ -27,7 +27,7 @@ To allow evaluation of your proposal, please include the following items:
    - Items, Internet-Drafts, speakers, timing
    - Or a URL
 
-## Links to the mailing list, draft charter if any, relevant Internet-Drafts, etc.
+## Links to the mailing list, draft charter if any (for WG-forming BoF), relevant Internet-Drafts, etc.
    - Mailing List: https://www.ietf.org/mailman/listinfo/example
    - Draft charter: {{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name='charter-ietf-EXAMPLE' %}
    - Relevant Internet-Drafts:
diff --git a/ietf/templates/doc/bofreq/new_bofreq.html b/ietf/templates/doc/bofreq/new_bofreq.html
index 9506d9c8e6..c6aa0054f9 100644
--- a/ietf/templates/doc/bofreq/new_bofreq.html
+++ b/ietf/templates/doc/bofreq/new_bofreq.html
@@ -6,15 +6,39 @@
     {% origin %}
     <h1>Start a new BOF Request</h1>
     <p>
-        The IAB will also attempt to provide BoF Shepherds as described in their document on the subject only on request from the IESG. If you feel that your BoF would benefit from an IAB BoF Shepherd, please discuss this with your Area Director.
+        BoF proponents are strongly encouraged to review the following sources before submitting requests:
+    </p>
+    <ul>
+        <li>Process: <a href="https://www.rfc-editor.org/rfc/rfc2418.html#section-2.4">RFC 2418 Section 2.4</a></li>
+        <li>Considerations for having a successful BoF: <a href="https://www.rfc-editor.org/rfc/rfc5434.html">RFC 5434</a></li>
+    </ul>
+    {# The following block needs to be commented out after the BoF deadline and re-opened before next BoF request opening #}
+    <hr>
+    <p class="lead">
+        Announcement for IETF 124: The IESG and the IAB have organized <em>Ask Me Anything</em> (AMA) virtual sessions 
+        for the community to help proponents who are interested in putting up BoF proposals for IETF 124
+        (see also the <a href="https://mailarchive.ietf.org/arch/msg/ietf-announce/mtAYV3Mu5RfJPR-przzr-Nzxhwc/">IETF-announce</a> email):
+    </p>
+    <ul class="lead">
+        <li>28th of August 13:00-14:00 UTC 
+            <a href="https://datatracker.ietf.org/meeting/interim-2025-iesg-22/sessions/iesg.ics" title="iCalendar entry" aria-label="iCalendar entry"><i class="bi bi-calendar"></i></a></li>
+        <li>28th of August 19:00-20:00 UTC 
+            <a href="https://datatracker.ietf.org/meeting/interim-2025-iesg-23/sessions/iesg.ics" title="iCalendar entry" aria-label="iCalendar entry"><i class="bi bi-calendar"></i></a></li>
+    </ul>
+    <hr>
+    {# End of the temporary block #}
+    <p>
+        The IAB will also attempt to provide BoF Shepherds as described in their document on the subject only on request from the IESG. 
+        If you feel that your BoF would benefit from an IAB BoF Shepherd, please discuss this with your Area Director.
     </p>
     <p>
-        Choose a short descriptive title for your request. Take time to choose a good initial title - it will be used to make the filename for your request's content. The title can be changed later, but the filename will not change.
+        Choose a short descriptive title for your request. Take time to choose a good initial title - it will be used to make the filename for your request's content.
+        The title can be changed later, but the filename will not change.
     </p>
     <p>
         For example, a request with a title of "A new important bit" will be saved as <code>bofreq-{{ user.person.last_name|xslugify|slice:"64" }}-a-new-important-bit-00.md</code>.
     </p>
-    <p>All the items in the template MUST be filed in.</p>
+    <p>All the items in the template <strong>MUST</strong> be filed in.</p>
     <form class="upload-content form-horizontal"
           method="post"
           enctype="multipart/form-data">

From b1cfa7082f60343210b8116668f182e3c67207bf Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 22 Aug 2025 22:08:59 -0400
Subject: [PATCH 442/656] ci: Increase wait-for-completion timeout to 30
 minutes for staging refresh db step

Increased the wait-for-completion timeout from 10 minutes to 30 minutes in the build workflow.
---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 79ef750b5d..8567446cae 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -454,7 +454,7 @@ jobs:
           token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
           inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "waitClusterReady":true }'
           wait-for-completion: true
-          wait-for-completion-timeout: 10m
+          wait-for-completion-timeout: 30m
           wait-for-completion-interval: 20s
           display-workflow-run-url: false
 

From b3f2756f6b5d6adf853eb7779412950291169c38 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 27 Aug 2025 13:06:48 -0500
Subject: [PATCH 443/656] fix: clearly show To and From groups in liaison
 statement email (#9432)

---
 ietf/group/templatetags/group_filters.py |  7 +++++++
 ietf/templates/liaisons/liaison_mail.txt | 11 +++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/ietf/group/templatetags/group_filters.py b/ietf/group/templatetags/group_filters.py
index c9481b767b..bf2ad71949 100644
--- a/ietf/group/templatetags/group_filters.py
+++ b/ietf/group/templatetags/group_filters.py
@@ -37,3 +37,10 @@ def role_person_link(role, **kwargs):
     plain_name = role.person.plain_name()
     email = role.email.address
     return {'name': name, 'plain_name': plain_name, 'email': email, 'title': title, 'class': cls}
+
+@register.filter
+def name_with_conditional_acronym(group):
+    if group.type_id in ("sdo", "isoc", "individ", "nomcom", "ietf", "irtf", ):
+        return group.name
+    else:
+        return f"{group.name} ({group.acronym})"
diff --git a/ietf/templates/liaisons/liaison_mail.txt b/ietf/templates/liaisons/liaison_mail.txt
index 6d6a07d7ef..18dfe610fd 100644
--- a/ietf/templates/liaisons/liaison_mail.txt
+++ b/ietf/templates/liaisons/liaison_mail.txt
@@ -1,13 +1,20 @@
-{% load ietf_filters %}{% autoescape off %}Title: {{ liaison.title|clean_whitespace }}
+{% load ietf_filters group_filters %}{% autoescape off %}Title: {{ liaison.title|clean_whitespace }}
 Submission Date: {{ liaison.submitted|date:"Y-m-d" }}
 URL of the IETF Web page: {{ liaison.get_absolute_url }}
+
+To: {% for g in liaison.to_groups.all %}{{g|name_with_conditional_acronym}}{% if not forloop.last %}, {% endif %}{% endfor %}
+From: {% for g in liaison.from_groups.all %}{{g|name_with_conditional_acronym}}{% if not forloop.last %}, {% endif %}{% endfor %}
+Purpose: {{ liaison.purpose.name }}
 {% if liaison.deadline %}Please reply by {{ liaison.deadline }}{% endif %}
+
+Email Addresses
+---------------
 From: {% if liaison.from_contact %}{{ liaison.from_contact }}{% endif %}
 To: {{ liaison.to_contacts }}
 Cc: {{ liaison.cc_contacts }}
 Response Contacts: {{ liaison.response_contacts }}
 Technical Contacts: {{ liaison.technical_contacts }}
-Purpose: {{ liaison.purpose.name }}
+
 {% for related in liaison.source_of_set.all %}
 Referenced liaison: {% if related.target.title %}{{ related.target.title }}{% else %}Liaison #{{ related.target.pk }}{% endif %} ({{ related.target.get_absolute_url }})
 {% endfor %}

From 6e62bb32771cb564f52201a376ad6e754155343c Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 29 Aug 2025 10:44:51 -0500
Subject: [PATCH 444/656] fix: remove iab executive director specialization
 from the liaison app (#9435)

---
 ietf/liaisons/forms.py       |  3 +--
 ietf/liaisons/tests.py       | 12 ------------
 ietf/liaisons/tests_forms.py | 12 ------------
 ietf/liaisons/utils.py       |  1 -
 ietf/liaisons/views.py       | 17 +++++++----------
 5 files changed, 8 insertions(+), 37 deletions(-)

diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py
index 7483981595..ef5b29535e 100644
--- a/ietf/liaisons/forms.py
+++ b/ietf/liaisons/forms.py
@@ -105,7 +105,6 @@ def internal_groups_for_person(person: Optional[Person]):
             "Secretariat",
             "IETF Chair",
             "IAB Chair",
-            "IAB Executive Director",
             "Liaison Manager",
             "Liaison Coordinator",
             "Authorized Individual",
@@ -115,7 +114,7 @@ def internal_groups_for_person(person: Optional[Person]):
     # Interesting roles, as Group queries
     queries = [
         Q(role__person=person, role__name="chair", acronym="ietf"),
-        Q(role__person=person, role__name__in=("chair", "execdir"), acronym="iab"),
+        Q(role__person=person, role__name="chair", acronym="iab"),
         Q(role__person=person, role__name="ad", type="area", state="active"),
         Q(
             role__person=person,
diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py
index 1d6cfe0c14..8bbaa4f053 100644
--- a/ietf/liaisons/tests.py
+++ b/ietf/liaisons/tests.py
@@ -123,7 +123,6 @@ def test_get_cc(self):
         cc = get_cc(Group.objects.get(acronym='iab'))
         self.assertTrue(EMAIL_ALIASES['IAB'] in cc)
         self.assertTrue(EMAIL_ALIASES['IABCHAIR'] in cc)
-        self.assertTrue(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR'] in cc)
         # test an Area
         area = Group.objects.filter(type='area').first()
         cc = get_cc(area)
@@ -166,7 +165,6 @@ def test_get_contacts_for_group(self):
         # test iab
         contacts = get_contacts_for_group(Group.objects.get(acronym='iab'))
         self.assertTrue(EMAIL_ALIASES['IABCHAIR'] in contacts)
-        self.assertTrue(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR'] in contacts)
         # test iesg
         contacts = get_contacts_for_group(Group.objects.get(acronym='iesg'))
         self.assertTrue(EMAIL_ALIASES['IESG'] in contacts)
@@ -534,7 +532,6 @@ def test_outgoing_access(self):
         RoleFactory(name_id='liaison_coordinator', group__acronym='iab', person__user__username='liaison-coordinator')
         mars = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group
         RoleFactory(name_id='secr',group=mars,person__user__username='mars-secr')
-        RoleFactory(name_id='execdir',group=Group.objects.get(acronym='iab'),person__user__username='iab-execdir')
 
         url = urlreverse('ietf.liaisons.views.liaison_list')
         addurl = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'outgoing'})
@@ -592,15 +589,6 @@ def test_outgoing_access(self):
         r = self.client.get(addurl)
         self.assertEqual(r.status_code, 200)
 
-        # IAB Executive Director
-        self.assertTrue(self.client.login(username="iab-execdir", password="iab-execdir+password"))
-        r = self.client.get(url)
-        self.assertEqual(r.status_code, 200)
-        q = PyQuery(r.content)
-        self.assertEqual(len(q("a.btn:contains('New outgoing liaison')")), 1)
-        r = self.client.get(addurl)
-        self.assertEqual(r.status_code, 200)
-
         # Liaison Manager has access
         self.assertTrue(self.client.login(username="ulm-liaiman", password="ulm-liaiman+password"))
         r = self.client.get(url)
diff --git a/ietf/liaisons/tests_forms.py b/ietf/liaisons/tests_forms.py
index c2afddea65..101c0c8298 100644
--- a/ietf/liaisons/tests_forms.py
+++ b/ietf/liaisons/tests_forms.py
@@ -94,11 +94,6 @@ def test_all_internal_groups(self):
     def test_internal_groups_for_person(self):
         # test relies on the data created in ietf.utils.test_data.make_immutable_test_data()
         # todo add liaison coordinator when modeled
-        RoleFactory(
-            name_id="execdir",
-            group=Group.objects.get(acronym="iab"),
-            person__user__username="iab-execdir",
-        )
         RoleFactory(
             name_id="auth",
             group__type_id="sdo",
@@ -121,7 +116,6 @@ def test_internal_groups_for_person(self):
             "secretary",
             "ietf-chair",
             "iab-chair",
-            "iab-execdir",
             "sdo-authperson",
         ):
             returned_queryset = internal_groups_for_person(
@@ -151,11 +145,6 @@ def test_internal_groups_for_person(self):
         )
 
     def test_external_groups_for_person(self):
-        RoleFactory(
-            name_id="execdir",
-            group=Group.objects.get(acronym="iab"),
-            person__user__username="iab-execdir",
-        )
         RoleFactory(name_id="liaison_coordinator", group__acronym="iab", person__user__username="liaison-coordinator")
         the_sdo = GroupFactory(type_id="sdo", acronym="the-sdo")
         liaison_manager = RoleFactory(name_id="liaiman", group=the_sdo).person
@@ -166,7 +155,6 @@ def test_external_groups_for_person(self):
             "secretary",
             "ietf-chair",
             "iab-chair",
-            "iab-execdir",
             "liaison-coordinator",
             "ad",
             "sopschairman",
diff --git a/ietf/liaisons/utils.py b/ietf/liaisons/utils.py
index ea06c5988e..469bbc5c87 100644
--- a/ietf/liaisons/utils.py
+++ b/ietf/liaisons/utils.py
@@ -8,7 +8,6 @@
 OUTGOING_LIAISON_ROLES = [
     "Area Director",
     "IAB Chair",
-    "IAB Executive Director",
     "IETF Chair",
     "Liaison Manager",
     "Liaison Coordinator",
diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py
index 1b7e8d63bb..9710149c90 100644
--- a/ietf/liaisons/views.py
+++ b/ietf/liaisons/views.py
@@ -30,11 +30,12 @@
 from ietf.utils.response import permission_denied
 
 EMAIL_ALIASES = {
-    'IETFCHAIR':'The IETF Chair <chair@ietf.org>',
-    'IESG':'The IESG <iesg@ietf.org>',
-    'IAB':'The IAB <iab@iab.org>',
-    'IABCHAIR':'The IAB Chair <iab-chair@iab.org>',
-    'IABEXECUTIVEDIRECTOR':'The IAB Executive Director <execd@iab.org>'}
+    "IETFCHAIR": "The IETF Chair <chair@ietf.org>",
+    "IESG": "The IESG <iesg@ietf.org>",
+    "IAB": "The IAB <iab@iab.org>",
+    "IABCHAIR": "The IAB Chair <iab-chair@iab.org>",
+}
+
 
 # -------------------------------------------------
 # Helper Functions
@@ -84,8 +85,6 @@ def _find_person_in_emails(liaison, person):
             return True
         elif addr in ('iab@iab.org', 'iab-chair@iab.org') and has_role(person.user, "IAB Chair"):
             return True
-        elif addr in ('execd@iab.org', ) and has_role(person.user, "IAB Executive Director"):
-            return True
 
     return False
 
@@ -110,7 +109,6 @@ def get_cc(group):
     elif group.acronym in ('iab'):
         emails.append(EMAIL_ALIASES['IAB'])
         emails.append(EMAIL_ALIASES['IABCHAIR'])
-        emails.append(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR'])
     elif group.type_id == 'area':
         emails.append(EMAIL_ALIASES['IETFCHAIR'])
         ad_roles = group.role_set.filter(name='ad')
@@ -151,7 +149,6 @@ def get_contacts_for_group(group):
         contacts.append(EMAIL_ALIASES['IETFCHAIR'])
     elif group.acronym == 'iab':
         contacts.append(EMAIL_ALIASES['IABCHAIR'])
-        contacts.append(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR'])
     elif group.acronym == 'iesg':
         contacts.append(EMAIL_ALIASES['IESG'])
 
@@ -171,7 +168,7 @@ def needs_approval(group,person):
     user = person.user
     if group.acronym in ('ietf','iesg') and has_role(user, 'IETF Chair'):
         return False
-    if group.acronym == 'iab' and (has_role(user,'IAB Chair') or has_role(user,'IAB Executive Director')):
+    if group.acronym == 'iab' and has_role(user,'IAB Chair'):
         return False
     if group.type_id == 'area' and group.role_set.filter(name='ad',person=person):
         return False

From 3ca4eec5abb6927837fbc849809b587f4bde6419 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 2 Sep 2025 14:41:52 -0300
Subject: [PATCH 445/656] feat: expose State.used in admin (#9449)

---
 ietf/doc/admin.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py
index b492aa3423..745536f9a1 100644
--- a/ietf/doc/admin.py
+++ b/ietf/doc/admin.py
@@ -22,8 +22,8 @@ class StateTypeAdmin(admin.ModelAdmin):
 admin.site.register(StateType, StateTypeAdmin)
 
 class StateAdmin(admin.ModelAdmin):
-    list_display = ["slug", "type", 'name', 'order', 'desc']
-    list_filter = ["type", ]
+    list_display = ["slug", "type", 'name', 'order', 'desc', "used"]
+    list_filter = ["type", "used"]
     search_fields = ["slug", "type__label", "type__slug", "name", "desc"]
     filter_horizontal = ["next_states"]
 admin.site.register(State, StateAdmin)

From 02dbe17fe7bfe707594531bb16dffd905c5c2a53 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 2 Sep 2025 16:48:38 -0300
Subject: [PATCH 446/656] feat: history for mailtrigger models (#9452)

* feat: history for mailtrigger models

* chore: update copyright years

* fix: use py3.9-compatible call_command syntax

It seems `option=[...]` does not work with positional arguments in py3.9's argparse.

* chore: update resources
---
 ietf/mailtrigger/admin.py                     |   7 +-
 ...storicalrecipient_historicalmailtrigger.py | 122 ++++++++++++++++++
 ietf/mailtrigger/models.py                    |   6 +-
 ietf/mailtrigger/resources.py                 |  42 +++++-
 4 files changed, 172 insertions(+), 5 deletions(-)
 create mode 100644 ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py

diff --git a/ietf/mailtrigger/admin.py b/ietf/mailtrigger/admin.py
index a60fd5b072..8c73f2ae02 100644
--- a/ietf/mailtrigger/admin.py
+++ b/ietf/mailtrigger/admin.py
@@ -1,9 +1,10 @@
-# Copyright The IETF Trust 2015-2019, All Rights Reserved
+# Copyright The IETF Trust 2015-2025, All Rights Reserved
 from django.contrib import admin
+from simple_history.admin import SimpleHistoryAdmin
 
 from ietf.mailtrigger.models import MailTrigger, Recipient
 
-class RecipientAdmin(admin.ModelAdmin):
+class RecipientAdmin(SimpleHistoryAdmin):
     list_display = [ 'slug', 'desc', 'template', 'has_code', ]
     def has_code(self, obj):
         return hasattr(obj,'gather_%s'%obj.slug) 
@@ -11,7 +12,7 @@ def has_code(self, obj):
 admin.site.register(Recipient, RecipientAdmin)
 
 
-class MailTriggerAdmin(admin.ModelAdmin):
+class MailTriggerAdmin(SimpleHistoryAdmin):
     list_display = [ 'slug', 'desc',  ]
     filter_horizontal = [ 'to', 'cc',  ]
 admin.site.register(MailTrigger, MailTriggerAdmin)
diff --git a/ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py b/ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py
new file mode 100644
index 0000000000..d23b72d737
--- /dev/null
+++ b/ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py
@@ -0,0 +1,122 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from io import StringIO
+
+from django.conf import settings
+from django.core import management
+from django.db import migrations, models
+import django.db.models.deletion
+import simple_history.models
+
+from ietf.utils.log import log
+
+
+def forward(apps, schema_editor):
+    # Fill in history for existing data using the populate_history management command
+    captured_stdout = StringIO()
+    captured_stderr = StringIO()
+    try:
+        management.call_command(
+            "populate_history",
+            "mailtrigger.MailTrigger",
+            "mailtrigger.Recipient",
+            stdout=captured_stdout,
+            stderr=captured_stderr,
+        )
+    except management.CommandError as err:
+        log(
+            "Failed to populate history for mailtrigger models.\n"
+            "\n"
+            f"stdout:\n{captured_stdout.getvalue() or '<none>'}\n"
+            "\n"
+            f"stderr:\n{captured_stderr.getvalue() or '<none>'}\n"
+        )
+        raise RuntimeError("Failed to populate history for mailtrigger models") from err
+    log(
+        "Populated history for mailtrigger models.\n"
+        "\n"
+        f"stdout:\n{captured_stdout.getvalue() or '<none>'}\n"
+        "\n"
+        f"stderr:\n{captured_stderr.getvalue() or '<none>'}\n"
+    )
+
+
+def reverse(apps, schema_editor):
+    pass  # nothing to do
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("mailtrigger", "0006_call_for_adoption_and_last_call_issued"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="HistoricalRecipient",
+            fields=[
+                ("slug", models.CharField(db_index=True, max_length=32)),
+                ("desc", models.TextField(blank=True)),
+                ("template", models.TextField(blank=True, null=True)),
+                ("history_id", models.AutoField(primary_key=True, serialize=False)),
+                ("history_date", models.DateTimeField(db_index=True)),
+                ("history_change_reason", models.CharField(max_length=100, null=True)),
+                (
+                    "history_type",
+                    models.CharField(
+                        choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")],
+                        max_length=1,
+                    ),
+                ),
+                (
+                    "history_user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="+",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "historical recipient",
+                "verbose_name_plural": "historical recipients",
+                "ordering": ("-history_date", "-history_id"),
+                "get_latest_by": ("history_date", "history_id"),
+            },
+            bases=(simple_history.models.HistoricalChanges, models.Model),
+        ),
+        migrations.CreateModel(
+            name="HistoricalMailTrigger",
+            fields=[
+                ("slug", models.CharField(db_index=True, max_length=64)),
+                ("desc", models.TextField(blank=True)),
+                ("history_id", models.AutoField(primary_key=True, serialize=False)),
+                ("history_date", models.DateTimeField(db_index=True)),
+                ("history_change_reason", models.CharField(max_length=100, null=True)),
+                (
+                    "history_type",
+                    models.CharField(
+                        choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")],
+                        max_length=1,
+                    ),
+                ),
+                (
+                    "history_user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="+",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "historical mail trigger",
+                "verbose_name_plural": "historical mail triggers",
+                "ordering": ("-history_date", "-history_id"),
+                "get_latest_by": ("history_date", "history_id"),
+            },
+            bases=(simple_history.models.HistoricalChanges, models.Model),
+        ),
+        migrations.RunPython(forward, reverse),
+    ]
diff --git a/ietf/mailtrigger/models.py b/ietf/mailtrigger/models.py
index 66b7139fa5..435729f893 100644
--- a/ietf/mailtrigger/models.py
+++ b/ietf/mailtrigger/models.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2015-2020, All Rights Reserved
+# Copyright The IETF Trust 2015-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -7,6 +7,8 @@
 
 from email.utils import parseaddr
 
+from simple_history.models import HistoricalRecords
+
 from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible
 from ietf.utils.mail import formataddr, get_email_addresses_from_text
 from ietf.group.models import Group, Role
@@ -38,6 +40,7 @@ class MailTrigger(models.Model):
     desc = models.TextField(blank=True)
     to   = models.ManyToManyField('mailtrigger.Recipient', blank=True, related_name='used_in_to')
     cc   = models.ManyToManyField('mailtrigger.Recipient', blank=True, related_name='used_in_cc')
+    history = HistoricalRecords()
 
     class Meta:
         ordering = ["slug"]
@@ -49,6 +52,7 @@ class Recipient(models.Model):
     slug = models.CharField(max_length=32, primary_key=True)
     desc = models.TextField(blank=True)
     template = models.TextField(null=True, blank=True)
+    history = HistoricalRecords()
 
     class Meta:
         ordering = ["slug"]
diff --git a/ietf/mailtrigger/resources.py b/ietf/mailtrigger/resources.py
index eb5466618a..daca055bf4 100644
--- a/ietf/mailtrigger/resources.py
+++ b/ietf/mailtrigger/resources.py
@@ -7,7 +7,7 @@
 
 from ietf import api
 
-from ietf.mailtrigger.models import Recipient, MailTrigger
+from ietf.mailtrigger.models import MailTrigger, Recipient
 
 
 class RecipientResource(ModelResource):
@@ -37,3 +37,43 @@ class Meta:
         }
 api.mailtrigger.register(MailTriggerResource())
 
+from ietf.utils.resources import UserResource
+class HistoricalMailTriggerResource(ModelResource):
+    history_user     = ToOneField(UserResource, 'history_user', null=True)
+    class Meta:
+        queryset = MailTrigger.history.model.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'historicalmailtrigger'
+        ordering = ['history_id', ]
+        filtering = { 
+            "slug": ALL,
+            "desc": ALL,
+            "history_id": ALL,
+            "history_date": ALL,
+            "history_change_reason": ALL,
+            "history_type": ALL,
+            "history_user": ALL_WITH_RELATIONS,
+        }
+api.mailtrigger.register(HistoricalMailTriggerResource())
+
+from ietf.utils.resources import UserResource
+class HistoricalRecipientResource(ModelResource):
+    history_user     = ToOneField(UserResource, 'history_user', null=True)
+    class Meta:
+        queryset = Recipient.history.model.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'historicalrecipient'
+        ordering = ['history_id', ]
+        filtering = { 
+            "slug": ALL,
+            "desc": ALL,
+            "template": ALL,
+            "history_id": ALL,
+            "history_date": ALL,
+            "history_change_reason": ALL,
+            "history_type": ALL,
+            "history_user": ALL_WITH_RELATIONS,
+        }
+api.mailtrigger.register(HistoricalRecipientResource())

From 2960164714f0c0380d3259408b028a9150c8c27e Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 3 Sep 2025 19:16:26 -0300
Subject: [PATCH 447/656] feat: Python 3.12 (#8811)

* refactor: smtpd -> aiosmtpd

* test: set mock return value for EmailOnFailureCommandTests

The test has been working, but in a broken way, for as long as it has existed. The smtpd-based test_smtpserver was masking an exception that did not interfere with the test's effectiveness.

* test: increase SMTP.line_length_limit

* chore: suppress known deprecation warnings

* refactor: utcfromtimestamp->fromtimestamp

* refactor: it's now spelled "datetime.UTC"

* feat: python 3.12

* chore: suppress deprecation warning

* fix: utcnow() -> now(datetime.UTC)

* chore: suppress deprecation warning

* chore: more deprecation warnings

* ci: update base image target version to 20250417T1507

* chore: reorg / clean up deprecation ignore list

Removed a few suppressions that were OBE based on running the tests and checking versions of the dependencies that were causing them. Reordered kwargs to make it more readable (to me anyway).

* chore: disable coverage test for now

See the comment in settings.py for details.

tl;dr coverage is unusably slow under python 3.12 as we're using it

* ci: update base image target version to 20250422T1458

* ci: update base image target version to 20250604T2012

* ci: build/use py312 images (#9168)

* ci: tag py312 base app

* ci: datatrackerbase-app:latest -> py312

* ci: update base image target version to 20250719T0833

* refactor: update to coverage 7.9.2 + cleanup (#9256)

* refactor: drop unused code_coverage_collection var

* refactor: @skip_coverage -> pragma: no cover

* chore(deps): bump coverage to current ver

* refactor: split up set_coverage_checking()

* refactor: inline IetfLiveServerTestCase

(there's only one subclass)

* feat: disable_coverage context mgr

* chore: remove unused import

* refactor: set_coverage_checking -> disable_coverage

* refactor: elim more set_coverage_checking

* refactor: start using coverage 7.9.2

* feat: working coverage 7.9 implementation

 * Extract coverage tools to ietf.utils.coverage
 * Revert to starting checker in settings_test

Does not exactly match previous coverage reports. Need to investigate.

* refactor: CustomJsonReporter->CustomDictReporter

* chore: remove "migration" coverage entry

Has not been populated in quite some time

* test: test CoverageManager class

* chore: exclude CustomDictReporter from coverage

Setting up to test this will be complex and we'll notice other test failures/coverage weirdness if this does not behave.

* chore: exclude coverage.py from coverage

Way too meta

* chore: update deps for py3.12 (#9270)

* chore(deps): argon2-cffi (supports py3.14)

* chore(deps): setuptools to latest (py3.9+)

* chore(deps): bump beautifulsoup4 (py3.7+)

* chore(deps): bump bibtexparser (py3)

* chore(deps): bump bleach (py3.13)

* chore(deps): bump bleach (py3.13)

* chore(deps): lift pin on boto3 + adjust settings

* chore(deps): bump celery (py3.13)

* chore(deps): bump django-admin-rangefilter (py3.12)

* chore(deps): bump django-analytical (py3.13)

* chore(deps): bump django-bootstrap5 (py3.13)

* chore(deps): bump django-celery-beat (py3.12)

Still holding back until their #894 is conclusively resolved. The 2.8.x release adds official py3.13 support.

* chore(deps): bump django-celery-results (py3.13)

* chore(deps): remove django-csp (not used)

* chore(deps): bump django-cors-headers (py3.13)

* chore(deps): bump django-debug-toolbar (py3.13)

* refactor: drop stale django-referrer-policy pkg

Supported via django's SecurityMiddleware since longtime

* chore(deps): bump django-simple-history (py3.13)

* chore(deps): bump django-storages (py3.12)

* chore(deps): bump django-tastypie+update patch

* chore(deps): bump django_vite+update config

* chore(deps): bump djangorestframework+remove cap

* chore(deps): remove djlint

* chore(deps): bump docutils (py3.14)

* chore(deps): bump drf-standardized-errors (py3.13)

* chore(deps): bump factory-boy (py3.13)

* chore(deps): bump github3.py (py3.11??)

* chore(deps): bump gunicorn (py3.12)

* chore(deps): bump html2text (py3.13)

* chore(deps): bump inflect

* chore(deps): bump jsonfield (py3.10-3.13)

* chore(deps): bump jsonschema (py3.13)

* chore(deps): bump logging_tree (py3.12)

* chore(deps): bump lxml (py3.13)

* chore(deps): bump markdown (py3.13)

* chore(deps): bump mock

* chore(deps): bump oic (py3.11)

* chore(deps): bump pillow (py3.13)

* chore(deps): bump psycopg2 (py3.13)

* chore(deps): bump pyang (py3.11)

* chore(deps): bump pydyf (py3.12)

* chore(deps): bump pyflakes (py3.9+)

* chore(deps): bump pyopenssl (py3.13)

* chore(deps): bump pyquery (py3.12)

* chore(deps): bump python-dateutil (py3.12)

* chore(deps): bump python-json-logger (py3.13)

* chore(deps): bump python-mimeparse (py3.13)

* chore(deps): bump pytz (py3.13)

Brings a meeting migration to adjust tz/country choices.

* chore(deps): bump requests (py3.13)

* chore(deps): bump requests-mock (py3.12)

* chore(deps): bump scout-apm (py3.12)

* chore(deps): bump selenium (py3.13)

* chore(deps): bump tblib (py3.13)

* chore(deps): bump tqdm (py3.12)

* chore(deps): bump unidecode (py3.11)

* chore(deps): adjust requirements.txt to install correctly

* chore(deps): bump urllib3, remove pin (py3.13)

Situation requiring the pin to < 2.0 appears to have resolved.

* chore(deps): bump weasyprint (py3.13)

* chore(deps): bump xml2rfc (py3.13)

* fix: lint

* ci: py312 base for celery in sandbox

* ci: update base image target version to 20250819T1645

* chore: finish dropping smtpd  (#9384)

* chore: smtpd debug server -> aiosmtpd

* chore(dev): accept long SMTP lines

* chore(dev): use correct aiosmtpd handler

* chore: update copyright years

* Revert "chore: update copyright years"

This reverts commit 2814cb85dc43c9a27f9834c629474e58d1dfb0f7.

---------

Co-authored-by: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com>
---
 .github/workflows/build-base-app.yml          |   1 +
 .github/workflows/tests-az.yml                |   2 +-
 .vscode/tasks.json                            |   5 +-
 README.md                                     |   2 +-
 dev/build/Dockerfile                          |   2 +-
 dev/build/TARGET_BASE                         |   2 +-
 dev/celery/Dockerfile                         |   0
 dev/deploy-to-container/cli.js                |   6 +-
 dev/diff/cli.js                               |   6 +-
 dev/tests/debug.sh                            |   2 +-
 dev/tests/docker-compose.debug.yml            |   2 +-
 docker/app.Dockerfile                         |   2 +-
 docker/base.Dockerfile                        |   2 +-
 docker/celery.Dockerfile                      |   2 +-
 docker/configs/settings_local.py              |   2 +
 docker/configs/settings_local_vite.py         |   6 +-
 docker/scripts/app-configure-blobstore.py     |   6 +-
 docker/scripts/app-init.sh                    |   2 +-
 ietf/api/__init__.py                          |   2 +-
 ietf/api/tests.py                             |   6 +-
 ietf/bin/aliases-from-json.py                 |   2 +-
 ietf/doc/models.py                            |   2 +-
 ietf/doc/templatetags/ballot_icon.py          |   2 +-
 ietf/doc/tests_draft.py                       |   4 +-
 ietf/doc/tests_utils.py                       |   2 +-
 ietf/doc/views_stats.py                       |   6 +-
 ietf/group/views.py                           |   2 +-
 ietf/idindex/index.py                         |   4 +-
 ietf/iesg/views.py                            |   2 +-
 ietf/ietfauth/views.py                        |   2 +-
 ietf/ipr/mail.py                              |   4 +-
 ietf/ipr/views.py                             |  14 +-
 ietf/liaisons/tests.py                        |   8 +-
 ...meeting_country_alter_meeting_time_zone.py |   1 +
 ietf/meeting/models.py                        |   6 +-
 ietf/meeting/tests_js.py                      |   2 +-
 ietf/meeting/tests_tasks.py                   |   2 +-
 ietf/meeting/tests_views.py                   |  29 ++--
 ietf/meeting/views.py                         |   6 +-
 ietf/nomcom/tests.py                          |   2 +-
 ietf/nomcom/views.py                          |   4 +-
 ietf/settings.py                              |  48 +++---
 ietf/settings_test.py                         |   5 +-
 ietf/submit/checkers.py                       |  57 ++++---
 ietf/sync/iana.py                             |   8 +-
 ietf/sync/tasks.py                            |   2 +-
 ietf/sync/tests.py                            |   6 +-
 .../utils/{test_smtpserver.py => aiosmtpd.py} |  21 ++-
 ietf/utils/coverage.py                        |  90 ++++++++++
 ietf/utils/decorators.py                      |  12 --
 ietf/utils/jstest.py                          |  41 ++++-
 ietf/utils/meetecho.py                        |   4 +-
 ietf/utils/serialize.py                       |   2 +-
 ietf/utils/test_runner.py                     | 155 ++++++------------
 ietf/utils/tests.py                           |  14 +-
 ietf/utils/tests_coverage.py                  |  56 +++++++
 ietf/utils/tests_meetecho.py                  |  26 +--
 ietf/utils/timezone.py                        |   2 +-
 k8s/settings_local.py                         |   6 +-
 ...astypie-django22-fielderror-response.patch |   8 +-
 requirements.txt                              | 135 ++++++++-------
 61 files changed, 505 insertions(+), 359 deletions(-)
 create mode 100644 dev/celery/Dockerfile
 rename ietf/utils/{test_smtpserver.py => aiosmtpd.py} (72%)
 create mode 100644 ietf/utils/coverage.py
 create mode 100644 ietf/utils/tests_coverage.py

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index ef8a17f6b4..4a4394fca0 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -51,6 +51,7 @@ jobs:
         push: true
         tags: |
           ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }}
+          ghcr.io/ietf-tools/datatracker-app-base:py312
           ${{ github.ref == 'refs/heads/main' && 'ghcr.io/ietf-tools/datatracker-app-base:latest' || '' }}
 
     - name: Update version references
diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml
index 8553563a19..d1fe0cdf62 100644
--- a/.github/workflows/tests-az.yml
+++ b/.github/workflows/tests-az.yml
@@ -62,7 +62,7 @@ jobs:
           echo "Starting Containers..."
           sudo docker network create dtnet
           sudo docker run -d --name db --network=dtnet ghcr.io/ietf-tools/datatracker-db:latest &
-          sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:latest sleep infinity &
+          sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:py312 sleep infinity &
           wait
 
           echo "Cloning datatracker repo..."
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
index 4bd0b99363..8b36b0e6ac 100644
--- a/.vscode/tasks.json
+++ b/.vscode/tasks.json
@@ -105,10 +105,11 @@
             "command": "/usr/local/bin/python",
             "args": [
                 "-m",
-                "smtpd",
+                "aiosmtpd",
                 "-n",
                 "-c",
-                "DebuggingServer",
+                "ietf.utils.aiosmtpd.DevDebuggingHandler",
+                "-l",
                 "localhost:2025"
             ],
             "presentation": {
diff --git a/README.md b/README.md
index abebb7ca02..4e1b7e1a45 100644
--- a/README.md
+++ b/README.md
@@ -142,7 +142,7 @@ Pages will gradually be updated to Vue 3 components. These components are locate
 
 Each Vue 3 app has its own sub-directory. For example, the agenda app is located under `/client/agenda`.
 
-The datatracker makes use of the Django-Vite plugin to point to either the Vite.js server or the precompiled production files. The `DJANGO_VITE_DEV_MODE` flag, found in the `ietf/settings_local.py` file determines whether the Vite.js server is used or not.
+The datatracker makes use of the Django-Vite plugin to point to either the Vite.js server or the precompiled production files. The `DJANGO_VITE["default"]["dev_mode"]` flag, found in the `ietf/settings_local.py` file determines whether the Vite.js server is used or not.
 
 In development mode, you must start the Vite.js development server, in addition to the usual Datatracker server:
 
diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index d619ee99ee..658f1e5695 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250821T1359
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250819T1645
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index b6fc12e128..9e510ad8db 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250821T1359
+20250819T1645
diff --git a/dev/celery/Dockerfile b/dev/celery/Dockerfile
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/dev/deploy-to-container/cli.js b/dev/deploy-to-container/cli.js
index 1a2d993ac4..2f0faad151 100644
--- a/dev/deploy-to-container/cli.js
+++ b/dev/deploy-to-container/cli.js
@@ -85,7 +85,7 @@ async function main () {
   
   // Pull latest Datatracker Base image
   console.info('Pulling latest Datatracker base docker image...')
-  const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:latest')
+  const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:py312')
   await new Promise((resolve, reject) => {
     dock.modem.followProgress(appImagePullStream, (err, res) => err ? reject(err) : resolve(res))
   })
@@ -214,7 +214,7 @@ async function main () {
   const celeryContainers = {}
   for (const conConf of conConfs) {
     celeryContainers[conConf.name] = await dock.createContainer({
-      Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest',
+      Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312',
       name: `dt-${conConf.name}-${branch}`,
       Hostname: `dt-${conConf.name}-${branch}`,
       Env: [
@@ -244,7 +244,7 @@ async function main () {
   // Create Datatracker container
   console.info(`Creating Datatracker docker container... [dt-app-${branch}]`)
   const appContainer = await dock.createContainer({
-    Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest',
+    Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312',
     name: `dt-app-${branch}`,
     Hostname: `dt-app-${branch}`,
     Env: [
diff --git a/dev/diff/cli.js b/dev/diff/cli.js
index 461b0c37a0..0cf353cc65 100644
--- a/dev/diff/cli.js
+++ b/dev/diff/cli.js
@@ -567,7 +567,7 @@ async function main () {
           {
             title: 'Pulling latest Datatracker base docker image...',
             task: async (subctx, subtask) => {
-              const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:latest')
+              const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:py312')
               await new Promise((resolve, reject) => {
                 dock.modem.followProgress(appImagePullStream, (err, res) => err ? reject(err) : resolve(res))
               })
@@ -648,7 +648,7 @@ async function main () {
             title: 'Creating source Datatracker docker container...',
             task: async (subctx, subtask) => {
               containers.appSource = await dock.createContainer({
-                Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest',
+                Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312',
                 name: 'dt-diff-app-source',
                 Tty: true,
                 Hostname: 'appsource',
@@ -664,7 +664,7 @@ async function main () {
             title: 'Creating target Datatracker docker container...',
             task: async (subctx, subtask) => {
               containers.appTarget = await dock.createContainer({
-                Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest',
+                Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312',
                 name: 'dt-diff-app-target',
                 Tty: true,
                 Hostname: 'apptarget',
diff --git a/dev/tests/debug.sh b/dev/tests/debug.sh
index d87c504bb9..e92e6d9b2a 100644
--- a/dev/tests/debug.sh
+++ b/dev/tests/debug.sh
@@ -9,7 +9,7 @@
 # Simply type "exit" + ENTER to exit and shutdown this test environment.
 
 echo "Fetching latest images..."
-docker pull ghcr.io/ietf-tools/datatracker-app-base:latest
+docker pull ghcr.io/ietf-tools/datatracker-app-base:py312
 docker pull ghcr.io/ietf-tools/datatracker-db:latest
 echo "Starting containers..."
 docker compose -f docker-compose.debug.yml -p dtdebug --compatibility up -d
diff --git a/dev/tests/docker-compose.debug.yml b/dev/tests/docker-compose.debug.yml
index 8117b92375..168bbd4e92 100644
--- a/dev/tests/docker-compose.debug.yml
+++ b/dev/tests/docker-compose.debug.yml
@@ -5,7 +5,7 @@ version: '3.8'
 
 services:
     app:
-        image: ghcr.io/ietf-tools/datatracker-app-base:latest
+        image: ghcr.io/ietf-tools/datatracker-app-base:py312
         command: -f /dev/null
         working_dir: /__w/datatracker/datatracker
         entrypoint: tail
diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile
index fee3833733..e3df9bd4b4 100644
--- a/docker/app.Dockerfile
+++ b/docker/app.Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:latest
+FROM ghcr.io/ietf-tools/datatracker-app-base:py312
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile
index 57aac8ee56..c1fe5b093e 100644
--- a/docker/base.Dockerfile
+++ b/docker/base.Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9-bookworm
+FROM python:3.12-bookworm
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/docker/celery.Dockerfile b/docker/celery.Dockerfile
index e7c7b9cc3f..279d5c7550 100644
--- a/docker/celery.Dockerfile
+++ b/docker/celery.Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:latest
+FROM ghcr.io/ietf-tools/datatracker-app-base:py312
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py
index ca51871463..3ee7a4295d 100644
--- a/docker/configs/settings_local.py
+++ b/docker/configs/settings_local.py
@@ -89,6 +89,8 @@
             secret_key="minio_pass",
             security_token=None,
             client_config=botocore.config.Config(
+                request_checksum_calculation="when_required",
+                response_checksum_validation="when_required",
                 signature_version="s3v4",
                 connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT,
                 read_timeout=BLOBSTORAGE_READ_TIMEOUT,
diff --git a/docker/configs/settings_local_vite.py b/docker/configs/settings_local_vite.py
index 7fb12a003d..9116905b12 100644
--- a/docker/configs/settings_local_vite.py
+++ b/docker/configs/settings_local_vite.py
@@ -2,5 +2,9 @@
 # -*- coding: utf-8 -*-
 
 from ietf.settings_local import *                  # pyflakes:ignore
+from ietf.settings_local import DJANGO_VITE
 
-DJANGO_VITE_DEV_MODE = True
+DJANGO_VITE["default"] |= {
+    "dev_mode": True,
+    "dev_server_port": 3000,
+}
diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py
index df4685b246..3140e39306 100755
--- a/docker/scripts/app-configure-blobstore.py
+++ b/docker/scripts/app-configure-blobstore.py
@@ -17,7 +17,11 @@ def init_blobstore():
         aws_access_key_id=os.environ.get("BLOB_STORE_ACCESS_KEY", "minio_root"),
         aws_secret_access_key=os.environ.get("BLOB_STORE_SECRET_KEY", "minio_pass"),
         aws_session_token=None,
-        config=botocore.config.Config(signature_version="s3v4"),
+        config=botocore.config.Config(
+            request_checksum_calculation="when_required",
+            response_checksum_validation="when_required",
+            signature_version="s3v4",
+        ),
     )
     for bucketname in ARTIFACT_STORAGE_NAMES:
         try:
diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh
index 17e0c6c764..1d895cdf53 100755
--- a/docker/scripts/app-init.sh
+++ b/docker/scripts/app-init.sh
@@ -108,7 +108,7 @@ echo "Running initial checks..."
 
 if [ -z "$EDITOR_VSCODE" ]; then
     CODE=0
-    python -m smtpd -n -c DebuggingServer localhost:2025 &
+    python -m aiosmtpd -n -c ietf.utils.aiosmtpd.DevDebuggingHandler -l localhost:2025 &
     if [ -z "$*" ]; then
         echo "-----------------------------------------------------------------"
         echo "Ready!"
diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py
index 230f8339bd..d4562f97dd 100644
--- a/ietf/api/__init__.py
+++ b/ietf/api/__init__.py
@@ -181,7 +181,7 @@ class Serializer(tastypie.serializers.Serializer):
     OPTION_ESCAPE_NULLS = "datatracker-escape-nulls"
 
     def format_datetime(self, data):
-        return data.astimezone(datetime.timezone.utc).replace(tzinfo=None).isoformat(timespec="seconds") + "Z"
+        return data.astimezone(datetime.UTC).replace(tzinfo=None).isoformat(timespec="seconds") + "Z"
 
     def to_simple(self, data, options):
         options = options or {}
diff --git a/ietf/api/tests.py b/ietf/api/tests.py
index 865f877bfb..2a44791a5c 100644
--- a/ietf/api/tests.py
+++ b/ietf/api/tests.py
@@ -462,12 +462,12 @@ def test_api_add_session_attendees(self):
         self.assertTrue(session.attended_set.filter(person=recman).exists())
         self.assertEqual(
             session.attended_set.get(person=recman).time,
-            datetime.datetime(2023, 9, 3, 12, 34, 56, tzinfo=datetime.timezone.utc),
+            datetime.datetime(2023, 9, 3, 12, 34, 56, tzinfo=datetime.UTC),
         )
         self.assertTrue(session.attended_set.filter(person=otherperson).exists())
         self.assertEqual(
             session.attended_set.get(person=otherperson).time,
-            datetime.datetime(2023, 9, 3, 3, 0, 19, tzinfo=datetime.timezone.utc),
+            datetime.datetime(2023, 9, 3, 3, 0, 19, tzinfo=datetime.UTC),
         )
 
     def test_api_upload_polls_and_chatlog(self):
@@ -871,7 +871,7 @@ def test_api_new_meeting_registration_v2_nomcom(self):
         self.assertEqual(volunteer.origin, 'registration')
 
     def test_api_version(self):
-        DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=datetime.timezone.utc), host='testapi.example.com',tz='UTC')
+        DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=datetime.UTC), host='testapi.example.com',tz='UTC')
         url = urlreverse('ietf.api.views.version')
         r = self.client.get(url)
         data = r.json()
diff --git a/ietf/bin/aliases-from-json.py b/ietf/bin/aliases-from-json.py
index a0c383a1ac..0da5d1f8b9 100644
--- a/ietf/bin/aliases-from-json.py
+++ b/ietf/bin/aliases-from-json.py
@@ -38,7 +38,7 @@ def generate_files(records, adest, vdest, postconfirm, vdomain):
         vpath = tmppath / "virtual"
 
         with apath.open("w") as afile, vpath.open("w") as vfile:
-            date = datetime.datetime.now(datetime.timezone.utc)
+            date = datetime.datetime.now(datetime.UTC)
             signature = f"# Generated by {Path(__file__).absolute()} at {date}\n"
             afile.write(signature)
             vfile.write(signature)
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index b6f36cb8a7..25ee734cbe 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -1157,7 +1157,7 @@ def fake_history_obj(self, rev):
             elif rev_events.exists():
                 time = rev_events.first().time
             else:
-                time = datetime.datetime.fromtimestamp(0, datetime.timezone.utc)
+                time = datetime.datetime.fromtimestamp(0, datetime.UTC)
             dh = DocHistory(name=self.name, rev=rev, doc=self, time=time, type=self.type, title=self.title,
                              stream=self.stream, group=self.group)
 
diff --git a/ietf/doc/templatetags/ballot_icon.py b/ietf/doc/templatetags/ballot_icon.py
index a94c145007..07a6c7f926 100644
--- a/ietf/doc/templatetags/ballot_icon.py
+++ b/ietf/doc/templatetags/ballot_icon.py
@@ -196,7 +196,7 @@ def state_age_colored(doc):
                 .time
             )
         except IndexError:
-            state_datetime = datetime.datetime(1990, 1, 1, tzinfo=datetime.timezone.utc)
+            state_datetime = datetime.datetime(1990, 1, 1, tzinfo=datetime.UTC)
         days = (timezone.now() - state_datetime).days
         # loosely based on the Publish Path page at the iesg wiki
         if iesg_state == "lc":
diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py
index ab7eaba768..ab33acebe6 100644
--- a/ietf/doc/tests_draft.py
+++ b/ietf/doc/tests_draft.py
@@ -678,11 +678,11 @@ def test_in_draft_expire_freeze(self):
             datetime.datetime.combine(
                 ietf_monday - datetime.timedelta(days=1),
                 datetime.time(0, 0, 0),
-                tzinfo=datetime.timezone.utc,
+                tzinfo=datetime.UTC,
             )
         ))
         self.assertFalse(in_draft_expire_freeze(
-            datetime.datetime.combine(ietf_monday, datetime.time(0, 0, 0), tzinfo=datetime.timezone.utc)
+            datetime.datetime.combine(ietf_monday, datetime.time(0, 0, 0), tzinfo=datetime.UTC)
         ))
         
     def test_warn_expirable_drafts(self):
diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py
index f610fe3d76..7db59819da 100644
--- a/ietf/doc/tests_utils.py
+++ b/ietf/doc/tests_utils.py
@@ -148,7 +148,7 @@ def test_update_action_holders_resets_age(self):
         doc = self.doc_in_iesg_state('pub-req')
         doc.action_holders.set([self.ad])
         dah = doc.documentactionholder_set.get(person=self.ad)
-        dah.time_added = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc)  # arbitrary date in the past
+        dah.time_added = datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC)  # arbitrary date in the past
         dah.save()
 
         right_now = timezone.now()
diff --git a/ietf/doc/views_stats.py b/ietf/doc/views_stats.py
index 0bbf0b91c5..028573b338 100644
--- a/ietf/doc/views_stats.py
+++ b/ietf/doc/views_stats.py
@@ -18,7 +18,7 @@
 from ietf.utils.timezone import date_today
 
 
-epochday = datetime.datetime.utcfromtimestamp(0).date().toordinal()
+epochday = datetime.datetime.fromtimestamp(0, datetime.UTC).date().toordinal()
 
 
 def dt(s):
@@ -35,13 +35,13 @@ def model_to_timeline_data(model, field='time', **kwargs):
     assert field in [ f.name for f in model._meta.get_fields() ]
 
     objects = ( model.objects.filter(**kwargs)
-                                .annotate(date=TruncDate(field, tzinfo=datetime.timezone.utc))
+                                .annotate(date=TruncDate(field, tzinfo=datetime.UTC))
                                 .order_by('date')
                                 .values('date')
                                 .annotate(count=Count('id')))
     if objects.exists():
         obj_list = list(objects)
-        today = date_today(datetime.timezone.utc)
+        today = date_today(datetime.UTC)
         if not obj_list[-1]['date'] == today:
             obj_list += [ {'date': today, 'count': 0} ]
         data = [ ((e['date'].toordinal()-epochday)*1000*60*60*24, e['count']) for e in obj_list ]
diff --git a/ietf/group/views.py b/ietf/group/views.py
index 3529b31f68..bc785ff81e 100644
--- a/ietf/group/views.py
+++ b/ietf/group/views.py
@@ -941,7 +941,7 @@ def meetings(request, acronym, group_type=None):
             cutoff_date = revsub_dates_by_meeting[s.meeting.pk]
         else:
             cutoff_date = s.meeting.date + datetime.timedelta(days=s.meeting.submission_correction_day_offset)
-        s.cached_is_cutoff = date_today(datetime.timezone.utc) > cutoff_date
+        s.cached_is_cutoff = date_today(datetime.UTC) > cutoff_date
 
     future, in_progress, recent, past = group_sessions(sessions)
 
diff --git a/ietf/idindex/index.py b/ietf/idindex/index.py
index 4f021c0dc7..19eb29d4da 100644
--- a/ietf/idindex/index.py
+++ b/ietf/idindex/index.py
@@ -276,7 +276,7 @@ def active_drafts_index_by_group(extra_values=()):
     groups = [g for g in groups_dict.values() if hasattr(g, "active_drafts")]
     groups.sort(key=lambda g: g.acronym)
 
-    fallback_time = datetime.datetime(1950, 1, 1, tzinfo=datetime.timezone.utc)
+    fallback_time = datetime.datetime(1950, 1, 1, tzinfo=datetime.UTC)
     for g in groups:
         g.active_drafts.sort(key=lambda d: d.get("initial_rev_time", fallback_time))
 
@@ -302,6 +302,6 @@ def id_index_txt(with_abstracts=False):
 
     return render_to_string("idindex/id_index.txt", {
             'groups': groups,
-            'time': timezone.now().astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z"),
+            'time': timezone.now().astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S %Z"),
             'with_abstracts': with_abstracts,
             })
diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py
index 7b9f489b44..ffd4515c98 100644
--- a/ietf/iesg/views.py
+++ b/ietf/iesg/views.py
@@ -101,7 +101,7 @@ def agenda_json(request, date=None):
 
     res = {
         "telechat-date": str(data["date"]),
-        "as-of": str(datetime.datetime.utcnow()),
+        "as-of": str(datetime.datetime.now(datetime.UTC)),
         "page-counts": telechat_page_count(date=get_agenda_date(date))._asdict(),
         "sections": {},
         }
diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py
index 4219747e12..b5256b14f8 100644
--- a/ietf/ietfauth/views.py
+++ b/ietf/ietfauth/views.py
@@ -517,7 +517,7 @@ def confirm_password_reset(request, auth):
         password = data['password']
         last_login = None
         if data['last_login']:
-            last_login = datetime.datetime.fromtimestamp(data['last_login'], datetime.timezone.utc)
+            last_login = datetime.datetime.fromtimestamp(data['last_login'], datetime.UTC)
     except django.core.signing.BadSignature:
         raise Http404("Invalid or expired auth")
 
diff --git a/ietf/ipr/mail.py b/ietf/ipr/mail.py
index 167b11956c..9bef751b95 100644
--- a/ietf/ipr/mail.py
+++ b/ietf/ipr/mail.py
@@ -66,9 +66,9 @@ def utc_from_string(s):
     if date is None:
         return None
     elif is_aware(date):
-        return date.astimezone(datetime.timezone.utc)
+        return date.astimezone(datetime.UTC)
     else:
-        return date.replace(tzinfo=datetime.timezone.utc)
+        return date.replace(tzinfo=datetime.UTC)
 
 # ----------------------------------------------------------------
 # Email Functions
diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py
index 24453df2d2..08979a3972 100644
--- a/ietf/ipr/views.py
+++ b/ietf/ipr/views.py
@@ -152,13 +152,13 @@ def ipr_rfc_number(disclosureDate, thirdPartyDisclosureFlag):
 
     # RFC publication date comes from the RFC Editor announcement
     ipr_rfc_pub_datetime = {
-        1310 : datetime.datetime(1992,  3, 13,  0,  0, tzinfo=datetime.timezone.utc),
-        1802 : datetime.datetime(1994,  3, 23,  0,  0, tzinfo=datetime.timezone.utc),
-        2026 : datetime.datetime(1996, 10, 29,  0,  0, tzinfo=datetime.timezone.utc),
-        3668 : datetime.datetime(2004,  2, 18,  0,  0, tzinfo=datetime.timezone.utc),
-        3979 : datetime.datetime(2005,  3,  2,  2, 23, tzinfo=datetime.timezone.utc),
-        4879 : datetime.datetime(2007,  4, 10, 18, 21, tzinfo=datetime.timezone.utc),
-        8179 : datetime.datetime(2017,  5, 31, 23,  1, tzinfo=datetime.timezone.utc),
+        1310 : datetime.datetime(1992,  3, 13,  0,  0, tzinfo=datetime.UTC),
+        1802 : datetime.datetime(1994,  3, 23,  0,  0, tzinfo=datetime.UTC),
+        2026 : datetime.datetime(1996, 10, 29,  0,  0, tzinfo=datetime.UTC),
+        3668 : datetime.datetime(2004,  2, 18,  0,  0, tzinfo=datetime.UTC),
+        3979 : datetime.datetime(2005,  3,  2,  2, 23, tzinfo=datetime.UTC),
+        4879 : datetime.datetime(2007,  4, 10, 18, 21, tzinfo=datetime.UTC),
+        8179 : datetime.datetime(2017,  5, 31, 23,  1, tzinfo=datetime.UTC),
     }
 
     if disclosureDate < ipr_rfc_pub_datetime[1310]:
diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py
index 8bbaa4f053..a1fbf77841 100644
--- a/ietf/liaisons/tests.py
+++ b/ietf/liaisons/tests.py
@@ -723,7 +723,7 @@ def test_add_incoming_liaison(self):
         from_groups = [ str(g.pk) for g in Group.objects.filter(type="sdo") ]
         to_group = Group.objects.get(acronym="mars")
         submitter = Person.objects.get(user__username="marschairman")
-        today = date_today(datetime.timezone.utc)
+        today = date_today(datetime.UTC)
         related_liaison = liaison
         r = self.client.post(url,
                              dict(from_groups=from_groups,
@@ -808,7 +808,7 @@ def test_add_outgoing_liaison(self):
         from_group = Group.objects.get(acronym="mars")
         to_group = Group.objects.filter(type="sdo")[0]
         submitter = Person.objects.get(user__username="marschairman")
-        today = date_today(datetime.timezone.utc)
+        today = date_today(datetime.UTC)
         related_liaison = liaison
         r = self.client.post(url,
                              dict(from_groups=str(from_group.pk),
@@ -878,7 +878,7 @@ def test_add_outgoing_liaison_unapproved_post_only(self):
         from_group = Group.objects.get(acronym="mars")
         to_group = Group.objects.filter(type="sdo")[0]
         submitter = Person.objects.get(user__username="marschairman")
-        today = date_today(datetime.timezone.utc)
+        today = date_today(datetime.UTC)
         r = self.client.post(url,
                              dict(from_groups=str(from_group.pk),
                                   from_contact=submitter.email_address(),
@@ -1062,7 +1062,7 @@ def test_search(self):
         LiaisonStatementEventFactory(type_id='posted', statement__body="Has recently in its body",statement__from_groups=[GroupFactory(type_id='sdo',acronym='ulm'),])
         # Statement 2
         s2 = LiaisonStatementEventFactory(type_id='posted', statement__body="That word does not occur here", statement__title="Nor does it occur here")
-        s2.time=datetime.datetime(2010, 1, 1, tzinfo=datetime.timezone.utc)
+        s2.time=datetime.datetime(2010, 1, 1, tzinfo=datetime.UTC)
         s2.save()
 
         # test list only, no search filters
diff --git a/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py b/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py
index 8f5db26112..8c467ea156 100644
--- a/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py
+++ b/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py
@@ -4,6 +4,7 @@
 
 
 class Migration(migrations.Migration):
+
     dependencies = [
         ("meeting", "0015_alter_meeting_time_zone"),
     ]
diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index de0192769e..f3df23e916 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -149,7 +149,7 @@ def get_00_cutoff(self):
             cutoff_date = importantdate.date
         else:
             cutoff_date = self.date + datetime.timedelta(days=ImportantDateName.objects.get(slug='idcutoff').default_offset_days)
-        cutoff_time = datetime_from_date(cutoff_date, datetime.timezone.utc) + self.idsubmit_cutoff_time_utc
+        cutoff_time = datetime_from_date(cutoff_date, datetime.UTC) + self.idsubmit_cutoff_time_utc
         return cutoff_time
 
     def get_01_cutoff(self):
@@ -161,7 +161,7 @@ def get_01_cutoff(self):
             cutoff_date = importantdate.date
         else:
             cutoff_date = self.date + datetime.timedelta(days=ImportantDateName.objects.get(slug='idcutoff').default_offset_days)
-        cutoff_time = datetime_from_date(cutoff_date, datetime.timezone.utc) + self.idsubmit_cutoff_time_utc
+        cutoff_time = datetime_from_date(cutoff_date, datetime.UTC) + self.idsubmit_cutoff_time_utc
         return cutoff_time
 
     def get_reopen_time(self):
@@ -1172,7 +1172,7 @@ def can_manage_materials(self, user):
         return can_manage_materials(user,self.group)
 
     def is_material_submission_cutoff(self):
-        return date_today(datetime.timezone.utc) > self.meeting.get_submission_correction_date()
+        return date_today(datetime.UTC) > self.meeting.get_submission_correction_date()
     
     def joint_with_groups_acronyms(self):
         return [group.acronym for group in self.joint_with_groups.all()]
diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py
index a184a7c6d0..262b47652c 100644
--- a/ietf/meeting/tests_js.py
+++ b/ietf/meeting/tests_js.py
@@ -1576,7 +1576,7 @@ def test_delete_timeslot_cancel(self):
 
     def do_delete_time_interval_test(self, cancel=False):
         delete_time_local = datetime_from_date(self.meeting.date, self.meeting.tz()).replace(hour=10)
-        delete_time = delete_time_local.astimezone(datetime.timezone.utc)
+        delete_time = delete_time_local.astimezone(datetime.UTC)
         duration = datetime.timedelta(minutes=60)
 
         delete: [TimeSlot] = TimeSlotFactory.create_batch(  # type: ignore[annotation-unchecked]
diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py
index 0c442c4bf7..a5da00ecbf 100644
--- a/ietf/meeting/tests_tasks.py
+++ b/ietf/meeting/tests_tasks.py
@@ -23,7 +23,7 @@ def test_proceedings_content_refresh_task(self, mock_generate):
         meeting127 = MeetingFactory(type_id="ietf", number="127")  # 24 * 5 + 7
         
         # Times to be returned
-        now_utc = datetime.datetime.now(tz=datetime.timezone.utc)
+        now_utc = datetime.datetime.now(tz=datetime.UTC)
         hour_00_utc = now_utc.replace(hour=0)
         hour_01_utc = now_utc.replace(hour=1)
         hour_07_utc = now_utc.replace(hour=7)
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index f382772485..bd3ab772fc 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -55,9 +55,8 @@
 from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule
 from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose, generate_agenda_data
 from ietf.name.models import SessionStatusName, ImportantDateName, RoleName, ProceedingsMaterialTypeName
-from ietf.utils.decorators import skip_coverage
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
-from ietf.utils.test_runner import TestBlobstoreManager
+from ietf.utils.test_runner import TestBlobstoreManager, disable_coverage
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent
 from ietf.utils.timezone import date_today, time_now
 
@@ -321,11 +320,11 @@ def test_meeting_agenda(self):
         self.assertContains(r, session.group.parent.acronym.upper())
         self.assertContains(r, slot.location.name)
         self.assertContains(r, "{}-{}".format(
-            slot.time.astimezone(datetime.timezone.utc).strftime("%H%M"),
-            (slot.time + slot.duration).astimezone(datetime.timezone.utc).strftime("%H%M"),
+            slot.time.astimezone(datetime.UTC).strftime("%H%M"),
+            (slot.time + slot.duration).astimezone(datetime.UTC).strftime("%H%M"),
         ))
         self.assertContains(r, "shown in UTC")
-        updated = meeting.updated().astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z")
+        updated = meeting.updated().astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S %Z")
         self.assertContains(r, f"Updated {updated}")
 
         # text, invalid updated (none)
@@ -369,8 +368,8 @@ def test_meeting_agenda(self):
         self.assertContains(r, session.group.parent.acronym.upper())
         self.assertContains(r, slot.location.name)
         self.assertContains(r, registration_text)
-        start_time = slot.time.astimezone(datetime.timezone.utc)
-        end_time = slot.end_time().astimezone(datetime.timezone.utc)
+        start_time = slot.time.astimezone(datetime.UTC)
+        end_time = slot.end_time().astimezone(datetime.UTC)
         self.assertContains(r, '"{}","{}","{}"'.format(
             start_time.strftime("%Y-%m-%d"),
             start_time.strftime("%H%M"),
@@ -1037,7 +1036,7 @@ def test_important_dates_ical(self):
 
         updated = meeting.updated()
         self.assertIsNotNone(updated)
-        expected_updated = updated.astimezone(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+        expected_updated = updated.astimezone(datetime.UTC).strftime("%Y%m%dT%H%M%SZ")
         self.assertContains(r, f"DTSTAMP:{expected_updated}")
         dtstamps_count = r.content.decode("utf-8").count(f"DTSTAMP:{expected_updated}")
         self.assertEqual(dtstamps_count, meeting.importantdate_set.count())
@@ -1181,8 +1180,8 @@ def test_session_draft_tarfile(self):
                 os.unlink(filename)
 
     @skipIf(skip_pdf_tests, skip_message)
-    @skip_coverage
-    def test_session_draft_pdf(self):
+    @disable_coverage()
+    def test_session_draft_pdf(self):  # pragma: no cover
         session, filenames = self.build_session_setup()
         try:
             url = urlreverse('ietf.meeting.views.session_draft_pdf', kwargs={'num':session.meeting.number,'acronym':session.group.acronym})
@@ -2117,8 +2116,8 @@ def test_editor_time_zone(self):
         # strftime() does not seem to support hours without leading 0, so do this manually
         time_label_string = f'{ts_start.hour:d}:{ts_start.minute:02d} - {ts_end.hour:d}:{ts_end.minute:02d}'
         self.assertIn(time_label_string, time_label.text())
-        self.assertEqual(time_label.attr('data-start'), ts_start.astimezone(datetime.timezone.utc).isoformat())
-        self.assertEqual(time_label.attr('data-end'), ts_end.astimezone(datetime.timezone.utc).isoformat())
+        self.assertEqual(time_label.attr('data-start'), ts_start.astimezone(datetime.UTC).isoformat())
+        self.assertEqual(time_label.attr('data-end'), ts_end.astimezone(datetime.UTC).isoformat())
 
         ts_swap = time_label.find('.swap-timeslot-col')
         origin_label = ts_swap.attr('data-origin-label')
@@ -2129,8 +2128,8 @@ def test_editor_time_zone(self):
 
         timeslot_elt = pq(f'#timeslot{timeslot.pk}')
         self.assertEqual(len(timeslot_elt), 1)
-        self.assertEqual(timeslot_elt.attr('data-start'), ts_start.astimezone(datetime.timezone.utc).isoformat())
-        self.assertEqual(timeslot_elt.attr('data-end'), ts_end.astimezone(datetime.timezone.utc).isoformat())
+        self.assertEqual(timeslot_elt.attr('data-start'), ts_start.astimezone(datetime.UTC).isoformat())
+        self.assertEqual(timeslot_elt.attr('data-end'), ts_end.astimezone(datetime.UTC).isoformat())
 
         timeslot_label = pq(f'#timeslot{timeslot.pk} .time-label')
         self.assertEqual(len(timeslot_label), 1)
@@ -5233,7 +5232,7 @@ def test_upcoming_ical(self):
 
         updated = meeting.updated()
         self.assertIsNotNone(updated)
-        expected_updated = updated.astimezone(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+        expected_updated = updated.astimezone(datetime.UTC).strftime("%Y%m%dT%H%M%SZ")
         self.assertContains(r, f"DTSTAMP:{expected_updated}")
 
         # With default cached_updated, 1970-01-01
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 7fa3d21259..fcc9312609 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -153,7 +153,7 @@ def materials(request, num=None):
     begin_date = meeting.get_submission_start_date()
     cut_off_date = meeting.get_submission_cut_off_date()
     cor_cut_off_date = meeting.get_submission_correction_date()
-    today_utc = date_today(datetime.timezone.utc)
+    today_utc = date_today(datetime.UTC)
     old = timezone.now() - datetime.timedelta(days=1)
     if settings.SERVER_MODE != 'production' and '_testoverride' in request.GET:
         pass
@@ -1921,7 +1921,7 @@ def slides_field(item):
 
     write_row(headings)
 
-    tz = datetime.timezone.utc if utc else schedule.meeting.tz()
+    tz = datetime.UTC if utc else schedule.meeting.tz()
     for item in filtered_assignments:
         row = []
         row.append(item.timeslot.time.astimezone(tz).strftime("%Y-%m-%d"))
@@ -2814,7 +2814,7 @@ def session_attendance(request, session_id, num):
             raise Http404("Bluesheets not found")
 
     cor_cut_off_date = session.meeting.get_submission_correction_date()
-    today_utc = date_today(datetime.timezone.utc)
+    today_utc = date_today(datetime.UTC)
     was_there = False
     can_add = False
     if request.user.is_authenticated:
diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py
index cc2e0826d3..dcdb9ef836 100644
--- a/ietf/nomcom/tests.py
+++ b/ietf/nomcom/tests.py
@@ -2930,7 +2930,7 @@ def test_decorate_volunteers_with_qualifications(self):
                     elig_date.year - 3,
                     elig_date.month,
                     28 if elig_date.month == 2 and elig_date.day == 29 else elig_date.day,
-                    tzinfo=datetime.timezone.utc,
+                    tzinfo=datetime.UTC,
                 )
             )
         nomcom.volunteer_set.create(person=author_person)
diff --git a/ietf/nomcom/views.py b/ietf/nomcom/views.py
index c04e13f92b..3f90be5253 100644
--- a/ietf/nomcom/views.py
+++ b/ietf/nomcom/views.py
@@ -981,7 +981,7 @@ def view_feedback_topic(request, year, topic_id):
     reviewer = request.user.person
 
     last_seen = TopicFeedbackLastSeen.objects.filter(reviewer=reviewer,topic=topic).first()
-    last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.timezone.utc)
+    last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.UTC)
     if last_seen:
         last_seen.save()
     else:
@@ -1044,7 +1044,7 @@ def view_feedback_nominee(request, year, nominee_id):
                                   })
 
     last_seen = FeedbackLastSeen.objects.filter(reviewer=reviewer,nominee=nominee).first()
-    last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.timezone.utc)
+    last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.UTC)
     if last_seen:
         last_seen.save()
     else:
diff --git a/ietf/settings.py b/ietf/settings.py
index 3af01d76e6..753508dc99 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -14,19 +14,27 @@
 from hashlib import sha384
 from typing import Any, Dict, List, Tuple # pyflakes:ignore
 
+# DeprecationWarnings are suppressed by default, enable them
 warnings.simplefilter("always", DeprecationWarning)
-warnings.filterwarnings("ignore", message="pkg_resources is deprecated as an API")
-warnings.filterwarnings("ignore", "Log out via GET requests is deprecated")  # happens in oidc_provider
-warnings.filterwarnings("ignore", module="tastypie", message="The django.utils.datetime_safe module is deprecated.")
-warnings.filterwarnings("ignore", module="oidc_provider", message="The django.utils.timezone.utc alias is deprecated.")
+
+# Warnings that must be resolved for Django 5.x
+warnings.filterwarnings("ignore", "Log out via GET requests is deprecated")  # caused by oidc_provider
+warnings.filterwarnings("ignore", message="The django.utils.timezone.utc alias is deprecated.", module="oidc_provider")
+warnings.filterwarnings("ignore", message="The django.utils.datetime_safe module is deprecated.", module="tastypie")
 warnings.filterwarnings("ignore", message="The USE_DEPRECATED_PYTZ setting,")  # https://github.com/ietf-tools/datatracker/issues/5635
 warnings.filterwarnings("ignore", message="The USE_L10N setting is deprecated.")  # https://github.com/ietf-tools/datatracker/issues/5648
 warnings.filterwarnings("ignore", message="django.contrib.auth.hashers.CryptPasswordHasher is deprecated.")  # https://github.com/ietf-tools/datatracker/issues/5663
-warnings.filterwarnings("ignore", message="'urllib3\\[secure\\]' extra is deprecated")
-warnings.filterwarnings("ignore", message="The logout\\(\\) view is superseded by")
+
+# Other DeprecationWarnings
+warnings.filterwarnings("ignore", message="pkg_resources is deprecated as an API", module="pyang.plugin")
 warnings.filterwarnings("ignore", message="Report.file_reporters will no longer be available in Coverage.py 4.2", module="coverage.report")
-warnings.filterwarnings("ignore", message="Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated", module="bleach")
-warnings.filterwarnings("ignore", message="HTTPResponse.getheader\\(\\) is deprecated", module='selenium.webdriver')
+warnings.filterwarnings("ignore", message="currentThread\\(\\) is deprecated", module="coverage.pytracer")
+warnings.filterwarnings("ignore", message="co_lnotab is deprecated", module="coverage.parser")
+warnings.filterwarnings("ignore", message="datetime.datetime.utcnow\\(\\) is deprecated", module="botocore.auth")
+warnings.filterwarnings("ignore", message="datetime.datetime.utcnow\\(\\) is deprecated", module="oic.utils.time_util")
+warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="oic.utils.time_util")
+warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="pytz.tzinfo")
+
 
 base_path = pathlib.Path(__file__).resolve().parent
 BASE_DIR = str(base_path)
@@ -447,23 +455,24 @@ def skip_unreadable_post(record):
     "ietf.middleware.SMTPExceptionMiddleware",
     "ietf.middleware.Utf8ExceptionMiddleware",
     "ietf.middleware.redirect_trailing_period_middleware",
-    "django_referrer_policy.middleware.ReferrerPolicyMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
     "django.middleware.security.SecurityMiddleware",
-    #"csp.middleware.CSPMiddleware",
     "ietf.middleware.unicode_nfkc_normalization_middleware",
     "ietf.middleware.is_authenticated_header_middleware",
 ]
 
 ROOT_URLCONF = 'ietf.urls'
 
-DJANGO_VITE_ASSETS_PATH = os.path.join(BASE_DIR, 'static/dist-neue')
+# Configure django_vite
+DJANGO_VITE: dict = {"default": {}}
 if DEBUG:
-    DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json')
+    DJANGO_VITE["default"]["manifest_path"] = os.path.join(
+        BASE_DIR, 'static/dist-neue/manifest.json'
+    )
 
 # Additional locations of static files (in addition to each app's static/ dir)
 STATICFILES_DIRS = (
-    DJANGO_VITE_ASSETS_PATH,
+    os.path.join(BASE_DIR, "static/dist-neue"),  # for django_vite
     os.path.join(BASE_DIR, 'static/dist'),
     os.path.join(BASE_DIR, 'secr/static/dist'),
 )
@@ -567,8 +576,6 @@ def skip_unreadable_post(record):
 CORS_ALLOW_METHODS = ( 'GET', 'OPTIONS', )
 CORS_URLS_REGEX = r'^(/api/.*|.*\.json|.*/json/?)$'
 
-# Setting for django_referrer_policy.middleware.ReferrerPolicyMiddleware
-REFERRER_POLICY = 'strict-origin-when-cross-origin'
 
 # django.middleware.security.SecurityMiddleware 
 SECURE_BROWSER_XSS_FILTER       = True
@@ -581,6 +588,7 @@ def skip_unreadable_post(record):
 #SECURE_SSL_REDIRECT             = True
 # Relax the COOP policy to allow Meetecho authentication pop-up
 SECURE_CROSS_ORIGIN_OPENER_POLICY = "unsafe-none"
+SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
 
 # Override this in your settings_local with the IP addresses relevant for you:
 INTERNAL_IPS = (
@@ -666,11 +674,6 @@ def skip_unreadable_post(record):
 IDNITS3_BASE_URL = "https://author-tools.ietf.org/idnits3/results"
 IDNITS_SERVICE_URL = "https://author-tools.ietf.org/idnits"
 
-# Content security policy configuration (django-csp)
-# (In current production, the Content-Security-Policy header is completely set by nginx configuration, but
-#  we try to keep this in sync to avoid confusion)
-CSP_DEFAULT_SRC = ("'self'", "'unsafe-inline'", f"data: {IDTRACKER_BASE_URL} http://ietf.org/ https://www.ietf.org/ https://analytics.ietf.org/ https://static.ietf.org")
-
 # The name of the method to use to invoke the test suite
 TEST_RUNNER = 'ietf.utils.test_runner.IetfTestRunner'
 
@@ -709,6 +712,7 @@ def skip_unreadable_post(record):
     "ietf/utils/patch.py",
     "ietf/utils/test_data.py",
     "ietf/utils/jstest.py",
+    "ietf/utils/coverage.py",
 ]
 
 # These are code line regex patterns
@@ -738,8 +742,8 @@ def skip_unreadable_post(record):
 
 TEST_CODE_COVERAGE_CHECKER = None
 if SERVER_MODE != 'production':
-    import coverage
-    TEST_CODE_COVERAGE_CHECKER = coverage.Coverage(source=[ BASE_DIR ], cover_pylib=False, omit=TEST_CODE_COVERAGE_EXCLUDE_FILES)
+    from ietf.utils.coverage import CoverageManager
+    TEST_CODE_COVERAGE_CHECKER = CoverageManager()
 
 TEST_CODE_COVERAGE_REPORT_PATH = "coverage/"
 TEST_CODE_COVERAGE_REPORT_URL = os.path.join(STATIC_URL, TEST_CODE_COVERAGE_REPORT_PATH, "index.html")
diff --git a/ietf/settings_test.py b/ietf/settings_test.py
index 9a42e8b99d..6479069db0 100755
--- a/ietf/settings_test.py
+++ b/ietf/settings_test.py
@@ -14,7 +14,7 @@
 import shutil
 import tempfile
 from ietf.settings import *                                          # pyflakes:ignore
-from ietf.settings import TEST_CODE_COVERAGE_CHECKER, ORIG_AUTH_PASSWORD_VALIDATORS
+from ietf.settings import ORIG_AUTH_PASSWORD_VALIDATORS
 import debug                            # pyflakes:ignore
 debug.debug = True
 
@@ -52,10 +52,9 @@ def __getitem__(self, item):
 BLOBDB_DATABASE = "default"
 DATABASE_ROUTERS = []  # type: ignore
 
-if TEST_CODE_COVERAGE_CHECKER and not TEST_CODE_COVERAGE_CHECKER._started: # pyflakes:ignore
+if TEST_CODE_COVERAGE_CHECKER: # pyflakes:ignore
     TEST_CODE_COVERAGE_CHECKER.start()                          # pyflakes:ignore
 
-
 def tempdir_with_cleanup(**kwargs):
     """Utility to create a temporary dir and arrange cleanup"""
     _dir = tempfile.mkdtemp(**kwargs)
diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py
index 89908748a7..e02b686576 100644
--- a/ietf/submit/checkers.py
+++ b/ietf/submit/checkers.py
@@ -18,7 +18,7 @@
 from ietf.utils import tool_version
 from ietf.utils.log import log, assertion
 from ietf.utils.pipe import pipe
-from ietf.utils.test_runner import set_coverage_checking
+from ietf.utils.test_runner import disable_coverage
 
 class DraftSubmissionChecker(object):
     name = ""
@@ -247,34 +247,33 @@ def check_file_txt(self, path):
                 )
 
                 # yanglint
-                set_coverage_checking(False) # we can't count the following as it may or may not be run, depending on setup
-                if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY):
-                    cmd_template = settings.SUBMIT_YANGLINT_COMMAND
-                    command = [ w for w in cmd_template.split() if not '=' in w ][0]
-                    cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir,
-                        draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR,
-                        cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, )
-                    code, out, err = pipe(cmd)
-                    out = out.decode('utf-8')
-                    err = err.decode('utf-8')
-                    if code > 0 or len(err.strip()) > 0:
-                        err_lines = err.splitlines()
-                        for line in err_lines:
-                            if line.strip():
-                                try:
-                                    if 'err : ' in line:
-                                        errors += 1
-                                    if 'warn: ' in line:
-                                        warnings += 1
-                                except ValueError:
-                                    pass
-                    #passed = passed and code == 0 # For the submission tool.  Yang checks always pass
-                    message += "{version}: {template}:\n{output}\n".format(
-                        version=tool_version[command],
-                        template=cmd_template,
-                        output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err,
-                    )
-                set_coverage_checking(True)
+                with disable_coverage():  # pragma: no cover
+                    if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY):
+                        cmd_template = settings.SUBMIT_YANGLINT_COMMAND
+                        command = [ w for w in cmd_template.split() if not '=' in w ][0]
+                        cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir,
+                            draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR,
+                            cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, )
+                        code, out, err = pipe(cmd)
+                        out = out.decode('utf-8')
+                        err = err.decode('utf-8')
+                        if code > 0 or len(err.strip()) > 0:
+                            err_lines = err.splitlines()
+                            for line in err_lines:
+                                if line.strip():
+                                    try:
+                                        if 'err : ' in line:
+                                            errors += 1
+                                        if 'warn: ' in line:
+                                            warnings += 1
+                                    except ValueError:
+                                        pass
+                        #passed = passed and code == 0 # For the submission tool.  Yang checks always pass
+                        message += "{version}: {template}:\n{output}\n".format(
+                            version=tool_version[command],
+                            template=cmd_template,
+                            output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err,
+                        )
             else:
                 errors += 1
                 message += "No such file: %s\nPossible mismatch between extracted xym file name and returned module name?\n" % (path)
diff --git a/ietf/sync/iana.py b/ietf/sync/iana.py
index f46fe407d4..0d40c5337e 100644
--- a/ietf/sync/iana.py
+++ b/ietf/sync/iana.py
@@ -66,8 +66,8 @@ def update_rfc_log_from_protocol_page(rfc_names, rfc_must_published_later_than):
     
 
 def fetch_changes_json(url, start, end):
-    url += "?start=%s&end=%s" % (urlquote(start.astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S")),
-                                 urlquote(end.astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S")))
+    url += "?start=%s&end=%s" % (urlquote(start.astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S")),
+                                 urlquote(end.astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S")))
     # HTTP basic auth
     username = "ietfsync"
     password = settings.IANA_SYNC_PASSWORD
@@ -161,7 +161,7 @@ def update_history_with_changes(changes, send_email=True):
 
     for c in changes:
         docname = c['doc']
-        timestamp = datetime.datetime.strptime(c["time"], "%Y-%m-%d %H:%M:%S",).replace(tzinfo=datetime.timezone.utc)
+        timestamp = datetime.datetime.strptime(c["time"], "%Y-%m-%d %H:%M:%S",).replace(tzinfo=datetime.UTC)
 
         if c['type'] in ("iana_state", "iana_review"):
             if c['type'] == "iana_state":
@@ -247,7 +247,7 @@ def parse_review_email(text):
         review_time = parsedate_to_datetime(msg["Date"])
         # parsedate_to_datetime() may return a naive timezone - treat as UTC
         if review_time.tzinfo is None or review_time.tzinfo.utcoffset(review_time) is None:
-            review_time = review_time.replace(tzinfo=datetime.timezone.utc)
+            review_time = review_time.replace(tzinfo=datetime.UTC)
 
     # by
     by = None
diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py
index 18ab4fe66e..e4174d3729 100644
--- a/ietf/sync/tasks.py
+++ b/ietf/sync/tasks.py
@@ -152,7 +152,7 @@ def iana_protocols_update_task():
         2012, 
         11, 
         26, 
-        tzinfo=datetime.timezone.utc,
+        tzinfo=datetime.UTC,
     )
 
     try:
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index 182b6e24c4..3432f6214a 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -206,7 +206,7 @@ def test_iana_review_mail(self):
                         doc_name, review_time, by, comment = iana.parse_review_email(msg.encode('utf-8'))
     
                         self.assertEqual(doc_name, draft.name)
-                        self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 12, 0, rtime, tzinfo=datetime.timezone.utc))
+                        self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 12, 0, rtime, tzinfo=datetime.UTC))
                         self.assertEqual(by, Person.objects.get(user__username="iana"))
                         self.assertIn("there are no IANA Actions", comment.replace("\n", ""))
     
@@ -240,7 +240,7 @@ def test_ingest_review_email(self, mock_parse_review_email, mock_add_review_comm
 
         args = (
             "doc-name",
-            datetime.datetime.now(tz=datetime.timezone.utc),
+            datetime.datetime.now(tz=datetime.UTC),
             PersonFactory(),
             "yadda yadda yadda",
         )
@@ -1121,7 +1121,7 @@ def test_iana_protocols_update_task(
         )
         self.assertEqual(
             published_later_than, 
-            {datetime.datetime(2012,11,26,tzinfo=datetime.timezone.utc)}
+            {datetime.datetime(2012,11,26,tzinfo=datetime.UTC)}
         )
 
         # try with an exception
diff --git a/ietf/utils/test_smtpserver.py b/ietf/utils/aiosmtpd.py
similarity index 72%
rename from ietf/utils/test_smtpserver.py
rename to ietf/utils/aiosmtpd.py
index 40da758d66..3e4cd65dd9 100644
--- a/ietf/utils/test_smtpserver.py
+++ b/ietf/utils/aiosmtpd.py
@@ -1,10 +1,14 @@
 # Copyright The IETF Trust 2014-2025, All Rights Reserved
-# -*- coding: utf-8 -*-
+"""aiosmtpd-related utilities
 
+These are for testing / dev use. If you're using this for production code, think very
+hard about the choices you're making...
+"""
+from aiosmtpd import handlers
 from aiosmtpd.controller import Controller
 from aiosmtpd.smtp import SMTP
 from email.utils import parseaddr
-from typing import Optional
+from typing import Optional, TextIO
 
 
 class SMTPTestHandler:
@@ -54,3 +58,16 @@ def start(self):
 
     def stop(self):
         self.controller.stop()
+
+
+class DevDebuggingHandler(handlers.Debugging):
+    """Debugging handler for use in dev ONLY"""
+    def __init__(self, stream: Optional[TextIO] = None):
+        # Allow longer lines than the 1001 that RFC 5321 requires. As of 2025-04-16 the
+        # datatracker emits some non-compliant messages.
+        # See https://aiosmtpd.aio-libs.org/en/latest/smtp.html
+        # Doing this in a handler class is a huge hack. Tests all pass with this set
+        # to 4000, but make the limit longer for dev just in case.
+        SMTP.line_length_limit = 10000
+        super().__init__(stream)
+        
diff --git a/ietf/utils/coverage.py b/ietf/utils/coverage.py
new file mode 100644
index 0000000000..bd205ce586
--- /dev/null
+++ b/ietf/utils/coverage.py
@@ -0,0 +1,90 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from coverage import Coverage, CoverageData, FileReporter
+from coverage.control import override_config as override_coverage_config
+from coverage.results import Numbers
+from coverage.report_core import get_analysis_to_report
+from coverage.results import Analysis
+from django.conf import settings
+
+
+class CoverageManager:
+    checker: Coverage | None = None
+    started = False
+
+    def start(self):
+        if settings.SERVER_MODE != "production" and not self.started:
+            self.checker = Coverage(
+                source=[settings.BASE_DIR],
+                cover_pylib=False,
+                omit=settings.TEST_CODE_COVERAGE_EXCLUDE_FILES,
+            )
+            for exclude_regex in getattr(
+                settings,
+                "TEST_CODE_COVERAGE_EXCLUDE_LINES",
+                [],
+            ):
+                self.checker.exclude(exclude_regex)
+            self.checker.start()
+            self.started = True
+
+    def stop(self):
+        if self.checker is not None:
+            self.checker.stop()
+
+    def save(self):
+        if self.checker is not None:
+            self.checker.save()
+
+    def report(self, include: list[str] | None = None):
+        if self.checker is None:
+            return None
+        reporter = CustomDictReporter()
+        with override_coverage_config(
+            self.checker,
+            report_include=include,
+        ):
+            return reporter.report(self.checker)
+
+
+class CustomDictReporter:  # pragma: no cover
+    total = Numbers()
+
+    def report(self, coverage):
+        coverage_data = coverage.get_data()
+        coverage_data.set_query_contexts(None)
+        measured_files = {}
+        for file_reporter, analysis in get_analysis_to_report(coverage, None):
+            measured_files[file_reporter.relative_filename()] = self.report_one_file(
+                coverage_data,
+                analysis,
+                file_reporter,
+            )
+        tot_numer, tot_denom = self.total.ratio_covered
+        return {
+            "coverage": 1 if tot_denom == 0 else tot_numer / tot_denom,
+            "covered": measured_files,
+            "format": 5,
+        }
+
+    def report_one_file(
+        self,
+        coverage_data: CoverageData,
+        analysis: Analysis,
+        file_reporter: FileReporter,
+    ):
+        """Extract the relevant report data for a single file."""
+        nums = analysis.numbers
+        self.total += nums
+        n_statements = nums.n_statements
+        numer, denom = nums.ratio_covered
+        fraction_covered = 1 if denom == 0 else numer / denom
+        missing_line_nums = sorted(analysis.missing)
+        # Extract missing lines from source files
+        source_lines = file_reporter.source().splitlines()
+        missing_lines = [source_lines[num - 1] for num in missing_line_nums]
+        return (
+            n_statements,
+            fraction_covered,
+            missing_line_nums,
+            missing_lines,
+        )
diff --git a/ietf/utils/decorators.py b/ietf/utils/decorators.py
index 5e94dda91d..b50e0e7f96 100644
--- a/ietf/utils/decorators.py
+++ b/ietf/utils/decorators.py
@@ -15,21 +15,9 @@
 
 import debug                            # pyflakes:ignore
 
-from ietf.utils.test_runner import set_coverage_checking
 from ietf.person.models import Person, PersonalApiKey, PersonApiKeyEvent
 from ietf.utils import log
 
-def skip_coverage(f):
-    @wraps(f)
-    def _wrapper(*args, **kwargs):
-        if settings.TEST_CODE_COVERAGE_CHECKER:
-            set_coverage_checking(False)
-            result = f(*args, **kwargs)
-            set_coverage_checking(True)
-            return result
-        else:
-            return  f(*args, **kwargs)
-    return _wrapper
 
 def person_required(f):
     @wraps(f)
diff --git a/ietf/utils/jstest.py b/ietf/utils/jstest.py
index 215d78d65f..cf242fc4eb 100644
--- a/ietf/utils/jstest.py
+++ b/ietf/utils/jstest.py
@@ -3,6 +3,8 @@
 
 import os
 
+from django.conf import settings
+from django.contrib.staticfiles.testing import StaticLiveServerTestCase
 from django.urls import reverse as urlreverse
 from unittest import skipIf
 
@@ -21,7 +23,11 @@
 
 
 from ietf.utils.pipe import pipe
-from ietf.utils.test_runner import IetfLiveServerTestCase
+from ietf.utils.test_runner import (
+    set_template_coverage,
+    set_url_coverage,
+    load_and_run_fixtures,
+)
 
 executable_name = 'geckodriver'
 code, out, err = pipe('{} --version'.format(executable_name))
@@ -49,17 +55,44 @@ def ifSeleniumEnabled(func):
     return skipIf(skip_selenium, skip_message)(func)
 
 
-class IetfSeleniumTestCase(IetfLiveServerTestCase):
+class IetfSeleniumTestCase(StaticLiveServerTestCase):  # pragma: no cover
     login_view = 'ietf.ietfauth.views.login'
 
+    @classmethod
+    def setUpClass(cls):
+        set_template_coverage(False)
+        set_url_coverage(False)
+        super().setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        super().tearDownClass()
+        set_template_coverage(True)
+        set_url_coverage(True)
+
     def setUp(self):
-        super(IetfSeleniumTestCase, self).setUp()
+        super().setUp()
+        # LiveServerTestCase uses TransactionTestCase which seems to
+        # somehow interfere with the fixture loading process in
+        # IetfTestRunner when running multiple tests (the first test
+        # is fine, in the next ones the fixtures have been wiped) -
+        # this is no doubt solvable somehow, but until then we simply
+        # recreate them here
+        from ietf.person.models import Person
+        if not Person.objects.exists():
+            load_and_run_fixtures(verbosity=0)
+        self.replaced_settings = dict()
+        if hasattr(settings, 'IDTRACKER_BASE_URL'):
+            self.replaced_settings['IDTRACKER_BASE_URL'] = settings.IDTRACKER_BASE_URL
+            settings.IDTRACKER_BASE_URL = self.live_server_url
         self.driver = start_web_driver()
         self.driver.set_window_size(1024,768)
     
     def tearDown(self):
-        super(IetfSeleniumTestCase, self).tearDown()
         self.driver.close()
+        for k, v in self.replaced_settings.items():
+            setattr(settings, k, v)
+        super().tearDown()
 
     def absreverse(self,*args,**kwargs):
         return '%s%s'%(self.live_server_url, urlreverse(*args, **kwargs))
diff --git a/ietf/utils/meetecho.py b/ietf/utils/meetecho.py
index 0dbf75736a..7654f67cd1 100644
--- a/ietf/utils/meetecho.py
+++ b/ietf/utils/meetecho.py
@@ -27,7 +27,7 @@
 
 
 class MeetechoAPI:
-    timezone = datetime.timezone.utc
+    timezone = datetime.UTC
 
     def __init__(
         self, api_base: str, client_id: str, client_secret: str, request_timeout=3.01
@@ -504,7 +504,7 @@ def _should_send_update(self, session):
         if self.slides_notify_time < datetime.timedelta(0):
             return True  # < 0 means "always" for a scheduled session
         else:
-            now = datetime.datetime.now(tz=datetime.timezone.utc)
+            now = datetime.datetime.now(tz=datetime.UTC)
             return (timeslot.time - self.slides_notify_time) < now < (timeslot.end_time() + self.slides_notify_time)
 
     def add(self, session: "Session", slides: "Document", order: int):
diff --git a/ietf/utils/serialize.py b/ietf/utils/serialize.py
index 342d211cf5..77f97942cb 100644
--- a/ietf/utils/serialize.py
+++ b/ietf/utils/serialize.py
@@ -16,7 +16,7 @@ def object_as_shallow_dict(obj):
         if isinstance(f, models.ManyToManyField):
             v = list(v.values_list("pk", flat=True))
         elif isinstance(f, models.DateTimeField):
-            v = v.astimezone(datetime.timezone.utc).isoformat()
+            v = v.astimezone(datetime.UTC).isoformat()
         elif isinstance(f, models.DateField):
             v = v.strftime('%Y-%m-%d')
 
diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py
index a9b2e5d572..1a3d4e5c3d 100644
--- a/ietf/utils/test_runner.py
+++ b/ietf/utils/test_runner.py
@@ -48,6 +48,8 @@
 import subprocess
 import tempfile
 import copy
+from contextlib import contextmanager
+
 import boto3
 import botocore.config
 import factory.random
@@ -57,10 +59,6 @@
 from typing import Callable, Optional
 from urllib.parse import urlencode
 
-from coverage.report import Reporter
-from coverage.results import Numbers
-from coverage.misc import NotPython
-
 import django
 from django.conf import settings
 from django.contrib.staticfiles.testing import StaticLiveServerTestCase
@@ -84,7 +82,7 @@
 import ietf
 import ietf.utils.mail
 from ietf.utils.management.commands import pyflakes
-from ietf.utils.test_smtpserver import SMTPTestServerDriver
+from ietf.utils.aiosmtpd import SMTPTestServerDriver
 from ietf.utils.test_utils import TestCase
 
 from mypy_boto3_s3.service_resource import Bucket
@@ -96,11 +94,11 @@
 old_destroy: Optional[Callable] = None
 old_create: Optional[Callable] = None
 
-template_coverage_collection = None
-code_coverage_collection = None
-url_coverage_collection = None
+template_coverage_collection = False
+url_coverage_collection = False
 validation_settings = {"validate_html": None, "validate_html_harder": None, "show_logging": False}
 
+
 def start_vnu_server(port=8888):
     "Start a vnu validation server on the indicated port"
     vnu = subprocess.Popen(
@@ -463,50 +461,29 @@ def save_test_results(failures, test_labels):
             tfile.write("%s OK\n" % (timestr, ))
     tfile.close()
 
-def set_coverage_checking(flag=True):
+
+def set_template_coverage(flag):
     global template_coverage_collection
-    global code_coverage_collection
+    orig = template_coverage_collection
+    template_coverage_collection = flag
+    return orig
+
+
+def set_url_coverage(flag):
     global url_coverage_collection
-    if settings.SERVER_MODE == 'test':
-        if flag:
-            settings.TEST_CODE_COVERAGE_CHECKER.collector.resume()
-            template_coverage_collection = True
-            code_coverage_collection = True
-            url_coverage_collection = True
-        else:
-            settings.TEST_CODE_COVERAGE_CHECKER.collector.pause()
-            template_coverage_collection = False
-            code_coverage_collection = False
-            url_coverage_collection = False
-
-class CoverageReporter(Reporter):
-    def report(self):
-        self.find_file_reporters(None)
-
-        total = Numbers()
-        result = {"coverage": 0.0, "covered": {}, "format": 5, }
-        for fr in self.file_reporters:
-            try:
-                analysis = self.coverage._analyze(fr)
-                nums = analysis.numbers
-                missing_nums = sorted(analysis.missing)
-                with io.open(analysis.filename, encoding='utf-8') as file:
-                    lines = file.read().splitlines()
-                missing_lines = [ lines[l-1] for l in missing_nums ]
-                result["covered"][fr.relative_filename()] = (nums.n_statements, nums.pc_covered/100.0, missing_nums, missing_lines)
-                total += nums
-            except KeyboardInterrupt:                   # pragma: not covered
-                raise
-            except Exception:
-                report_it = not self.config.ignore_errors
-                if report_it:
-                    typ, msg = sys.exc_info()[:2]
-                    if typ is NotPython and not fr.should_be_python():
-                        report_it = False
-                if report_it:
-                    raise
-        result["coverage"] = total.pc_covered/100.0
-        return result
+    orig = url_coverage_collection
+    url_coverage_collection = flag
+    return orig
+
+
+@contextmanager
+def disable_coverage():
+    """Context manager/decorator that disables template/url coverage"""
+    orig_template = set_template_coverage(False)
+    orig_url = set_url_coverage(False)
+    yield
+    set_template_coverage(orig_template)
+    set_url_coverage(orig_url)
 
 
 class CoverageTest(unittest.TestCase):
@@ -594,23 +571,24 @@ def ignore_pattern(regex, pattern):
             self.skipTest("Coverage switched off with --skip-coverage")
 
     def code_coverage_test(self):
-        if self.runner.check_coverage:
-            include = [ os.path.join(path, '*') for path in self.runner.test_paths ]
-            checker = self.runner.code_coverage_checker
-            checker.stop()
+        if (
+            self.runner.check_coverage
+            and settings.TEST_CODE_COVERAGE_CHECKER is not None
+        ):
+            coverage_manager = settings.TEST_CODE_COVERAGE_CHECKER
+            coverage_manager.stop()
             # Save to the .coverage file
-            checker.save()
+            coverage_manager.save()
             # Apply the configured and requested omit and include data
-            checker.config.from_args(ignore_errors=None, omit=settings.TEST_CODE_COVERAGE_EXCLUDE_FILES,
-                include=include, file=None)
-            for pattern in settings.TEST_CODE_COVERAGE_EXCLUDE_LINES:
-                checker.exclude(pattern)
             # Maybe output an HTML report
             if self.runner.run_full_test_suite and self.runner.html_report:
-                checker.html_report(directory=settings.TEST_CODE_COVERAGE_REPORT_DIR)
-            # In any case, build a dictionary with per-file data for this run
-            reporter = CoverageReporter(checker, checker.config)
-            self.runner.coverage_data["code"] = reporter.report()
+                coverage_manager.checker.html_report(
+                    directory=settings.TEST_CODE_COVERAGE_REPORT_DIR
+                )
+            # Generate the output report data
+            self.runner.coverage_data["code"] = coverage_manager.report(
+                include=[str(pathlib.Path(p) / "*") for p in self.runner.test_paths]
+            )
             self.report_test_result("code")
         else:
             self.skipTest("Coverage switched off with --skip-coverage")
@@ -824,23 +802,12 @@ def setup_test_environment(self, **kwargs):
                     "covered": {},
                     "format": 1,
                 },
-                "migration": {
-                    "present": {},
-                    "format": 3,
-                }
             }
 
             settings.TEMPLATES[0]['OPTIONS']['loaders'] = ('ietf.utils.test_runner.TemplateCoverageLoader',) + settings.TEMPLATES[0]['OPTIONS']['loaders']
 
             settings.MIDDLEWARE = ('ietf.utils.test_runner.record_urls_middleware',) + tuple(settings.MIDDLEWARE)
 
-            self.code_coverage_checker = settings.TEST_CODE_COVERAGE_CHECKER
-            if not self.code_coverage_checker._started:
-                sys.stderr.write(" **  Warning: In %s: Expected the coverage checker to have\n"
-                                 "       been started already, but it wasn't. Doing so now.  Coverage numbers\n"
-                                 "       will be off, though.\n" % __name__)
-                self.code_coverage_checker.start()
-
         if settings.SITE_ID != 1:
             print("     Changing SITE_ID to '1' during testing.")
             settings.SITE_ID = 1
@@ -1140,9 +1107,8 @@ def _extra_tests(self):
                 ),
             ]
         if self.check_coverage:
-            global template_coverage_collection, code_coverage_collection, url_coverage_collection
+            global template_coverage_collection, url_coverage_collection
             template_coverage_collection = True
-            code_coverage_collection = True
             url_coverage_collection = True
             tests += [
                 PyFlakesTestCase(test_runner=self, methodName='pyflakes_test'),
@@ -1226,37 +1192,6 @@ def run_tests(self, test_labels, extra_tests=None, **kwargs):
 
         return failures
 
-class IetfLiveServerTestCase(StaticLiveServerTestCase):
-    @classmethod
-    def setUpClass(cls):
-        set_coverage_checking(False)
-        super(IetfLiveServerTestCase, cls).setUpClass()
-
-    def setUp(self):
-        super(IetfLiveServerTestCase, self).setUp()
-        # LiveServerTestCase uses TransactionTestCase which seems to
-        # somehow interfere with the fixture loading process in
-        # IetfTestRunner when running multiple tests (the first test
-        # is fine, in the next ones the fixtures have been wiped) -
-        # this is no doubt solvable somehow, but until then we simply
-        # recreate them here
-        from ietf.person.models import Person
-        if not Person.objects.exists():
-            load_and_run_fixtures(verbosity=0)
-        self.replaced_settings = dict()
-        if hasattr(settings, 'IDTRACKER_BASE_URL'):
-            self.replaced_settings['IDTRACKER_BASE_URL'] = settings.IDTRACKER_BASE_URL
-            settings.IDTRACKER_BASE_URL = self.live_server_url
-
-    @classmethod
-    def tearDownClass(cls):
-        super(IetfLiveServerTestCase, cls).tearDownClass()
-        set_coverage_checking(True)
-
-    def tearDown(self):
-        for k, v in self.replaced_settings.items():
-            setattr(settings, k, v)
-        super().tearDown()
 
 class TestBlobstoreManager():
     # N.B. buckets and blobstore are intentional Class-level attributes
@@ -1267,7 +1202,11 @@ class TestBlobstoreManager():
         aws_access_key_id="minio_root",
         aws_secret_access_key="minio_pass",
         aws_session_token=None,
-        config = botocore.config.Config(signature_version="s3v4"),
+        config = botocore.config.Config(
+            request_checksum_calculation="when_required",
+            response_checksum_validation="when_required",
+            signature_version="s3v4",
+        ),
         #config=botocore.config.Config(signature_version=botocore.UNSIGNED),
         verify=False
     )
diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py
index 01433888fe..3288309095 100644
--- a/ietf/utils/tests.py
+++ b/ietf/utils/tests.py
@@ -54,7 +54,11 @@
     decode_header_value,
     show_that_mail_was_sent,
 )
-from ietf.utils.test_runner import get_template_paths, set_coverage_checking
+from ietf.utils.test_runner import (
+    get_template_paths,
+    set_template_coverage,
+    set_url_coverage,
+)
 from ietf.utils.test_utils import TestCase, unicontent
 from ietf.utils.text import parse_unicode
 from ietf.utils.timezone import timezone_not_near_midnight
@@ -311,14 +315,15 @@ def qualified(name):
 
     return list(callbacks)
 
-class TemplateChecksTestCase(TestCase):
+class TemplateChecksTestCase(TestCase):  # pragma: no cover
 
     paths = []                          # type: List[str]
     templates = {}                      # type: Dict[str, Template]
 
     def setUp(self):
         super().setUp()
-        set_coverage_checking(False)
+        set_template_coverage(False)
+        set_url_coverage(False)
         self.paths = get_template_paths()  # already filtered ignores
         self.paths.sort()
         for path in self.paths:
@@ -328,7 +333,8 @@ def setUp(self):
                 pass
 
     def tearDown(self):
-        set_coverage_checking(True)
+        set_template_coverage(True)
+        set_url_coverage(True)
         super().tearDown()
 
     def test_parse_templates(self):
diff --git a/ietf/utils/tests_coverage.py b/ietf/utils/tests_coverage.py
new file mode 100644
index 0000000000..68795994a7
--- /dev/null
+++ b/ietf/utils/tests_coverage.py
@@ -0,0 +1,56 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+"""Tests of the coverage.py module"""
+
+from unittest import mock
+
+from django.test import override_settings
+
+from .coverage import CoverageManager
+from .test_utils import TestCase
+
+
+class CoverageManagerTests(TestCase):
+    @override_settings(
+        BASE_DIR="/path/to/project/ietf",
+        TEST_CODE_COVERAGE_EXCLUDE_FILES=["a.py"],
+        TEST_CODE_COVERAGE_EXCLUDE_LINES=["some-regex"],
+    )
+    @mock.patch("ietf.utils.coverage.Coverage")
+    def test_coverage_manager(self, mock_coverage):
+        """CoverageManager managed coverage correctly in non-production mode
+
+        Presumes we're not running tests in production mode.
+        """
+        cm = CoverageManager()
+        self.assertFalse(cm.started)
+
+        cm.start()
+        self.assertTrue(cm.started)
+        self.assertEqual(cm.checker, mock_coverage.return_value)
+        self.assertTrue(mock_coverage.called)
+        coverage_kwargs = mock_coverage.call_args.kwargs
+        self.assertEqual(coverage_kwargs["source"], ["/path/to/project/ietf"])
+        self.assertEqual(coverage_kwargs["omit"], ["a.py"])
+        self.assertTrue(isinstance(cm.checker.exclude, mock.Mock))
+        assert isinstance(cm.checker.exclude, mock.Mock)  # for type checker
+        self.assertEqual(cm.checker.exclude.call_count, 1)
+        cm.checker.exclude.assert_called_with("some-regex")
+
+    @mock.patch("ietf.utils.coverage.Coverage")
+    def test_coverage_manager_is_defanged_in_production(self, mock_coverage):
+        """CoverageManager is a no-op in production mode"""
+        # Be careful faking settings.SERVER_MODE, but there's really no other way to
+        # test this.
+        with override_settings(SERVER_MODE="production"):
+            cm = CoverageManager()
+            cm.start()
+
+        # Check that nothing actually happened
+        self.assertFalse(mock_coverage.called)
+        self.assertIsNone(cm.checker)
+        self.assertFalse(cm.started)
+
+        # Check that other methods are guarded appropriately
+        cm.stop()
+        cm.save()
+        self.assertIsNone(cm.report())
diff --git a/ietf/utils/tests_meetecho.py b/ietf/utils/tests_meetecho.py
index a10ac68c27..502e936483 100644
--- a/ietf/utils/tests_meetecho.py
+++ b/ietf/utils/tests_meetecho.py
@@ -98,7 +98,7 @@ def test_schedule_meeting(self):
         api_response = api.schedule_meeting(
             wg_token='my-token',
             room_id=18,
-            start_time=datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.timezone.utc),
+            start_time=datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.UTC),
             duration=datetime.timedelta(minutes=130),
             description='interim-2021-wgname-01',
             extrainfo='message for staff',
@@ -127,7 +127,7 @@ def test_schedule_meeting(self):
         )
         # same time in different time zones
         for start_time in [
-            datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.timezone.utc),
+            datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.UTC),
             datetime.datetime(2021, 9, 14, 7, 0, 0, tzinfo=ZoneInfo('America/Halifax')),
             datetime.datetime(2021, 9, 14, 13, 0, 0, tzinfo=ZoneInfo('Europe/Kiev')),
             datetime.datetime(2021, 9, 14, 5, 0, 0, tzinfo=ZoneInfo('Pacific/Easter')),
@@ -198,7 +198,7 @@ def test_fetch_meetings(self):
                     '3d55bce0-535e-4ba8-bb8e-734911cf3c32': {
                         'room': {
                             'id': 18,
-                            'start_time': datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.timezone.utc),
+                            'start_time': datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.UTC),
                             'duration': datetime.timedelta(minutes=130),
                             'description': 'interim-2021-wgname-01',
                         },
@@ -208,7 +208,7 @@ def test_fetch_meetings(self):
                     'e68e96d4-d38f-475b-9073-ecab46ca96a5': {
                         'room': {
                             'id': 23,
-                            'start_time': datetime.datetime(2021, 9, 15, 14, 30, 0, tzinfo=datetime.timezone.utc),
+                            'start_time': datetime.datetime(2021, 9, 15, 14, 30, 0, tzinfo=datetime.UTC),
                             'duration': datetime.timedelta(minutes=30),
                             'description': 'interim-2021-wgname-02',
                         },
@@ -386,7 +386,7 @@ def test_request_helper_exception(self):
 
     def test_time_serialization(self):
         """Time de/serialization should be consistent"""
-        time = timezone.now().astimezone(datetime.timezone.utc).replace(microsecond=0)  # cut off to 0 microseconds
+        time = timezone.now().astimezone(datetime.UTC).replace(microsecond=0)  # cut off to 0 microseconds
         api = MeetechoAPI(API_BASE, CLIENT_ID, CLIENT_SECRET)
         self.assertEqual(api._deserialize_time(api._serialize_time(time)), time)
 
@@ -400,7 +400,7 @@ def test_conference_from_api_dict(self):
                 'session-1-uuid': {
                     'room': {
                         'id': 1,
-                        'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc),
+                        'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC),
                         'duration': datetime.timedelta(minutes=45),
                         'description': 'some-description',
                     },
@@ -410,7 +410,7 @@ def test_conference_from_api_dict(self):
                 'session-2-uuid': {
                     'room': {
                         'id': 2,
-                        'start_time': datetime.datetime(2022,2,5,4,5,6, tzinfo=datetime.timezone.utc),
+                        'start_time': datetime.datetime(2022,2,5,4,5,6, tzinfo=datetime.UTC),
                         'duration': datetime.timedelta(minutes=90),
                         'description': 'another-description',
                     },
@@ -427,7 +427,7 @@ def test_conference_from_api_dict(self):
                     id=1,
                     public_id='session-1-uuid',
                     description='some-description',
-                    start_time=datetime.datetime(2022, 2, 4, 1, 2, 3, tzinfo=datetime.timezone.utc),
+                    start_time=datetime.datetime(2022, 2, 4, 1, 2, 3, tzinfo=datetime.UTC),
                     duration=datetime.timedelta(minutes=45),
                     url='https://example.com/some/url',
                     deletion_token='delete-me',
@@ -437,7 +437,7 @@ def test_conference_from_api_dict(self):
                     id=2,
                     public_id='session-2-uuid',
                     description='another-description',
-                    start_time=datetime.datetime(2022, 2, 5, 4, 5, 6, tzinfo=datetime.timezone.utc),
+                    start_time=datetime.datetime(2022, 2, 5, 4, 5, 6, tzinfo=datetime.UTC),
                     duration=datetime.timedelta(minutes=90),
                     url='https://example.com/another/url',
                     deletion_token='delete-me-too',
@@ -453,7 +453,7 @@ def test_fetch(self, mock_fetch, _):
                 'session-1-uuid': {
                     'room': {
                         'id': 1,
-                        'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc),
+                        'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC),
                         'duration': datetime.timedelta(minutes=45),
                         'description': 'some-description',
                     },
@@ -472,7 +472,7 @@ def test_fetch(self, mock_fetch, _):
                 id=1,
                 public_id='session-1-uuid',
                 description='some-description',
-                start_time=datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc),
+                start_time=datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC),
                 duration=datetime.timedelta(minutes=45),
                 url='https://example.com/some/url',
                 deletion_token='delete-me',
@@ -488,7 +488,7 @@ def test_create(self, mock_schedule, _):
                 'session-1-uuid': {
                     'room': {
                         'id': 1,  # value should match session_id param to cm.create() below 
-                        'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc),
+                        'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC),
                         'duration': datetime.timedelta(minutes=45),
                         'description': 'some-description',
                     },
@@ -506,7 +506,7 @@ def test_create(self, mock_schedule, _):
                 id=1,
                 public_id='session-1-uuid',
                 description='some-description',
-                start_time=datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc),
+                start_time=datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC),
                 duration=datetime.timedelta(minutes=45),
                 url='https://example.com/some/url',
                 deletion_token='delete-me',
diff --git a/ietf/utils/timezone.py b/ietf/utils/timezone.py
index a396b5e82d..e08dfa02f2 100644
--- a/ietf/utils/timezone.py
+++ b/ietf/utils/timezone.py
@@ -26,7 +26,7 @@ def _tzinfo(tz: Union[str, datetime.tzinfo, None]):
     Accepts a tzinfo or string containing a timezone name. Defaults to UTC if tz is None.
     """
     if tz is None:
-        return datetime.timezone.utc
+        return datetime.UTC
     elif isinstance(tz, datetime.tzinfo):
         return tz
     else:
diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 482a4b110a..c1436e158b 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -280,7 +280,9 @@ def _multiline_to_list(s):
 PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME
 
 # Normally only set for debug, but needed until we have a real FS
-DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, "static/dist-neue/manifest.json")
+DJANGO_VITE["default"]["manifest_path"] = os.path.join(
+    BASE_DIR, "static/dist-neue/manifest.json"
+)
 
 # Binaries that are different in the docker image
 DE_GFM_BINARY = "/usr/local/bin/de-gfm"
@@ -379,6 +381,8 @@ def _multiline_to_list(s):
             secret_key=_blob_store_secret_key,
             security_token=None,
             client_config=botocore.config.Config(
+                request_checksum_calculation="when_required",
+                response_checksum_validation="when_required",
                 signature_version="s3v4",
                 connect_timeout=_blob_store_connect_timeout,
                 read_timeout=_blob_store_read_timeout,
diff --git a/patch/tastypie-django22-fielderror-response.patch b/patch/tastypie-django22-fielderror-response.patch
index ffb152d319..3b4418fc66 100644
--- a/patch/tastypie-django22-fielderror-response.patch
+++ b/patch/tastypie-django22-fielderror-response.patch
@@ -1,5 +1,5 @@
---- tastypie/resources.py.orig	2020-08-24 13:14:25.463166100 +0200
-+++ tastypie/resources.py	2020-08-24 13:15:55.133759224 +0200
+--- tastypie/resources.py.orig	2025-07-29 19:00:01.526948002 +0000
++++ tastypie/resources.py	2025-07-29 19:07:15.324127008 +0000
 @@ -12,7 +12,7 @@
      ObjectDoesNotExist, MultipleObjectsReturned, ValidationError, FieldDoesNotExist
  )
@@ -9,13 +9,13 @@
  from django.db.models.fields.related import ForeignKey
  from django.urls.conf import re_path
  from tastypie.utils.timezone import make_naive_utc
-@@ -2198,6 +2198,8 @@
+@@ -2216,6 +2216,8 @@
              return self.authorized_read_list(objects, bundle)
          except ValueError:
              raise BadRequest("Invalid resource lookup data provided (mismatched type).")
 +        except FieldError as e:
 +            raise BadRequest("Invalid resource lookup: %s." % e)
- 
+
      def obj_get(self, bundle, **kwargs):
          """
 --- tastypie/paginator.py.orig	2020-08-25 15:24:46.391588425 +0200
diff --git a/requirements.txt b/requirements.txt
index 60d3d8152e..cf7c920fa3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,86 +1,85 @@
 # -*- conf-mode -*-
-setuptools>=51.1.0    # Require this first, to prevent later errors
+setuptools>=80.9.0  # Require this first, to prevent later errors
 #
 aiosmtpd>=1.4.6
-argon2-cffi>=21.3.0    # For the Argon2 password hasher option
-beautifulsoup4>=4.11.1    # Only used in tests
-bibtexparser>=1.2.0    # Only used in tests
-bleach>=6
-types-bleach>=6
-boto3>=1.35,<1.36
-boto3-stubs[s3]>=1.35,<1.36
-botocore>=1.35,<1.36
-celery>=5.2.6
-coverage>=4.5.4,<5.0    # Coverage 5.x moves from a json database to SQLite.  Moving to 5.x will require substantial rewrites in ietf.utils.test_runner and ietf.release.views
+argon2-cffi>=25.1.0  # For the Argon2 password hasher option
+beautifulsoup4>=4.13.4  # Only used in tests
+bibtexparser>=1.4.3  # Only used in tests
+bleach>=6.2.0  # project is deprecated but supported
+types-bleach>=6.2.0
+boto3>=1.39.15
+boto3-stubs[s3]>=1.39.15
+botocore>=1.39.15
+celery>=5.5.3
+coverage>=7.9.2
 defusedxml>=0.7.1    # for TastyPie when using xml; not a declared dependency
 Django>4.2,<5
-django-admin-rangefilter>=0.13.2
-django-analytical>=3.1.0
-django-bootstrap5>=21.3
-django-celery-beat>=2.3.0,<2.8.0  # pin until https://github.com/celery/django-celery-beat/issues/875 is resolved, then revisit
-django-celery-results>=2.5.1
-django-csp>=3.7
-django-cors-headers>=3.11.0
-django-debug-toolbar>=3.2.4
-django-markup>=1.5    # Limited use - need to reconcile against direct use of markdown
+django-admin-rangefilter>=0.13.3
+django-analytical>=3.2.0
+django-bootstrap5>=25.1
+django-celery-beat>=2.7.0,<2.8.0  # pin until https://github.com/celery/django-celery-beat/issues/875 is resolved, then revisit
+django-celery-results>=2.6.0
+django-cors-headers>=4.7.0
+django-debug-toolbar>=6.0.0
+django-markup>=1.10    # Limited use - need to reconcile against direct use of markdown
 django-oidc-provider==0.8.2    # 0.8.3 changes logout flow and claim return
-django-referrer-policy>=1.0
-django-simple-history>=3.0.0
-django-storages>=1.14.4
+django-simple-history>=3.10.1
+django-storages>=1.14.6
 django-stubs>=4.2.7,<5    # The django-stubs version used determines the the mypy version indicated below
-django-tastypie>=0.14.7,<0.15.0    # Version must be locked in sync with version of Django
-django-vite>=2.0.2,<3
+django-tastypie>=0.15.1  # Version must be kept in sync with Django
+django-vite>=3.1.0
 django-widget-tweaks>=1.4.12
-djangorestframework>=3.15,<4
-djlint>=1.0.0    # To auto-indent templates via "djlint --profile django --reformat"
-docutils>=0.18.1    # Used only by dbtemplates for RestructuredText
+djangorestframework>=3.16.0
+docutils>=0.22.0  # Used only by dbtemplates for RestructuredText
+types-docutils>=0.21.0  # should match docutils (0.22.0 not out yet)
 drf-spectacular>=0.27
-drf-standardized-errors[openapi] >= 0.14
-types-docutils>=0.18.1
-factory-boy>=3.3
-gunicorn>=20.1.0
+drf-standardized-errors[openapi] >= 0.15.0
+factory-boy>=3.3.3
+gunicorn>=23.0.0
 hashids>=1.3.1
-html2text>=2020.1.16    # Used only to clean comment field of secr/sreq
+html2text>=2025.4.15  # Used only to clean comment field of secr/sreq
 html5lib>=1.1    # Only used in tests
 icalendar>=5.0.0 
-inflect>= 6.0.2
-jsonfield>=3.1.0,<3.2.0  # 3.2.0 needs py3.10; deprecated-replace with Django JSONField
-jsonschema[format]>=4.2.1
-jwcrypto>=1.2    # for signed notifications - this is aspirational, and is not really used.
-logging_tree>=1.9    # Used only by the showloggers management command
-lxml>=5.3.0
-markdown>=3.3.6
-types-markdown>=3.3.6
-mypy~=1.7.0    # Version requirements determined by django-stubs.
-oic>=1.3    # Used only by tests
-Pillow>=9.1.0
-psycopg2>=2.9.6
-pyang>=2.5.3
-pydyf>0.8.0
-pyflakes>=2.4.0
-pyopenssl>=22.0.0    # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency
-pyquery>=1.4.3
-python-dateutil>=2.8.2
-types-python-dateutil>=2.8.2
-python-json-logger>=3.1.0
+inflect>= 7.5.0
+jsonfield>=3.2.0  # deprecated - need to replace with Django's JSONField
+jsonschema[format]>=4.25.0
+jwcrypto>=1.5.6  # for signed notifications - this is aspirational, and is not really used.
+logging_tree>=1.10  # Used only by the showloggers management command
+lxml>=6.0.0
+markdown>=3.8.0
+types-markdown>=3.8.0
+mock>=5.2.0  # should replace with unittest.mock and remove dependency
+types-mock>=5.2.0
+mypy~=1.7.0  # Version requirements determined by django-stubs.
+oic>=1.7.0  # Used only by tests
+pillow>=11.3.0
+psycopg2>=2.9.10
+pyang>=2.6.1
+pydyf>=0.11.0
+pyflakes>=3.4.0
+pyopenssl>=25.1.0  # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency
+pyquery>=2.0.1
+python-dateutil>=2.9.0
+types-python-dateutil>=2.9.0
+python-json-logger>=3.3.0
 python-magic==0.4.18    # Versions beyond the yanked .19 and .20 introduce form failures
 pymemcache>=4.0.0  # for django.core.cache.backends.memcached.PyMemcacheCache
-python-mimeparse>=1.6    # from TastyPie
+python-mimeparse>=2.0.0  # from TastyPie
 pytz==2025.2  # Pinned as changes need to be vetted for their effect on Meeting fields
-types-pytz==2025.2.0.20250809  # match pytz versionrequests>=2.31.0
-requests>=2.31.0
-types-requests>=2.27.1
-requests-mock>=1.9.3
+types-pytz==2025.2.0.20250809  # match pytz version
+requests>=2.32.4
+types-requests>=2.32.4
+requests-mock>=1.12.1
 rfc2html>=2.0.3
-scout-apm>=2.24.2
-selenium>=4.0
-tblib>=1.7.0    # So that the django test runner provides tracebacks
-tlds>=2022042700    # Used to teach bleach about which TLDs currently exist
-tqdm>=4.64.0
-types-zxcvbn~=4.5.0.20250223  # match zxcvbn version
-Unidecode>=1.3.4
-urllib3>=1.26,<2
-weasyprint>=64.1
-xml2rfc>=3.23.0
+scout-apm>=3.4.0
+selenium>=4.34.2
+tblib>=3.1.0  # So that the django test runner provides tracebacks
+tlds>=2022042700  # Used to teach bleach about which TLDs currently exist
+tqdm>=4.67.1
+unidecode>=1.4.0
+urllib3>=2.5.0
+weasyprint>=66.0
+xml2rfc>=3.30.0
 xym>=0.6,<1.0
 zxcvbn>=4.5.0
+types-zxcvbn~=4.5.0.20250223  # match zxcvbn version

From b14512e840d8dfccf4e418ac184c77321595278b Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Wed, 3 Sep 2025 22:29:19 +0000
Subject: [PATCH 448/656] ci: update base image target version to 20250903T2216

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 658f1e5695..d3b186e1f5 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250819T1645
+FROM ghcr.io/ietf-tools/datatracker-app-base:20250903T2216
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 9e510ad8db..9d8427efdb 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250819T1645
+20250903T2216

From e444d9e73c78a1100ad5b909f2b15012be287889 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 3 Sep 2025 20:55:17 -0300
Subject: [PATCH 449/656] chore: use :latest instead of :py312 (#9460)

---
 .github/workflows/tests-az.yml     | 2 +-
 dev/deploy-to-container/cli.js     | 6 +++---
 dev/diff/cli.js                    | 6 +++---
 dev/tests/debug.sh                 | 2 +-
 dev/tests/docker-compose.debug.yml | 2 +-
 docker/app.Dockerfile              | 2 +-
 docker/celery.Dockerfile           | 2 +-
 7 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml
index d1fe0cdf62..8553563a19 100644
--- a/.github/workflows/tests-az.yml
+++ b/.github/workflows/tests-az.yml
@@ -62,7 +62,7 @@ jobs:
           echo "Starting Containers..."
           sudo docker network create dtnet
           sudo docker run -d --name db --network=dtnet ghcr.io/ietf-tools/datatracker-db:latest &
-          sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:py312 sleep infinity &
+          sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:latest sleep infinity &
           wait
 
           echo "Cloning datatracker repo..."
diff --git a/dev/deploy-to-container/cli.js b/dev/deploy-to-container/cli.js
index 2f0faad151..1a2d993ac4 100644
--- a/dev/deploy-to-container/cli.js
+++ b/dev/deploy-to-container/cli.js
@@ -85,7 +85,7 @@ async function main () {
   
   // Pull latest Datatracker Base image
   console.info('Pulling latest Datatracker base docker image...')
-  const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:py312')
+  const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:latest')
   await new Promise((resolve, reject) => {
     dock.modem.followProgress(appImagePullStream, (err, res) => err ? reject(err) : resolve(res))
   })
@@ -214,7 +214,7 @@ async function main () {
   const celeryContainers = {}
   for (const conConf of conConfs) {
     celeryContainers[conConf.name] = await dock.createContainer({
-      Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312',
+      Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest',
       name: `dt-${conConf.name}-${branch}`,
       Hostname: `dt-${conConf.name}-${branch}`,
       Env: [
@@ -244,7 +244,7 @@ async function main () {
   // Create Datatracker container
   console.info(`Creating Datatracker docker container... [dt-app-${branch}]`)
   const appContainer = await dock.createContainer({
-    Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312',
+    Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest',
     name: `dt-app-${branch}`,
     Hostname: `dt-app-${branch}`,
     Env: [
diff --git a/dev/diff/cli.js b/dev/diff/cli.js
index 0cf353cc65..461b0c37a0 100644
--- a/dev/diff/cli.js
+++ b/dev/diff/cli.js
@@ -567,7 +567,7 @@ async function main () {
           {
             title: 'Pulling latest Datatracker base docker image...',
             task: async (subctx, subtask) => {
-              const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:py312')
+              const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:latest')
               await new Promise((resolve, reject) => {
                 dock.modem.followProgress(appImagePullStream, (err, res) => err ? reject(err) : resolve(res))
               })
@@ -648,7 +648,7 @@ async function main () {
             title: 'Creating source Datatracker docker container...',
             task: async (subctx, subtask) => {
               containers.appSource = await dock.createContainer({
-                Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312',
+                Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest',
                 name: 'dt-diff-app-source',
                 Tty: true,
                 Hostname: 'appsource',
@@ -664,7 +664,7 @@ async function main () {
             title: 'Creating target Datatracker docker container...',
             task: async (subctx, subtask) => {
               containers.appTarget = await dock.createContainer({
-                Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312',
+                Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest',
                 name: 'dt-diff-app-target',
                 Tty: true,
                 Hostname: 'apptarget',
diff --git a/dev/tests/debug.sh b/dev/tests/debug.sh
index e92e6d9b2a..d87c504bb9 100644
--- a/dev/tests/debug.sh
+++ b/dev/tests/debug.sh
@@ -9,7 +9,7 @@
 # Simply type "exit" + ENTER to exit and shutdown this test environment.
 
 echo "Fetching latest images..."
-docker pull ghcr.io/ietf-tools/datatracker-app-base:py312
+docker pull ghcr.io/ietf-tools/datatracker-app-base:latest
 docker pull ghcr.io/ietf-tools/datatracker-db:latest
 echo "Starting containers..."
 docker compose -f docker-compose.debug.yml -p dtdebug --compatibility up -d
diff --git a/dev/tests/docker-compose.debug.yml b/dev/tests/docker-compose.debug.yml
index 168bbd4e92..8117b92375 100644
--- a/dev/tests/docker-compose.debug.yml
+++ b/dev/tests/docker-compose.debug.yml
@@ -5,7 +5,7 @@ version: '3.8'
 
 services:
     app:
-        image: ghcr.io/ietf-tools/datatracker-app-base:py312
+        image: ghcr.io/ietf-tools/datatracker-app-base:latest
         command: -f /dev/null
         working_dir: /__w/datatracker/datatracker
         entrypoint: tail
diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile
index e3df9bd4b4..fee3833733 100644
--- a/docker/app.Dockerfile
+++ b/docker/app.Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:py312
+FROM ghcr.io/ietf-tools/datatracker-app-base:latest
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/docker/celery.Dockerfile b/docker/celery.Dockerfile
index 279d5c7550..e7c7b9cc3f 100644
--- a/docker/celery.Dockerfile
+++ b/docker/celery.Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:py312
+FROM ghcr.io/ietf-tools/datatracker-app-base:latest
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive

From c4d69d0118a068c873dc066fe9adde829e86f14e Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 5 Sep 2025 17:22:52 -0500
Subject: [PATCH 450/656] feat: links to postorious (#9470)

* feat: links to postorious

* fix: remove redundant divider

* chore: better use of whitespace

* chore: remove what the cat typed in

* chore: more stray removal
---
 ietf/templates/base/menu_user.html | 31 ++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/ietf/templates/base/menu_user.html b/ietf/templates/base/menu_user.html
index 9a0bf56838..fd921638a4 100644
--- a/ietf/templates/base/menu_user.html
+++ b/ietf/templates/base/menu_user.html
@@ -115,6 +115,37 @@
             </a>
         </li>
     {% endif %}
+    <li class="dropend">
+      <a class="dropdown-item dropdown-toggle" href="#">
+        List subscriptions
+      </a>
+      <ul class="dropdown-menu">
+            <li>
+                <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                href="https://mailman3.ietf.org/mailman3/lists/">
+                    IETF Lists
+                </a>
+            </li>
+            <li>
+                <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                href="https://mailman3.irtf.org/mailman3/lists/">
+                IRTF Lists
+                </a>
+            </li>
+            <li>
+                <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                href="https://mailman3.iab.org/mailman3/lists/">
+                    IAB Lists
+                </a>
+            </li>
+            <li>
+                <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                href="https://mailman3.rfc-editor.org/mailman3/lists/">
+                    RFC-Editor Lists
+                </a>
+            </li>
+        </ul>
+    </li>
     {% if user|has_role:"Reviewer" %}
         <li>
             <a class="dropdown-item {% if flavor != 'top' %} text-wrap{% endif %}"

From 32c237f6b3b2572eaffa3922e20d2ab1c119d3a9 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 16 Sep 2025 17:55:07 -0300
Subject: [PATCH 451/656] chore: hide weasyprint internal deprecation warning
 (#9544)

---
 ietf/settings.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ietf/settings.py b/ietf/settings.py
index 753508dc99..d6be1d1e0f 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -34,6 +34,7 @@
 warnings.filterwarnings("ignore", message="datetime.datetime.utcnow\\(\\) is deprecated", module="oic.utils.time_util")
 warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="oic.utils.time_util")
 warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="pytz.tzinfo")
+warnings.filterwarnings("ignore", message="'instantiateVariableFont' is deprecated", module="weasyprint")
 
 
 base_path = pathlib.Path(__file__).resolve().parent

From c71871855769d9c2980cad853cf92a9ec25cb50a Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 16 Sep 2025 15:55:45 -0500
Subject: [PATCH 452/656] fix: normalize 3gpp groups and resolve duplication
 (#9505)

* fix: don't bother the rfc-editor with group type sdo name changes

* fix: normalize 3gpp groups and resolve duplication

* fix: improve guard, update t2

* fix: exclude the task from test coverage

* fix: exclude harder

* fix: tweak the pragma
---
 ietf/group/models.py |   2 +
 ietf/group/tasks.py  | 121 ++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 121 insertions(+), 2 deletions(-)

diff --git a/ietf/group/models.py b/ietf/group/models.py
index 608dcc86b9..2d5e7c4e6f 100644
--- a/ietf/group/models.py
+++ b/ietf/group/models.py
@@ -491,6 +491,8 @@ def notify_rfceditor_of_group_name_change(sender, instance=None, **kwargs):
             current = Group.objects.get(pk=instance.pk)
         except Group.DoesNotExist:
             return
+        if current.type_id == "sdo":
+            return
         addr = settings.RFC_EDITOR_GROUP_NOTIFICATION_EMAIL
         if addr and instance.name != current.name:
             msg = """
diff --git a/ietf/group/tasks.py b/ietf/group/tasks.py
index 693aafb385..ada83e80e2 100644
--- a/ietf/group/tasks.py
+++ b/ietf/group/tasks.py
@@ -9,12 +9,15 @@
 
 from django.conf import settings
 from django.template.loader import render_to_string
+from django.utils import timezone
 
 from ietf.doc.storage_utils import store_file
+from ietf.liaisons.models import LiaisonStatement
 from ietf.utils import log
+from ietf.utils.test_runner import disable_coverage
 
-from .models import Group
-from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles
+from .models import Group, GroupHistory
+from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles, save_group_in_history
 from .views import extract_last_name, roles
 
 
@@ -113,3 +116,117 @@ def generate_wg_summary_files_task():
         store_file("indexes", "1wg-summary.txt", f, allow_overwrite=True)
     with summary_by_acronym_file.open("rb") as f:
         store_file("indexes", "1wg-summary-by-acronym.txt", f, allow_overwrite=True)
+
+@shared_task
+@disable_coverage()
+def run_once_adjust_liaison_groups():  # pragma: no cover
+    log.log("Starting run_once_adjust_liaison_groups")
+    if all(
+        [
+            Group.objects.filter(
+                acronym__in=[
+                    "3gpp-tsg-ct",
+                    "3gpp-tsg-ran-wg1",
+                    "3gpp-tsg-ran-wg4",
+                    "3gpp-tsg-sa",
+                    "3gpp-tsg-sa-wg5",
+                    "3gpp-tsgct",  # duplicates 3gpp-tsg-ct above already
+                    "3gpp-tsgct-ct1",  # will normalize all acronyms to hyphenated form
+                    "3gpp-tsgct-ct3",  # and consistently match the name
+                    "3gpp-tsgct-ct4",  # (particularly use of WG)
+                    "3gpp-tsgran",
+                    "3gpp-tsgran-ran2",
+                    "3gpp-tsgsa",  # duplicates 3gpp-tsg-sa above
+                    "3gpp-tsgsa-sa2",  # will normalize
+                    "3gpp-tsgsa-sa3",
+                    "3gpp-tsgsa-sa4",
+                    "3gpp-tsgt-wg2",
+                ]
+            ).count()
+            == 16,
+            not Group.objects.filter(
+                acronym__in=[
+                    "3gpp-tsg-ran-wg3",
+                    "3gpp-tsg-ct-wg1",
+                    "3gpp-tsg-ct-wg3",
+                    "3gpp-tsg-ct-wg4",
+                    "3gpp-tsg-ran",
+                    "3gpp-tsg-ran-wg2",
+                    "3gpp-tsg-sa-wg2",
+                    "3gpp-tsg-sa-wg3",
+                    "3gpp-tsg-sa-wg4",
+                    "3gpp-tsg-t-wg2",
+                ]
+            ).exists(),
+            Group.objects.filter(acronym="o3gpptsgran3").exists(),
+            not LiaisonStatement.objects.filter(
+                to_groups__acronym__in=["3gpp-tsgct", "3gpp-tsgsa"]
+            ).exists(),
+            not LiaisonStatement.objects.filter(
+                from_groups__acronym="3gpp-tsgct"
+            ).exists(),
+            LiaisonStatement.objects.filter(from_groups__acronym="3gpp-tsgsa").count()
+            == 1,
+            LiaisonStatement.objects.get(from_groups__acronym="3gpp-tsgsa").pk == 1448,
+        ]
+    ):
+        for old_acronym, new_acronym, new_name in (
+            ("o3gpptsgran3", "3gpp-tsg-ran-wg3", "3GPP TSG RAN WG3"),
+            ("3gpp-tsgct-ct1", "3gpp-tsg-ct-wg1", "3GPP TSG CT WG1"),
+            ("3gpp-tsgct-ct3", "3gpp-tsg-ct-wg3", "3GPP TSG CT WG3"),
+            ("3gpp-tsgct-ct4", "3gpp-tsg-ct-wg4", "3GPP TSG CT WG4"),
+            ("3gpp-tsgran", "3gpp-tsg-ran", "3GPP TSG RAN"),
+            ("3gpp-tsgran-ran2", "3gpp-tsg-ran-wg2", "3GPP TSG RAN WG2"),
+            ("3gpp-tsgsa-sa2", "3gpp-tsg-sa-wg2", "3GPP TSG SA WG2"),
+            ("3gpp-tsgsa-sa3", "3gpp-tsg-sa-wg3", "3GPP TSG SA WG3"),
+            ("3gpp-tsgsa-sa4", "3gpp-tsg-sa-wg4", "3GPP TSG SA WG4"),
+            ("3gpp-tsgt-wg2", "3gpp-tsg-t-wg2", "3GPP TSG T WG2"),
+        ):
+            group = Group.objects.get(acronym=old_acronym)
+            save_group_in_history(group)
+            group.time = timezone.now()
+            group.acronym = new_acronym
+            group.name = new_name
+            if old_acronym.startswith("3gpp-tsgct-"):
+                group.parent = Group.objects.get(acronym="3gpp-tsg-ct")
+            elif old_acronym.startswith("3gpp-tsgsa-"):
+                group.parent = Group.objects.get(acronym="3gpp-tsg-sa")
+            group.save()
+            group.groupevent_set.create(
+                time=group.time,
+                by_id=1,  # (System)
+                type="info_changed",
+                desc=f"acronym changed from {old_acronym} to {new_acronym}, name set to {new_name}",
+            )
+
+        for acronym, new_name in (("3gpp-tsg-ct", "3GPP TSG CT"),):
+            group = Group.objects.get(acronym=acronym)
+            save_group_in_history(group)
+            group.time = timezone.now()
+            group.name = new_name
+            group.save()
+            group.groupevent_set.create(
+                time=group.time,
+                by_id=1,  # (System)
+                type="info_changed",
+                desc=f"name set to {new_name}",
+            )
+
+        ls = LiaisonStatement.objects.get(pk=1448)
+        ls.from_groups.remove(Group.objects.get(acronym="3gpp-tsgsa"))
+        ls.from_groups.add(Group.objects.get(acronym="3gpp-tsg-sa"))
+
+        # Rewriting history to effectively merge the histories of the duplicate groups
+        GroupHistory.objects.filter(parent__acronym="3gpp-tsgsa").update(
+            parent=Group.objects.get(acronym="3gpp-tsg-sa")
+        )
+        GroupHistory.objects.filter(parent__acronym="3gpp-tsgct").update(
+            parent=Group.objects.get(acronym="3gpp-tsg-ct")
+        )
+
+        deleted = Group.objects.filter(
+            acronym__in=["3gpp-tsgsa", "3gpp-tsgct"]
+        ).delete()
+        log.log(f"Deleted Groups: {deleted}")
+    else:
+        log.log("* Refusing to continue as preconditions have changed")

From 0a1705193dfde6695921191540049b88d91d9ec9 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 17 Sep 2025 12:45:32 -0500
Subject: [PATCH 453/656] fix: update draft-stream-ietf state descriptions
 (#9543)

---
 .../0026_change_wg_state_descriptions.py      | 117 ++++++++++++++++++
 1 file changed, 117 insertions(+)
 create mode 100644 ietf/doc/migrations/0026_change_wg_state_descriptions.py

diff --git a/ietf/doc/migrations/0026_change_wg_state_descriptions.py b/ietf/doc/migrations/0026_change_wg_state_descriptions.py
new file mode 100644
index 0000000000..b02b12c97e
--- /dev/null
+++ b/ietf/doc/migrations/0026_change_wg_state_descriptions.py
@@ -0,0 +1,117 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations
+
+def forward(apps, schema_editor):
+    State = apps.get_model("doc","State")
+    for name, desc in [
+        ("WG Document","The document has been adopted by the Working Group (WG) and is under development.  A document can only be adopted by one WG at a time.  However, a document may be transferred between WGs."),
+        ("Parked WG Document","The Working Group (WG) document is in a temporary state where it will not be actively developed.  The reason for the pause is explained via a datatracker comments section."),
+        ("Dead WG Document","The Working Group (WG) document has been abandoned by the WG.  No further development is planned in this WG.  A decision to resume work on this document and move it out of this state is possible."),
+        ("In WG Last Call","The Working Group (WG) document is currently subject to an active WG Last Call (WGLC) review per Section 7.4 of RFC2418."),
+        ("Waiting for Implementation","The progression of this Working Group (WG) document towards publication is paused as it awaits implementation. The process governing the approach to implementations is WG-specific."),
+        ("Held by WG","Held by Working Group (WG) chairs for administrative reasons.  See document history for details."),
+        ("Waiting for WG Chair Go-Ahead","The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed"),
+        ("WG Consensus: Waiting for Write-Up","The Working Group (WG) document has consensus to proceed to publication.  However, the document is waiting for a document shepherd write-up per RFC4858."),
+        ("Submitted to IESG for Publication","The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication.  See the “IESG State” or “RFC Editor State” for further details on the state of the document."),
+        ("Candidate for WG Adoption","The individual submission document has been marked by the Working Group (WG) chairs as a candidate for adoption by the WG, but no adoption call has been started."),
+        ("Call For Adoption By WG Issued","A call for adoption of the individual submission document has been issued by the Working Group (WG) chairs.  This call is still running but the WG has not yet reached consensus for adoption."),
+        ("Adopted by a WG","The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted."),
+        ("Adopted for WG Info Only","The document is adopted by the Working Group (WG) for its internal use.  The WG has decided that it will not pursue publication of it as an RFC."),
+    ]:
+        State.objects.filter(name=name).update(desc=desc)
+
+def reverse(apps, schema_editor):
+    State = apps.get_model("doc","State")
+    for name, desc in [
+       ("WG Document","""<a href="https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.4" target="_blank">4.2.4. WG Document</a>
+
+ The "WG Document" state describes an I-D that has been adopted by an IETF WG and is being actively developed.
+
+ A WG Chair may transition an I-D into the "WG Document" state at any time as long as the I-D is not being considered or developed in any other WG.
+
+ Alternatively, WG Chairs may rely upon new functionality to be added to the Datatracker to automatically move version-00 drafts into the "WG Document" state as described in Section 4.1.
+
+ Under normal conditions, it should not be possible for an I-D to be in the "WG Document" state in more than one WG at a time. This said, I-Ds may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs."""),
+        ("Parked WG Document","""<a href="https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.5" target="_blank">4.2.5. Parked WG Document</a>
+
+ A "Parked WG Document" is an I-D that has lost its author or editor, is waiting for another document to be written or for a review to be completed, or cannot be progressed by the working group for some other reason.
+
+ Some of the annotation tags described in Section 4.3 may be used in conjunction with this state to indicate why an I-D has been parked, and/or what may need to happen for the I-D to be un-parked.
+
+ Parking a WG draft will not prevent it from expiring; however, this state can be used to indicate why the I-D has stopped progressing in the WG.
+
+ A "Parked WG Document" that is not expired may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs."""),
+        ("Dead WG Document","""<a href="https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.6" target="_blank">4.2.6. Dead WG Document</a>
+
+ A "Dead WG Document" is an I-D that has been abandoned. Note that 'Dead' is not always a final state for a WG I-D. If consensus is subsequently achieved, a "Dead WG Document" may be resurrected. A "Dead WG Document" that is not resurrected will eventually expire.
+
+ Note that an I-D that is declared to be "Dead" in one WG and that is not expired may be transferred to a non-dead state in another WG with the consent of the WG Chairs and the responsible ADs."""),
+        ("In WG Last Call","""<a href="https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.7" target="_blank">4.2.7. In WG Last Call</a>
+
+ A document "In WG Last Call" is an I-D for which a WG Last Call (WGLC) has been issued and is in progress.
+
+ Note that conducting a WGLC is an optional part of the IETF WG process, per Section 7.4 of RFC 2418 [RFC2418].
+
+ If a WG Chair decides to conduct a WGLC on an I-D, the "In WG Last Call" state can be used to track the progress of the WGLC. The Chair may configure the Datatracker to send a WGLC message to one or more mailing lists when the Chair moves the I-D into this state. The WG Chair may also be able to select a different set of mailing lists for a different document undergoing a WGLC; some documents may deserve coordination with other WGs.
+
+ A WG I-D in this state should remain "In WG Last Call" until the WG Chair moves it to another state. The WG Chair may configure the Datatracker to send an e-mail after a specified period of time to remind or 'nudge' the Chair to conclude the WGLC and to determine the next state for the document.
+
+ It is possible for one WGLC to lead into another WGLC for the same document. For example, an I-D that completed a WGLC as an "Informational" document may need another WGLC if a decision is taken to convert the I-D into a Standards Track document."""),
+        ("Waiting for Implementation","""In some areas, it can be desirable to wait for multiple interoperable implementations before progressing a draft to be an RFC, and in some WGs this is required.  This state should be entered after WG Last Call has completed."""),
+        ("Held by WG","""Held by WG, see document history for details."""),
+        ("Waiting for WG Chair Go-Ahead","""<a href="https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.8" target="_blank">4.2.8. Waiting for WG Chair Go-Ahead</a>
+
+ A WG Chair may wish to place an I-D that receives a lot of comments during a WGLC into the "Waiting for WG Chair Go-Ahead" state. This state describes an I-D that has undergone a WGLC; however, the Chair is not yet ready to call consensus on the document.
+
+ If comments from the WGLC need to be responded to, or a revision to the I-D is needed, the Chair may place an I-D into this state until all of the WGLC comments are adequately addressed and the (possibly revised) document is in the I-D repository."""),
+        ("WG Consensus: Waiting for Write-Up","""<a href="https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.9" target="_blank">4.2.9. WG Consensus: Waiting for Writeup</a>
+
+ A document in the "WG Consensus: Waiting for Writeup" state has essentially completed its development within the working group, and is nearly ready to be sent to the IESG for publication. The last thing to be done is the preparation of a protocol writeup by a Document Shepherd. The IESG requires that a document shepherd writeup be completed before publication of the I-D is requested. The IETF document shepherding process and the role of a WG Document Shepherd is described in RFC 4858 [RFC4858]
+
+ A WG Chair may call consensus on an I-D without a formal WGLC and transition an I-D that was in the "WG Document" state directly into this state.
+
+ The name of this state includes the words "Waiting for Writeup" because a good document shepherd writeup takes time to prepare."""),
+        ("Submitted to IESG for Publication","""<a href="https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.10" target="_blank">4.2.10. Submitted to IESG for Publication</a>
+
+ This state describes a WG document that has been submitted to the IESG for publication and that has not been sent back to the working group for revision.
+
+ An I-D in this state may be under review by the IESG, it may have been approved and be in the RFC Editor's queue, or it may have been published as an RFC. Other possibilities exist too. The document may be "Dead" (in the IESG state machine) or in a "Do Not Publish" state."""),
+        ("Candidate for WG Adoption","""The document has been marked as a candidate for WG adoption by the WG Chair.  This state can be used before a call for adoption is issued (and the document is put in the "Call For Adoption By WG Issued" state), to indicate that the document is in the queue for a call for adoption, even if none has been issued yet."""),
+        ("Call For Adoption By WG Issued","""<a href="https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.1" target="_blank">4.2.1. Call for Adoption by WG Issued</a>
+
+ The "Call for Adoption by WG Issued" state should be used to indicate when an I-D is being considered for adoption by an IETF WG. An I-D that is in this state is actively being considered for adoption and has not yet achieved consensus, preference, or selection in the WG.
+
+ This state may be used to describe an I-D that someone has asked a WG to consider for adoption, if the WG Chair has agreed with the request. This state may also be used to identify an I-D that a WG Chair asked an author to write specifically for consideration as a candidate WG item [WGDTSPEC], and/or an I-D that is listed as a 'candidate draft' in the WG's charter.
+
+ Under normal conditions, it should not be possible for an I-D to be in the "Call for Adoption by WG Issued" state in more than one working group at the same time. This said, it is not uncommon for authors to "shop" their I-Ds to more than one WG at a time, with the hope of getting their documents adopted somewhere.
+
+ After this state is implemented in the Datatracker, an I-D that is in the "Call for Adoption by WG Issued" state will not be able to be "shopped" to any other WG without the consent of the WG Chairs and the responsible ADs impacted by the shopping.
+
+ Note that Figure 1 includes an arc leading from this state to outside of the WG state machine. This illustrates that some I-Ds that are considered do not get adopted as WG drafts. An I-D that is not adopted as a WG draft will transition out of the WG state machine and revert back to having no stream-specific state; however, the status change history log of the I-D will record that the I-D was previously in the "Call for Adoption by WG Issued" state."""),
+        ("Adopted by a WG","""<a href="https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.2" target="_blank">4.2.2. Adopted by a WG</a>
+
+ The "Adopted by a WG" state describes an individual submission I-D that an IETF WG has agreed to adopt as one of its WG drafts.
+
+ WG Chairs who use this state will be able to clearly indicate when their WGs adopt individual submission I-Ds. This will facilitate the Datatracker's ability to correctly capture "Replaces" information for WG drafts and correct "Replaced by" information for individual submission I-Ds that have been replaced by WG drafts.
+
+ This state is needed because the Datatracker uses the filename of an I-D as a key to search its database for status information about the I-D, and because the filename of a WG I-D is supposed to be different from the filename of an individual submission I-D. The filename of an individual submission I-D will typically be formatted as 'draft-author-wgname-topic-nn'.
+
+ The filename of a WG document is supposed to be formatted as 'draft- ietf-wgname-topic-nn'.
+
+ An individual I-D that is adopted by a WG may take weeks or months to be resubmitted by the author as a new (version-00) WG draft. If the "Adopted by a WG" state is not used, the Datatracker has no way to determine that an I-D has been adopted until a new version of the I-D is submitted to the WG by the author and until the I-D is approved for posting by a WG Chair."""),
+        ("Adopted for WG Info Only","""<a href="https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.3" target="_blank">4.2.3. Adopted for WG Info Only</a>
+
+ The "Adopted for WG Info Only" state describes a document that contains useful information for the WG that adopted it, but the document is not intended to be published as an RFC. The WG will not actively develop the contents of the I-D or progress it for publication as an RFC. The only purpose of the I-D is to provide information for internal use by the WG."""),
+    ]:
+        State.objects.filter(name=name).update(desc=desc)
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("doc", "0025_storedobject_storedobject_unique_name_per_store"),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse)
+    ]

From 327447f91fa21ef7620d958b5f8fc1f00d4f85a5 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 17 Sep 2025 13:42:09 -0500
Subject: [PATCH 454/656] feat: iesg dashboard of wg documents (#9363)

* feat: iesg dashboard of wg documents (#8999)

* fix: removed template html cruft

* fix: avoid triggering a Ghostery false positive

* fix: remove related-id, milestone, and last meeting columns

* fix: make wgs with no docs show in last table

* fix: remove wg w/wo docs columns from first three thables

* fix: Make table names closer to original request

* chore: ruff format ietf.iesg.utils

* feat: refactor, test, cleanup

* chore: added comment about the test wg acronyms

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 ietf/iesg/tests.py                      | 1583 +++++++++++++++++++++++
 ietf/iesg/urls.py                       |    1 +
 ietf/iesg/utils.py                      |  296 ++++-
 ietf/iesg/views.py                      |   12 +-
 ietf/templates/iesg/working_groups.html |  159 +++
 5 files changed, 2028 insertions(+), 23 deletions(-)
 create mode 100644 ietf/templates/iesg/working_groups.html

diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py
index 746ea3f56f..f3778d1ded 100644
--- a/ietf/iesg/tests.py
+++ b/ietf/iesg/tests.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 
+from collections import Counter
 import datetime
 import io
 import tarfile
@@ -24,7 +25,9 @@
 from ietf.group.models import Group, GroupMilestone, Role
 from ietf.iesg.agenda import get_agenda_date, agenda_data, fill_in_agenda_administrivia, agenda_sections
 from ietf.iesg.models import TelechatDate, TelechatAgendaContent
+from ietf.iesg.utils import get_wg_dashboard_info
 from ietf.name.models import StreamName, TelechatAgendaSectionName
+from ietf.person.factories import PersonFactory
 from ietf.person.models import Person
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent
 from ietf.iesg.factories import IESGMgmtItemFactory, TelechatAgendaContentFactory
@@ -182,6 +185,1586 @@ def test_ietf_activity(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
 
+    def test_working_groups(self):
+        # Clean away the wasted built-for-every-test noise
+        Group.objects.filter(type__in=["wg", "area"]).delete()
+
+        (
+            area_summary,
+            area_totals,
+            ad_summary,
+            noad_summary,
+            ad_totals,
+            noad_totals,
+            totals,
+            wg_summary,
+        ) = get_wg_dashboard_info()
+        self.assertEqual(area_summary, [])
+        self.assertEqual(
+            area_totals, {"group_count": 0, "doc_count": 0, "page_count": 0}
+        )
+        self.assertEqual(ad_summary, [])
+        self.assertEqual(noad_summary, [])
+        self.assertEqual(
+            ad_totals,
+            {
+                "ad_group_count": 0,
+                "doc_group_count": 0,
+                "doc_count": 0,
+                "page_count": 0,
+            },
+        )
+        self.assertEqual(
+            noad_totals,
+            {
+                "ad_group_count": 0,
+                "doc_group_count": 0,
+                "doc_count": 0,
+                "page_count": 0,
+            },
+        )
+        self.assertEqual(
+            totals,
+            {
+                "group_count": 0,
+                "doc_count": 0,
+                "page_count": 0,
+                "groups_with_docs_count": 0,
+            },
+        )
+        self.assertEqual(wg_summary, [])
+
+        # Construct Areas with WGs similar in shape to a real moment of the IETF
+
+        # Note that this test construciton uses the first letter of the wg acronyms
+        # for convenience to switch on whether groups have documents with assigned ADs.
+        # (Search for ` if wg_acronym[0] > "g"`)
+        # There's no other significance to the names of the area directors or the
+        # acronyms of the areas and groups other than being distinct. Taking the
+        # values from sets of similar things hopefully helps with debugging the tests.
+
+        areas = {}
+        for area_acronym in ["red", "orange", "yellow", "green", "blue", "violet"]:
+            areas[area_acronym] = GroupFactory(type_id="area", acronym=area_acronym)
+        for ad, area, wgs in [
+            ("Alpha", "red", ["bassoon"]),
+            ("Bravo", "orange", ["celesta"]),
+            ("Charlie", "orange", ["clarinet", "cymbals"]),
+            ("Delta", "yellow", ["flute"]),
+            ("Echo", "yellow", ["glockenspiel"]),
+            ("Foxtrot", "green", ["gong", "guitar"]),
+            ("Golf", "green", ["harp"]),
+            ("Hotel", "blue", ["harpsichord"]),
+            ("Indigo", "blue", ["oboe", "organ"]),
+            ("Juliet", "violet", ["piano"]),
+            ("Kilo", "violet", ["piccolo"]),
+            ("Lima", "violet", ["saxophone", "tambourine"]),
+        ]:
+            p = Person.objects.filter(name=ad).first() or PersonFactory(name=ad)
+            RoleFactory(group=areas[area], person=p, name_id="ad")
+            for wg in wgs:
+                g = GroupFactory(acronym=wg, type_id="wg", parent=areas[area])
+                RoleFactory(group=g, person=p, name_id="ad")
+
+        # Some ADs have out of area groups
+        g = GroupFactory(acronym="timpani", parent=areas["orange"])
+        RoleFactory(group=g, person=Person.objects.get(name="Juliet"), name_id="ad")
+
+        (
+            area_summary,
+            area_totals,
+            ad_summary,
+            noad_summary,
+            ad_totals,
+            noad_totals,
+            totals,
+            wg_summary,
+        ) = get_wg_dashboard_info()
+
+        self.assertEqual(
+            area_summary,
+            [
+                {
+                    "area": "red",
+                    "groups_in_area": 1,
+                    "groups_with_docs": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "area": "orange",
+                    "groups_in_area": 4,
+                    "groups_with_docs": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "area": "yellow",
+                    "groups_in_area": 2,
+                    "groups_with_docs": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "area": "green",
+                    "groups_in_area": 3,
+                    "groups_with_docs": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "area": "blue",
+                    "groups_in_area": 3,
+                    "groups_with_docs": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "area": "violet",
+                    "groups_in_area": 4,
+                    "groups_with_docs": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+            ],
+        )
+        self.assertEqual(
+            area_totals, {"group_count": 0, "doc_count": 0, "page_count": 0}
+        )
+        self.assertEqual(
+            ad_summary,
+            [
+                {
+                    "ad": "Alpha",
+                    "area": "red",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Bravo",
+                    "area": "orange",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Charlie",
+                    "area": "orange",
+                    "ad_group_count": 2,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Delta",
+                    "area": "yellow",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Echo",
+                    "area": "yellow",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Foxtrot",
+                    "area": "green",
+                    "ad_group_count": 2,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Golf",
+                    "area": "green",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Hotel",
+                    "area": "blue",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Indigo",
+                    "area": "blue",
+                    "ad_group_count": 2,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Juliet",
+                    "area": "orange",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Juliet",
+                    "area": "violet",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Kilo",
+                    "area": "violet",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Lima",
+                    "area": "violet",
+                    "ad_group_count": 2,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+            ],
+        )
+        self.assertEqual(
+            noad_summary,
+            [
+                {
+                    "ad": "Alpha",
+                    "area": "red",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Bravo",
+                    "area": "orange",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Charlie",
+                    "area": "orange",
+                    "ad_group_count": 2,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Delta",
+                    "area": "yellow",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Echo",
+                    "area": "yellow",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Foxtrot",
+                    "area": "green",
+                    "ad_group_count": 2,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Golf",
+                    "area": "green",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Hotel",
+                    "area": "blue",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Indigo",
+                    "area": "blue",
+                    "ad_group_count": 2,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Juliet",
+                    "area": "orange",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Juliet",
+                    "area": "violet",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Kilo",
+                    "area": "violet",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+                {
+                    "ad": "Lima",
+                    "area": "violet",
+                    "ad_group_count": 2,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0,
+                    "doc_percent": 0,
+                    "page_percent": 0,
+                },
+            ],
+        )
+        self.assertEqual(
+            ad_totals,
+            {
+                "ad_group_count": 17,
+                "doc_group_count": 0,
+                "doc_count": 0,
+                "page_count": 0,
+            },
+        )
+        self.assertEqual(
+            noad_totals,
+            {
+                "ad_group_count": 17,
+                "doc_group_count": 0,
+                "doc_count": 0,
+                "page_count": 0,
+            },
+        )
+        self.assertEqual(
+            totals,
+            {
+                "group_count": 17,
+                "doc_count": 0,
+                "page_count": 0,
+                "groups_with_docs_count": 0,
+            },
+        )
+        self.assertEqual(
+            wg_summary,
+            [
+                {
+                    "wg": "bassoon",
+                    "area": "red",
+                    "ad": "Alpha",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "celesta",
+                    "area": "orange",
+                    "ad": "Bravo",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "clarinet",
+                    "area": "orange",
+                    "ad": "Charlie",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "cymbals",
+                    "area": "orange",
+                    "ad": "Charlie",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "flute",
+                    "area": "yellow",
+                    "ad": "Delta",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "glockenspiel",
+                    "area": "yellow",
+                    "ad": "Echo",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "gong",
+                    "area": "green",
+                    "ad": "Foxtrot",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "guitar",
+                    "area": "green",
+                    "ad": "Foxtrot",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "harp",
+                    "area": "green",
+                    "ad": "Golf",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "harpsichord",
+                    "area": "blue",
+                    "ad": "Hotel",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "oboe",
+                    "area": "blue",
+                    "ad": "Indigo",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "organ",
+                    "area": "blue",
+                    "ad": "Indigo",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "piano",
+                    "area": "violet",
+                    "ad": "Juliet",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "piccolo",
+                    "area": "violet",
+                    "ad": "Kilo",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "saxophone",
+                    "area": "violet",
+                    "ad": "Lima",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "tambourine",
+                    "area": "violet",
+                    "ad": "Lima",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "timpani",
+                    "area": "orange",
+                    "ad": "Juliet",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+            ],
+        )
+
+        # As seen above, all doc and page counts are currently 0
+
+        # We'll give a group a document but not assign it to its AD
+        WgDraftFactory(
+            group=Group.objects.get(acronym="saxophone"), pages=len("saxophone")
+        )
+        (
+            area_summary,
+            area_totals,
+            ad_summary,
+            noad_summary,
+            ad_totals,
+            noad_totals,
+            totals,
+            wg_summary,
+        ) = get_wg_dashboard_info()
+        count_violet_dicts = 0
+        for d in area_summary:
+            if d["area"] == "violet":
+                count_violet_dicts += 1
+                self.assertEqual(d["groups_with_docs"], 1)
+                self.assertEqual(d["doc_count"], 1)
+                self.assertEqual(d["page_count"], 9)
+                self.assertEqual(d["group_percent"], 100.0)
+                self.assertEqual(d["doc_percent"], 100.0)
+                self.assertEqual(d["page_percent"], 100.0)
+            else:
+                self.assertEqual(d["groups_with_docs"], 0)
+                self.assertEqual(d["doc_count"], 0)
+                self.assertEqual(d["page_count"], 0)
+                self.assertEqual(d["group_percent"], 0)
+                self.assertEqual(d["doc_percent"], 0)
+                self.assertEqual(d["page_percent"], 0)
+        self.assertEqual(count_violet_dicts, 1)
+
+        self.assertEqual(
+            area_totals, {"group_count": 1, "doc_count": 1, "page_count": 9}
+        )
+
+        # No AD has this document, even though it's in Lima's group
+        count_lima_dicts = 0
+        for d in ad_summary:
+            if d["ad"] == "Lima":
+                count_lima_dicts += 1
+            self.assertEqual(d["doc_group_count"], 0)
+            self.assertEqual(d["doc_count"], 0)
+            self.assertEqual(d["page_count"], 0)
+            self.assertEqual(d["group_percent"], 0)
+            self.assertEqual(d["doc_percent"], 0)
+            self.assertEqual(d["page_percent"], 0)
+        self.assertEqual(count_lima_dicts, 1)
+
+        # It's in Lima's group, so normally it will eventually land on Lima
+        count_lima_dicts = 0
+        for d in noad_summary:
+            if d["ad"] == "Lima":
+                count_lima_dicts += 1
+                self.assertEqual(d["doc_group_count"], 1)
+                self.assertEqual(d["doc_count"], 1)
+                self.assertEqual(d["page_count"], 9)
+                self.assertEqual(d["group_percent"], 100.0)
+                self.assertEqual(d["doc_percent"], 100.0)
+                self.assertEqual(d["page_percent"], 100.0)
+            else:
+                self.assertEqual(d["doc_group_count"], 0)
+                self.assertEqual(d["doc_count"], 0)
+                self.assertEqual(d["page_count"], 0)
+                self.assertEqual(d["group_percent"], 0)
+                self.assertEqual(d["doc_percent"], 0)
+                self.assertEqual(d["page_percent"], 0)
+        self.assertEqual(count_lima_dicts, 1)
+
+        self.assertEqual(
+            ad_totals,
+            {
+                "ad_group_count": 17,
+                "doc_group_count": 0,
+                "doc_count": 0,
+                "page_count": 0,
+            },
+        )
+        self.assertEqual(
+            noad_totals,
+            {
+                "ad_group_count": 17,
+                "doc_group_count": 1,
+                "doc_count": 1,
+                "page_count": 9,
+            },
+        )
+        self.assertEqual(
+            totals,
+            {
+                "group_count": 17,
+                "doc_count": 1,
+                "page_count": 9,
+                "groups_with_docs_count": 1,
+            },
+        )
+
+        count_sax_dicts = 0
+        for d in wg_summary:
+            if d["wg"] == "saxophone":
+                count_sax_dicts += 1
+                self.assertEqual(d["doc_count"], 1)
+                self.assertEqual(d["page_count"], 9)
+            else:
+                self.assertEqual(d["doc_count"], 0)
+                self.assertEqual(d["page_count"], 0)
+        self.assertEqual(count_sax_dicts, 1)
+
+        # Assign that doc to Lima
+        self.assertEqual(Document.objects.count(), 1)
+        Document.objects.all().update(ad=Person.objects.get(name="Lima"))
+        (
+            area_summary,
+            area_totals,
+            ad_summary,
+            noad_summary,
+            ad_totals,
+            noad_totals,
+            totals,
+            wg_summary,
+        ) = get_wg_dashboard_info()
+        count_violet_dicts = 0
+        for d in area_summary:
+            if d["area"] == "violet":
+                count_violet_dicts += 1
+                self.assertEqual(d["groups_with_docs"], 1)
+                self.assertEqual(d["doc_count"], 1)
+                self.assertEqual(d["page_count"], 9)
+                self.assertEqual(d["group_percent"], 100.0)
+                self.assertEqual(d["doc_percent"], 100.0)
+                self.assertEqual(d["page_percent"], 100.0)
+            else:
+                self.assertEqual(d["groups_with_docs"], 0)
+                self.assertEqual(d["doc_count"], 0)
+                self.assertEqual(d["page_count"], 0)
+                self.assertEqual(d["group_percent"], 0)
+                self.assertEqual(d["doc_percent"], 0)
+                self.assertEqual(d["page_percent"], 0)
+        self.assertEqual(count_violet_dicts, 1)
+
+        self.assertEqual(
+            area_totals, {"group_count": 1, "doc_count": 1, "page_count": 9}
+        )
+
+        # This time it will show up as a doc assigned to Lima
+        count_lima_dicts = 0
+        for d in ad_summary:
+            if d["ad"] == "Lima":
+                count_lima_dicts += 1
+                self.assertEqual(d["doc_group_count"], 1)
+                self.assertEqual(d["doc_count"], 1)
+                self.assertEqual(d["page_count"], 9)
+                self.assertEqual(d["group_percent"], 100.0)
+                self.assertEqual(d["doc_percent"], 100.0)
+                self.assertEqual(d["page_percent"], 100.0)
+            else:
+                self.assertEqual(d["doc_group_count"], 0)
+                self.assertEqual(d["doc_count"], 0)
+                self.assertEqual(d["page_count"], 0)
+                self.assertEqual(d["group_percent"], 0)
+                self.assertEqual(d["doc_percent"], 0)
+                self.assertEqual(d["page_percent"], 0)
+        self.assertEqual(count_lima_dicts, 1)
+
+        # and there will be no noad documents
+        count_lima_dicts = 0
+        for d in noad_summary:
+            if d["ad"] == "Lima":
+                count_lima_dicts += 1
+            self.assertEqual(d["doc_group_count"], 0)
+            self.assertEqual(d["doc_count"], 0)
+            self.assertEqual(d["page_count"], 0)
+            self.assertEqual(d["group_percent"], 0)
+            self.assertEqual(d["doc_percent"], 0)
+            self.assertEqual(d["page_percent"], 0)
+        self.assertEqual(count_lima_dicts, 1)
+
+        self.assertEqual(
+            ad_totals,
+            {
+                "ad_group_count": 17,
+                "doc_group_count": 1,
+                "doc_count": 1,
+                "page_count": 9,
+            },
+        )
+        self.assertEqual(
+            noad_totals,
+            {
+                "ad_group_count": 17,
+                "doc_group_count": 0,
+                "doc_count": 0,
+                "page_count": 0,
+            },
+        )
+        self.assertEqual(
+            totals,
+            {
+                "group_count": 17,
+                "doc_count": 1,
+                "page_count": 9,
+                "groups_with_docs_count": 1,
+            },
+        )
+
+        count_sax_dicts = 0
+        for d in wg_summary:
+            if d["wg"] == "saxophone":
+                count_sax_dicts += 1
+                self.assertEqual(d["doc_count"], 1)
+                self.assertEqual(d["page_count"], 9)
+            else:
+                self.assertEqual(d["doc_count"], 0)
+                self.assertEqual(d["page_count"], 0)
+        self.assertEqual(count_sax_dicts, 1)
+
+        # Now give Lima a document in a group that's not in their area:
+        WgDraftFactory(
+            group=Group.objects.get(acronym="gong"),
+            pages=len("gong"),
+            ad=Person.objects.get(name="Lima"),
+        )
+        (
+            area_summary,
+            area_totals,
+            ad_summary,
+            noad_summary,
+            ad_totals,
+            noad_totals,
+            totals,
+            wg_summary,
+        ) = get_wg_dashboard_info()
+        seen_dicts = Counter([d["area"] for d in area_summary])
+        for d in areas:
+            self.assertEqual(seen_dicts[area], 1 if area in ["violet", "green"] else 0)
+        for d in area_summary:
+            if d["area"] in ["violet", "green"]:
+                self.assertEqual(d["doc_count"], 1)
+                self.assertEqual(d["page_count"], 9 if d["area"] == "violet" else 4)
+                self.assertEqual(d["group_percent"], 50)
+                self.assertEqual(d["doc_percent"], 50)
+                self.assertEqual(
+                    d["page_percent"],
+                    100 * 9 / 13 if d["area"] == "violet" else 100 * 4 / 13,
+                )
+            else:
+                self.assertEqual(d["doc_count"], 0)
+                self.assertEqual(d["page_count"], 0)
+                self.assertEqual(d["group_percent"], 0)
+                self.assertEqual(d["doc_percent"], 0)
+                self.assertEqual(d["page_percent"], 0)
+
+        self.assertEqual(
+            area_totals, {"group_count": 2, "doc_count": 2, "page_count": 13}
+        )
+
+        for d in ad_summary:
+            if d["ad"] == "Lima":
+                self.assertEqual(d["doc_group_count"], 1)
+                self.assertEqual(d["doc_count"], 1)
+                self.assertEqual(d["page_count"], 9 if d["area"] == "violet" else 4)
+                self.assertEqual(d["group_percent"], 50)
+                self.assertEqual(d["doc_percent"], 50)
+                self.assertEqual(
+                    d["page_percent"],
+                    100 * 9 / 13 if d["area"] == "violet" else 100 * 4 / 13,
+                )
+            else:
+                self.assertEqual(d["doc_group_count"], 0)
+                self.assertEqual(
+                    d["doc_count"], 0
+                )  # Note in particular this is 0 for Foxtrot
+                self.assertEqual(d["page_count"], 0)
+                self.assertEqual(d["group_percent"], 0)
+                self.assertEqual(d["doc_percent"], 0)
+                self.assertEqual(d["page_percent"], 0)
+
+        for d in wg_summary:
+            if d["wg"] == "gong":
+                # Lima's doc in gong above counts at the dict for gong even though the ad reported there is Foxtrot.
+                self.assertEqual(
+                    d,
+                    {
+                        "wg": "gong",
+                        "area": "green",
+                        "ad": "Foxtrot",
+                        "doc_count": 1,
+                        "page_count": 4,
+                        "rfc_count": 0,
+                        "recent_rfc_count": 0,
+                    },
+                )
+            elif d["ad"] == "Lima":
+                self.assertEqual(
+                    d["area"], "violet"
+                )  # The out of area assignment is not reflected in the wg_summary at all.
+
+        # Now pile on a lot of documents
+        for wg_acronym in [
+            "bassoon",
+            "celesta",
+            "clarinet",
+            "cymbals",
+            "flute",
+            "glockenspiel",
+            "gong",
+            "guitar",
+            "harp",
+            "harpsichord",
+            "oboe",
+            "organ",
+            "piano",
+            "piccolo",
+            "saxophone",
+            "tambourine",
+            "timpani",
+        ]:
+            if wg_acronym in ["bassoon", "celesta"]:
+                continue  # Those WGs have no docs
+            # The rest have a doc that's not assigned to any ad
+            WgDraftFactory(
+                group=Group.objects.get(acronym=wg_acronym), pages=len(wg_acronym)
+            )
+            if wg_acronym[0] > "g":
+                # Some have a doc assigned to the responsible ad
+                WgDraftFactory(
+                    group=Group.objects.get(acronym=wg_acronym),
+                    pages=len(wg_acronym),
+                    ad=Role.objects.get(name_id="ad", group__acronym=wg_acronym).person,
+                )
+        # The other AD for an area might be covering a doc
+        WgDraftFactory(
+            group=Group.objects.get(acronym="saxophone"),
+            pages=len("saxophone"),
+            ad=Person.objects.get(name="Juliet"),
+        )
+        # An Ad not associated with the group or the area is responsible for a doc
+        WgDraftFactory(
+            group=Group.objects.get(acronym="bassoon"),
+            pages=len("bassoon"),
+            ad=Person.objects.get(name="Juliet"),
+        )
+
+        (
+            area_summary,
+            area_totals,
+            ad_summary,
+            noad_summary,
+            ad_totals,
+            noad_totals,
+            totals,
+            wg_summary,
+        ) = get_wg_dashboard_info()
+
+        self.assertEqual(
+            area_summary,
+            [
+                {
+                    "area": "red",
+                    "groups_in_area": 1,
+                    "groups_with_docs": 1,
+                    "doc_count": 1,
+                    "page_count": 7,
+                    "group_percent": 6.25,
+                    "doc_percent": 3.571428571428571,
+                    "page_percent": 3.5897435897435894,
+                },
+                {
+                    "area": "orange",
+                    "groups_in_area": 4,
+                    "groups_with_docs": 3,
+                    "doc_count": 4,
+                    "page_count": 29,
+                    "group_percent": 18.75,
+                    "doc_percent": 14.285714285714285,
+                    "page_percent": 14.871794871794872,
+                },
+                {
+                    "area": "yellow",
+                    "groups_in_area": 2,
+                    "groups_with_docs": 2,
+                    "doc_count": 2,
+                    "page_count": 17,
+                    "group_percent": 12.5,
+                    "doc_percent": 7.142857142857142,
+                    "page_percent": 8.717948717948717,
+                },
+                {
+                    "area": "green",
+                    "groups_in_area": 3,
+                    "groups_with_docs": 3,
+                    "doc_count": 5,
+                    "page_count": 22,
+                    "group_percent": 18.75,
+                    "doc_percent": 17.857142857142858,
+                    "page_percent": 11.282051282051283,
+                },
+                {
+                    "area": "blue",
+                    "groups_in_area": 3,
+                    "groups_with_docs": 3,
+                    "doc_count": 6,
+                    "page_count": 40,
+                    "group_percent": 18.75,
+                    "doc_percent": 21.428571428571427,
+                    "page_percent": 20.51282051282051,
+                },
+                {
+                    "area": "violet",
+                    "groups_in_area": 4,
+                    "groups_with_docs": 4,
+                    "doc_count": 10,
+                    "page_count": 80,
+                    "group_percent": 25.0,
+                    "doc_percent": 35.714285714285715,
+                    "page_percent": 41.02564102564102,
+                },
+            ],
+        )
+        self.assertEqual(
+            area_totals, {"group_count": 16, "doc_count": 28, "page_count": 195}
+        )
+        self.assertEqual(
+            ad_summary,
+            [
+                {
+                    "ad": "Alpha",
+                    "area": "red",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0.0,
+                    "doc_percent": 0.0,
+                    "page_percent": 0.0,
+                },
+                {
+                    "ad": "Bravo",
+                    "area": "orange",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0.0,
+                    "doc_percent": 0.0,
+                    "page_percent": 0.0,
+                },
+                {
+                    "ad": "Charlie",
+                    "area": "orange",
+                    "ad_group_count": 2,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0.0,
+                    "doc_percent": 0.0,
+                    "page_percent": 0.0,
+                },
+                {
+                    "ad": "Delta",
+                    "area": "yellow",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0.0,
+                    "doc_percent": 0.0,
+                    "page_percent": 0.0,
+                },
+                {
+                    "ad": "Echo",
+                    "area": "yellow",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0.0,
+                    "doc_percent": 0.0,
+                    "page_percent": 0.0,
+                },
+                {
+                    "ad": "Foxtrot",
+                    "area": "green",
+                    "ad_group_count": 2,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0.0,
+                    "doc_percent": 0.0,
+                    "page_percent": 0.0,
+                },
+                {
+                    "ad": "Golf",
+                    "area": "green",
+                    "ad_group_count": 1,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 4,
+                    "group_percent": 8.333333333333332,
+                    "doc_percent": 7.6923076923076925,
+                    "page_percent": 4.395604395604396,
+                },
+                {
+                    "ad": "Hotel",
+                    "area": "blue",
+                    "ad_group_count": 1,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 11,
+                    "group_percent": 8.333333333333332,
+                    "doc_percent": 7.6923076923076925,
+                    "page_percent": 12.087912087912088,
+                },
+                {
+                    "ad": "Indigo",
+                    "area": "blue",
+                    "ad_group_count": 2,
+                    "doc_group_count": 2,
+                    "doc_count": 2,
+                    "page_count": 9,
+                    "group_percent": 16.666666666666664,
+                    "doc_percent": 15.384615384615385,
+                    "page_percent": 9.89010989010989,
+                },
+                {
+                    "ad": "Juliet",
+                    "area": "orange",
+                    "ad_group_count": 1,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 7,
+                    "group_percent": 8.333333333333332,
+                    "doc_percent": 7.6923076923076925,
+                    "page_percent": 7.6923076923076925,
+                },
+                {
+                    "ad": "Juliet",
+                    "area": "red",
+                    "ad_group_count": 0,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 7,
+                    "group_percent": 8.333333333333332,
+                    "doc_percent": 7.6923076923076925,
+                    "page_percent": 7.6923076923076925,
+                },
+                {
+                    "ad": "Juliet",
+                    "area": "violet",
+                    "ad_group_count": 1,
+                    "doc_group_count": 2,
+                    "doc_count": 2,
+                    "page_count": 14,
+                    "group_percent": 16.666666666666664,
+                    "doc_percent": 15.384615384615385,
+                    "page_percent": 15.384615384615385,
+                },
+                {
+                    "ad": "Kilo",
+                    "area": "violet",
+                    "ad_group_count": 1,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 7,
+                    "group_percent": 8.333333333333332,
+                    "doc_percent": 7.6923076923076925,
+                    "page_percent": 7.6923076923076925,
+                },
+                {
+                    "ad": "Lima",
+                    "area": "green",
+                    "ad_group_count": 0,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 4,
+                    "group_percent": 8.333333333333332,
+                    "doc_percent": 7.6923076923076925,
+                    "page_percent": 4.395604395604396,
+                },
+                {
+                    "ad": "Lima",
+                    "area": "violet",
+                    "ad_group_count": 2,
+                    "doc_group_count": 2,
+                    "doc_count": 3,
+                    "page_count": 28,
+                    "group_percent": 16.666666666666664,
+                    "doc_percent": 23.076923076923077,
+                    "page_percent": 30.76923076923077,
+                },
+            ],
+        )
+        self.assertEqual(
+            noad_summary,
+            [
+                {
+                    "ad": "Alpha",
+                    "area": "red",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0.0,
+                    "doc_percent": 0.0,
+                    "page_percent": 0.0,
+                },
+                {
+                    "ad": "Bravo",
+                    "area": "orange",
+                    "ad_group_count": 1,
+                    "doc_group_count": 0,
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "group_percent": 0.0,
+                    "doc_percent": 0.0,
+                    "page_percent": 0.0,
+                },
+                {
+                    "ad": "Charlie",
+                    "area": "orange",
+                    "ad_group_count": 2,
+                    "doc_group_count": 2,
+                    "doc_count": 2,
+                    "page_count": 15,
+                    "group_percent": 13.333333333333334,
+                    "doc_percent": 13.333333333333334,
+                    "page_percent": 14.423076923076922,
+                },
+                {
+                    "ad": "Delta",
+                    "area": "yellow",
+                    "ad_group_count": 1,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 5,
+                    "group_percent": 6.666666666666667,
+                    "doc_percent": 6.666666666666667,
+                    "page_percent": 4.807692307692308,
+                },
+                {
+                    "ad": "Echo",
+                    "area": "yellow",
+                    "ad_group_count": 1,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 12,
+                    "group_percent": 6.666666666666667,
+                    "doc_percent": 6.666666666666667,
+                    "page_percent": 11.538461538461538,
+                },
+                {
+                    "ad": "Foxtrot",
+                    "area": "green",
+                    "ad_group_count": 2,
+                    "doc_group_count": 2,
+                    "doc_count": 2,
+                    "page_count": 10,
+                    "group_percent": 13.333333333333334,
+                    "doc_percent": 13.333333333333334,
+                    "page_percent": 9.615384615384617,
+                },
+                {
+                    "ad": "Golf",
+                    "area": "green",
+                    "ad_group_count": 1,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 4,
+                    "group_percent": 6.666666666666667,
+                    "doc_percent": 6.666666666666667,
+                    "page_percent": 3.8461538461538463,
+                },
+                {
+                    "ad": "Hotel",
+                    "area": "blue",
+                    "ad_group_count": 1,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 11,
+                    "group_percent": 6.666666666666667,
+                    "doc_percent": 6.666666666666667,
+                    "page_percent": 10.576923076923077,
+                },
+                {
+                    "ad": "Indigo",
+                    "area": "blue",
+                    "ad_group_count": 2,
+                    "doc_group_count": 2,
+                    "doc_count": 2,
+                    "page_count": 9,
+                    "group_percent": 13.333333333333334,
+                    "doc_percent": 13.333333333333334,
+                    "page_percent": 8.653846153846153,
+                },
+                {
+                    "ad": "Juliet",
+                    "area": "orange",
+                    "ad_group_count": 1,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 7,
+                    "group_percent": 6.666666666666667,
+                    "doc_percent": 6.666666666666667,
+                    "page_percent": 6.730769230769231,
+                },
+                {
+                    "ad": "Juliet",
+                    "area": "violet",
+                    "ad_group_count": 1,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 5,
+                    "group_percent": 6.666666666666667,
+                    "doc_percent": 6.666666666666667,
+                    "page_percent": 4.807692307692308,
+                },
+                {
+                    "ad": "Kilo",
+                    "area": "violet",
+                    "ad_group_count": 1,
+                    "doc_group_count": 1,
+                    "doc_count": 1,
+                    "page_count": 7,
+                    "group_percent": 6.666666666666667,
+                    "doc_percent": 6.666666666666667,
+                    "page_percent": 6.730769230769231,
+                },
+                {
+                    "ad": "Lima",
+                    "area": "violet",
+                    "ad_group_count": 2,
+                    "doc_group_count": 2,
+                    "doc_count": 2,
+                    "page_count": 19,
+                    "group_percent": 13.333333333333334,
+                    "doc_percent": 13.333333333333334,
+                    "page_percent": 18.269230769230766,
+                },
+            ],
+        )
+        self.assertEqual(
+            ad_totals,
+            {
+                "ad_group_count": 17,
+                "doc_group_count": 12,
+                "doc_count": 13,
+                "page_count": 91,
+            },
+        )
+        self.assertEqual(
+            noad_totals,
+            {
+                "ad_group_count": 17,
+                "doc_group_count": 15,
+                "doc_count": 15,
+                "page_count": 104,
+            },
+        )
+        self.assertEqual(
+            totals,
+            {
+                "group_count": 17,
+                "doc_count": 28,
+                "page_count": 195,
+                "groups_with_docs_count": 16,
+            },
+        )
+        self.assertEqual(
+            wg_summary,
+            [
+                {
+                    "wg": "bassoon",
+                    "area": "red",
+                    "ad": "Alpha",
+                    "doc_count": 1,
+                    "page_count": 7,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "celesta",
+                    "area": "orange",
+                    "ad": "Bravo",
+                    "doc_count": 0,
+                    "page_count": 0,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "clarinet",
+                    "area": "orange",
+                    "ad": "Charlie",
+                    "doc_count": 1,
+                    "page_count": 8,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "cymbals",
+                    "area": "orange",
+                    "ad": "Charlie",
+                    "doc_count": 1,
+                    "page_count": 7,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "flute",
+                    "area": "yellow",
+                    "ad": "Delta",
+                    "doc_count": 1,
+                    "page_count": 5,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "glockenspiel",
+                    "area": "yellow",
+                    "ad": "Echo",
+                    "doc_count": 1,
+                    "page_count": 12,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "gong",
+                    "area": "green",
+                    "ad": "Foxtrot",
+                    "doc_count": 2,
+                    "page_count": 8,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "guitar",
+                    "area": "green",
+                    "ad": "Foxtrot",
+                    "doc_count": 1,
+                    "page_count": 6,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "harp",
+                    "area": "green",
+                    "ad": "Golf",
+                    "doc_count": 2,
+                    "page_count": 8,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "harpsichord",
+                    "area": "blue",
+                    "ad": "Hotel",
+                    "doc_count": 2,
+                    "page_count": 22,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "oboe",
+                    "area": "blue",
+                    "ad": "Indigo",
+                    "doc_count": 2,
+                    "page_count": 8,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "organ",
+                    "area": "blue",
+                    "ad": "Indigo",
+                    "doc_count": 2,
+                    "page_count": 10,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "piano",
+                    "area": "violet",
+                    "ad": "Juliet",
+                    "doc_count": 2,
+                    "page_count": 10,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "piccolo",
+                    "area": "violet",
+                    "ad": "Kilo",
+                    "doc_count": 2,
+                    "page_count": 14,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "saxophone",
+                    "area": "violet",
+                    "ad": "Lima",
+                    "doc_count": 4,
+                    "page_count": 36,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "tambourine",
+                    "area": "violet",
+                    "ad": "Lima",
+                    "doc_count": 2,
+                    "page_count": 20,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+                {
+                    "wg": "timpani",
+                    "area": "orange",
+                    "ad": "Juliet",
+                    "doc_count": 2,
+                    "page_count": 14,
+                    "rfc_count": 0,
+                    "recent_rfc_count": 0,
+                },
+            ],
+        )
+
+        # Make sure the view doesn't _crash_ - the template is a dead-simple rendering of the dicts, but this test doesn't prove that
+        url = urlreverse("ietf.iesg.views.working_groups")
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+
 
 class IESGAgendaTests(TestCase):
     def setUp(self):
diff --git a/ietf/iesg/urls.py b/ietf/iesg/urls.py
index d8cfec9f90..5fd9dea0cc 100644
--- a/ietf/iesg/urls.py
+++ b/ietf/iesg/urls.py
@@ -59,6 +59,7 @@
     url(r'^agenda/telechat-(?:%(date)s-)?docs.tgz' % settings.URL_REGEXPS, views.telechat_docs_tarfile),
     url(r'^discusses/$', views.discusses),
     url(r'^ietf-activity/$', views.ietf_activity),
+    url(r'^working-groups/$', views.working_groups),
     url(r'^milestones/$', views.milestones_needing_review),
     url(r'^photos/$', views.photos),
 ]
diff --git a/ietf/iesg/utils.py b/ietf/iesg/utils.py
index 56571dc753..9051cf92b2 100644
--- a/ietf/iesg/utils.py
+++ b/ietf/iesg/utils.py
@@ -1,32 +1,45 @@
-from collections import namedtuple
+from collections import Counter, defaultdict, namedtuple
 
-import debug                            # pyflakes:ignore
+import datetime
+
+import debug  # pyflakes:ignore
+
+from django.db import models
+from django.utils import timezone
 
 from ietf.doc.models import Document, STATUSCHANGE_RELATIONS
 from ietf.doc.utils_search import fill_in_telechat_date
+from ietf.group.models import Group
 from ietf.iesg.agenda import get_doc_section
+from ietf.person.utils import get_active_ads
+
 
+TelechatPageCount = namedtuple(
+    "TelechatPageCount",
+    ["for_approval", "for_action", "related", "ad_pages_left_to_ballot_on"],
+)
 
-TelechatPageCount = namedtuple('TelechatPageCount',['for_approval','for_action','related','ad_pages_left_to_ballot_on'])
 
 def telechat_page_count(date=None, docs=None, ad=None):
     if not date and not docs:
         return TelechatPageCount(0, 0, 0, 0)
 
     if not docs:
-        candidates = Document.objects.filter(docevent__telechatdocevent__telechat_date=date).distinct() 
+        candidates = Document.objects.filter(
+            docevent__telechatdocevent__telechat_date=date
+        ).distinct()
         fill_in_telechat_date(candidates)
-        docs = [ doc for doc in candidates if doc.telechat_date()==date ]
+        docs = [doc for doc in candidates if doc.telechat_date() == date]
 
-    for_action =[d for d in docs if get_doc_section(d).endswith('.3')]
+    for_action = [d for d in docs if get_doc_section(d).endswith(".3")]
 
-    for_approval = set(docs)-set(for_action)
+    for_approval = set(docs) - set(for_action)
 
-    drafts = [d for d in for_approval if d.type_id == 'draft']
+    drafts = [d for d in for_approval if d.type_id == "draft"]
 
     ad_pages_left_to_ballot_on = 0
     pages_for_approval = 0
-    
+
     for draft in drafts:
         pages_for_approval += draft.pages or 0
         if ad:
@@ -39,30 +52,269 @@ def telechat_page_count(date=None, docs=None, ad=None):
 
     pages_for_action = 0
     for d in for_action:
-        if d.type_id == 'draft':
+        if d.type_id == "draft":
             pages_for_action += d.pages or 0
-        elif d.type_id == 'statchg':
+        elif d.type_id == "statchg":
             for rel in d.related_that_doc(STATUSCHANGE_RELATIONS):
                 pages_for_action += rel.pages or 0
-        elif d.type_id == 'conflrev':
-            for rel in d.related_that_doc('conflrev'):
+        elif d.type_id == "conflrev":
+            for rel in d.related_that_doc("conflrev"):
                 pages_for_action += rel.pages or 0
         else:
             pass
 
     related_pages = 0
-    for d in for_approval-set(drafts):
-        if d.type_id == 'statchg':
+    for d in for_approval - set(drafts):
+        if d.type_id == "statchg":
             for rel in d.related_that_doc(STATUSCHANGE_RELATIONS):
                 related_pages += rel.pages or 0
-        elif d.type_id == 'conflrev':
-            for rel in d.related_that_doc('conflrev'):
+        elif d.type_id == "conflrev":
+            for rel in d.related_that_doc("conflrev"):
                 related_pages += rel.pages or 0
         else:
             # There's really nothing to rely on to give a reading load estimate for charters
             pass
-    
-    return TelechatPageCount(for_approval=pages_for_approval,
-                             for_action=pages_for_action,
-                             related=related_pages,
-                             ad_pages_left_to_ballot_on=ad_pages_left_to_ballot_on)
+
+    return TelechatPageCount(
+        for_approval=pages_for_approval,
+        for_action=pages_for_action,
+        related=related_pages,
+        ad_pages_left_to_ballot_on=ad_pages_left_to_ballot_on,
+    )
+
+
+def get_wg_dashboard_info():
+    docs = (
+        Document.objects.filter(
+            group__type="wg",
+            group__state="active",
+            states__type="draft",
+            states__slug="active",
+        )
+        .filter(models.Q(ad__isnull=True) | models.Q(ad__in=get_active_ads()))
+        .distinct()
+        .prefetch_related("group", "group__parent")
+        .exclude(
+            states__type="draft-stream-ietf",
+            states__slug__in=["c-adopt", "wg-cand", "dead", "parked", "info"],
+        )
+    )
+    groups = Group.objects.filter(state="active", type="wg")
+    areas = Group.objects.filter(state="active", type="area")
+
+    total_group_count = groups.count()
+    total_doc_count = docs.count()
+    total_page_count = docs.aggregate(models.Sum("pages"))["pages__sum"] or 0
+    totals = {
+        "group_count": total_group_count,
+        "doc_count": total_doc_count,
+        "page_count": total_page_count,
+    }
+
+    # Since this view is primarily about counting subsets of the above docs query and the
+    # expected number of returned documents is just under 1000 typically - do the totaling
+    # work in python rather than asking the db to do it.
+
+    groups_for_area = defaultdict(set)
+    pages_for_area = defaultdict(lambda: 0)
+    docs_for_area = defaultdict(lambda: 0)
+    groups_for_ad = defaultdict(lambda: defaultdict(set))
+    pages_for_ad = defaultdict(lambda: defaultdict(lambda: 0))
+    docs_for_ad = defaultdict(lambda: defaultdict(lambda: 0))
+    groups_for_noad = defaultdict(lambda: defaultdict(set))
+    pages_for_noad = defaultdict(lambda: defaultdict(lambda: 0))
+    docs_for_noad = defaultdict(lambda: defaultdict(lambda: 0))
+    docs_for_wg = defaultdict(lambda: 0)
+    pages_for_wg = defaultdict(lambda: 0)
+    groups_total = set()
+    pages_total = 0
+    docs_total = 0
+
+    responsible_for_group = defaultdict(lambda: defaultdict(lambda: "None"))
+    responsible_count = defaultdict(lambda: defaultdict(lambda: 0))
+    for group in groups:
+        responsible = f"{', '.join([r.person.plain_name() for r in group.role_set.filter(name_id='ad')])}"
+        docs_for_noad[responsible][group.parent.acronym] = (
+            0  # Ensure these keys are present later
+        )
+        docs_for_ad[responsible][group.parent.acronym] = 0
+        responsible_for_group[group.acronym][group.parent.acronym] = responsible
+        responsible_count[responsible][group.parent.acronym] += 1
+
+    for doc in docs:
+        docs_for_wg[doc.group] += 1
+        pages_for_wg[doc.group] += doc.pages
+        groups_for_area[doc.group.area.acronym].add(doc.group.acronym)
+        pages_for_area[doc.group.area.acronym] += doc.pages
+        docs_for_area[doc.group.area.acronym] += 1
+
+        if doc.ad is None:
+            responsible = responsible_for_group[doc.group.acronym][
+                doc.group.parent.acronym
+            ]
+            groups_for_noad[responsible][doc.group.parent.acronym].add(
+                doc.group.acronym
+            )
+            pages_for_noad[responsible][doc.group.parent.acronym] += doc.pages
+            docs_for_noad[responsible][doc.group.parent.acronym] += 1
+        else:
+            responsible = f"{doc.ad.plain_name()}"
+            groups_for_ad[responsible][doc.group.parent.acronym].add(doc.group.acronym)
+            pages_for_ad[responsible][doc.group.parent.acronym] += doc.pages
+            docs_for_ad[responsible][doc.group.parent.acronym] += 1
+
+        docs_total += 1
+        groups_total.add(doc.group.acronym)
+        pages_total += doc.pages
+
+    groups_total = len(groups_total)
+    totals["groups_with_docs_count"] = groups_total
+
+    area_summary = []
+
+    for area in areas:
+        group_count = len(groups_for_area[area.acronym])
+        doc_count = docs_for_area[area.acronym]
+        page_count = pages_for_area[area.acronym]
+        area_summary.append(
+            {
+                "area": area.acronym,
+                "groups_in_area": groups.filter(parent=area).count(),
+                "groups_with_docs": group_count,
+                "doc_count": doc_count,
+                "page_count": page_count,
+                "group_percent": group_count / groups_total * 100
+                if groups_total != 0
+                else 0,
+                "doc_percent": doc_count / docs_total * 100 if docs_total != 0 else 0,
+                "page_percent": page_count / pages_total * 100
+                if pages_total != 0
+                else 0,
+            }
+        )
+    area_totals = {
+        "group_count": groups_total,
+        "doc_count": docs_total,
+        "page_count": pages_total,
+    }
+
+    noad_summary = []
+    noad_totals = {
+        "ad_group_count": 0,
+        "doc_group_count": 0,
+        "doc_count": 0,
+        "page_count": 0,
+    }
+    for ad in docs_for_noad:
+        for area in docs_for_noad[ad]:
+            noad_totals["ad_group_count"] += responsible_count[ad][area]
+            noad_totals["doc_group_count"] += len(groups_for_noad[ad][area])
+            noad_totals["doc_count"] += docs_for_noad[ad][area]
+            noad_totals["page_count"] += pages_for_noad[ad][area]
+    for ad in docs_for_noad:
+        for area in docs_for_noad[ad]:
+            noad_summary.append(
+                {
+                    "ad": ad,
+                    "area": area,
+                    "ad_group_count": responsible_count[ad][area],
+                    "doc_group_count": len(groups_for_noad[ad][area]),
+                    "doc_count": docs_for_noad[ad][area],
+                    "page_count": pages_for_noad[ad][area],
+                    "group_percent": len(groups_for_noad[ad][area])
+                    / noad_totals["doc_group_count"]
+                    * 100
+                    if noad_totals["doc_group_count"] != 0
+                    else 0,
+                    "doc_percent": docs_for_noad[ad][area]
+                    / noad_totals["doc_count"]
+                    * 100
+                    if noad_totals["doc_count"] != 0
+                    else 0,
+                    "page_percent": pages_for_noad[ad][area]
+                    / noad_totals["page_count"]
+                    * 100
+                    if noad_totals["page_count"] != 0
+                    else 0,
+                }
+            )
+    noad_summary.sort(key=lambda r: (r["ad"], r["area"]))
+
+    ad_summary = []
+    ad_totals = {
+        "ad_group_count": 0,
+        "doc_group_count": 0,
+        "doc_count": 0,
+        "page_count": 0,
+    }
+    for ad in docs_for_ad:
+        for area in docs_for_ad[ad]:
+            ad_totals["ad_group_count"] += responsible_count[ad][area]
+            ad_totals["doc_group_count"] += len(groups_for_ad[ad][area])
+            ad_totals["doc_count"] += docs_for_ad[ad][area]
+            ad_totals["page_count"] += pages_for_ad[ad][area]
+    for ad in docs_for_ad:
+        for area in docs_for_ad[ad]:
+            ad_summary.append(
+                {
+                    "ad": ad,
+                    "area": area,
+                    "ad_group_count": responsible_count[ad][area],
+                    "doc_group_count": len(groups_for_ad[ad][area]),
+                    "doc_count": docs_for_ad[ad][area],
+                    "page_count": pages_for_ad[ad][area],
+                    "group_percent": len(groups_for_ad[ad][area])
+                    / ad_totals["doc_group_count"]
+                    * 100
+                    if ad_totals["doc_group_count"] != 0
+                    else 0,
+                    "doc_percent": docs_for_ad[ad][area] / ad_totals["doc_count"] * 100
+                    if ad_totals["doc_count"] != 0
+                    else 0,
+                    "page_percent": pages_for_ad[ad][area]
+                    / ad_totals["page_count"]
+                    * 100
+                    if ad_totals["page_count"] != 0
+                    else 0,
+                }
+            )
+    ad_summary.sort(key=lambda r: (r["ad"], r["area"]))
+
+    rfc_counter = Counter(
+        Document.objects.filter(type="rfc").values_list("group__acronym", flat=True)
+    )
+    recent_rfc_counter = Counter(
+        Document.objects.filter(
+            type="rfc",
+            docevent__type="published_rfc",
+            docevent__time__gte=timezone.now() - datetime.timedelta(weeks=104),
+        ).values_list("group__acronym", flat=True)
+    )
+    for wg in set(groups) - set(docs_for_wg.keys()):
+        docs_for_wg[wg] += 0
+        pages_for_wg[wg] += 0
+    wg_summary = []
+    for wg in docs_for_wg:
+        wg_summary.append(
+            {
+                "wg": wg.acronym,
+                "area": wg.parent.acronym,
+                "ad": responsible_for_group[wg.acronym][wg.parent.acronym],
+                "doc_count": docs_for_wg[wg],
+                "page_count": pages_for_wg[wg],
+                "rfc_count": rfc_counter[wg.acronym],
+                "recent_rfc_count": recent_rfc_counter[wg.acronym],
+            }
+        )
+    wg_summary.sort(key=lambda r: (r["wg"], r["area"]))
+
+    return (
+        area_summary,
+        area_totals,
+        ad_summary,
+        noad_summary,
+        ad_totals,
+        noad_totals,
+        totals,
+        wg_summary,
+    )
diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py
index ffd4515c98..014b290425 100644
--- a/ietf/iesg/views.py
+++ b/ietf/iesg/views.py
@@ -61,7 +61,7 @@
 from ietf.group.models import GroupMilestone, Role
 from ietf.iesg.agenda import agenda_data, agenda_sections, fill_in_agenda_docs, get_agenda_date
 from ietf.iesg.models import TelechatDate, TelechatAgendaContent
-from ietf.iesg.utils import telechat_page_count
+from ietf.iesg.utils import get_wg_dashboard_info, telechat_page_count
 from ietf.ietfauth.utils import has_role, role_required, user_is_person
 from ietf.name.models import TelechatAgendaSectionName
 from ietf.person.models import Person
@@ -626,3 +626,13 @@ def telechat_agenda_content_view(request, section):
         content=content.text,
         content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
     )
+
+def working_groups(request):
+ 
+    area_summary, area_totals, ad_summary, noad_summary, ad_totals, noad_totals, totals, wg_summary = get_wg_dashboard_info()
+
+    return render(
+        request,
+        "iesg/working_groups.html",
+        dict(area_summary=area_summary, area_totals=area_totals, ad_summary=ad_summary, noad_summary=noad_summary, ad_totals=ad_totals, noad_totals=noad_totals, totals=totals, wg_summary=wg_summary),
+    )
diff --git a/ietf/templates/iesg/working_groups.html b/ietf/templates/iesg/working_groups.html
new file mode 100644
index 0000000000..b799636857
--- /dev/null
+++ b/ietf/templates/iesg/working_groups.html
@@ -0,0 +1,159 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% load origin static %}
+{% block pagehead %}
+    <link rel="stylesheet" href="{% static 'ietf/css/list.css' %}">
+{% endblock %}
+{% block title %}IESG view of working groups{% endblock %}
+{% block content %}
+    {% origin %}
+    <h1 class="mt-4">IESG view of working groups</h1>
+    <h2 class="mt-3'">Area Size and Load</h2>
+    <table class="table table-sm table-striped tablesorter">
+        <thead>
+            <tr>
+                <th scope="col" data-sort="areaname">Area Name</th>
+                <th scope="col" data-sort="wgcount">WGs</th>
+                <th scope="col" data-sort="idcount">I-Ds</th>
+                <th scope="col" data-sort="idpages">Pages</th>
+                {# (divider) #}
+                <th scope="col" data-sort="idpercent">% I-Ds</th>
+                <th scope="col" data-sort="pagespercent">% Pages</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for area in area_summary %}
+              <tr>
+                <td>{{area.area}}</td>
+                <td>{{area.groups_in_area}}</td>
+                <td>{{area.doc_count}}</td>
+                <td>{{area.page_count}}</td>
+                <td>{{area.doc_percent|floatformat:1}}</td>
+                <td>{{area.page_percent|floatformat:1}}</td>
+              </tr>
+            {% endfor %}
+        </tbody>
+        <tfoot>
+            <tr>
+                <td>Totals</td>
+                <td>{{totals.group_count}}</td>
+                <td>{{totals.doc_count}}</td>
+                <td>{{totals.page_count}}</td>
+                <td></td>
+                <td></td>
+            </tr>
+        </tfoot>
+    </table>
+
+    <h2 class="mt-3'">Area Director Load: Documents not yet directly assigned to AD</h2>
+    <div class="small text-body-secondary">Typically these are pre-pubreq documents</div>
+    <table class="table table-sm table-striped tablesorter">
+        <thead>
+            <tr>
+                <th scope="col" data-sort="areadirname">AD</th>
+                <th scope="col" data-sort="areaname">Area Name</th>
+                <th scope="col" data-sort="areagroups">WGs for AD</th>
+                <th scope="col" data-sort="idcount">I-Ds</th>
+                <th scope="col" data-sort="idpages">Pages</th>
+                {# (divider) #}
+                <th scope="col" data-sort="idpercent">% I-Ds</th>
+                <th scope="col" data-sort="pagespercent">% Pages</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for ad in noad_summary %}
+                <tr>
+                    <td>{{ad.ad}}</td>
+                    <td>{{ad.area}}</td>
+                    <td>{{ad.ad_group_count}}</td>
+                    <td>{{ad.doc_count}}</td>
+                    <td>{{ad.page_count}}</td>
+                    <td>{{ad.doc_percent|floatformat:1}}</td>
+                    <td>{{ad.page_percent|floatformat:1}}</td>
+                </tr>
+            {% endfor %}
+        </tbody>
+        <tfoot>
+            <tr>
+                <td>Totals</td>
+                <td></td>
+                <td>{{noad_totals.ad_group_count}}</td>
+                <td>{{noad_totals.doc_count}}</td>
+                <td>{{noad_totals.page_count}}</td>
+                <td></td>
+                <td></td>
+            </tr>
+        </tfoot>
+    </table>    
+
+    <h2 class="mt-3'">Area Director Load: Documents directly assigned to AD</h2>
+    <table class="table table-sm table-striped tablesorter">
+        <thead>
+            <tr>
+                <th scope="col" data-sort="areadirname">AD</th>
+                <th scope="col" data-sort="areaname">Area Name</th>
+                <th scope="col" data-sort="areagroups">WGs for AD</th>
+                <th scope="col" data-sort="idcount">I-Ds</th>
+                <th scope="col" data-sort="idpages">Pages</th>
+                {# (divider) #}
+                <th scope="col" data-sort="idpercent">% I-Ds</th>
+                <th scope="col" data-sort="pagespercent">% Pages</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for ad in ad_summary %}
+                <tr>
+                    <td>{{ad.ad}}</td>
+                    <td>{{ad.area}}</td>
+                    <td>{{ad.ad_group_count}}</td>
+                    <td>{{ad.doc_count}}</td>
+                    <td>{{ad.page_count}}</td>
+                    <td>{{ad.doc_percent|floatformat:1}}</td>
+                    <td>{{ad.page_percent|floatformat:1}}</td>
+                </tr>
+            {% endfor %}
+        </tbody>
+        <tfoot>
+            <tr>
+                <td>Totals</td>
+                <td></td>
+                <td>{{ad_totals.ad_group_count}}</td>
+                <td>{{ad_totals.doc_count}}</td>
+                <td>{{ad_totals.page_count}}</td>
+                <td></td>
+                <td></td>
+            </tr>
+        </tfoot>
+    </table>  
+
+    <h2 class="mt-3'">Working Group Summary</h2>
+    <table class="table table-sm table-striped tablesorter">
+        <thead>
+            <tr>
+                <th scope="col" data-sort="wg">WG</th>
+                <th scope="col" data-sort="area">Area</th>
+                <th scope="col" data-sort="ad">AD</th>
+                <th scope="col" data-sort="ids">I-Ds</th>
+                <th scope="col" data-sort="pages">Pages</th>
+                <th scope="col" data-sort="rfcs">RFCs</th>
+                <th scope="col" data-sort="rfcpubrate">RFCs in last 2 years</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for wg in wg_summary %}
+            <tr>
+                <td>{{wg.wg}}</td>
+                <td>{{wg.area}}</td>
+                <td>{{wg.ad}}</td>
+                <td>{{wg.doc_count}}</td>
+                <td>{{wg.page_count}}</td>
+                <td>{{wg.rfc_count}}</td>
+                <td>{{wg.recent_rfc_count}}</td>
+            </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+{% endblock %}
+{% block js %}
+    <script src="{% static "ietf/js/list.js" %}"></script>
+{% endblock %}
\ No newline at end of file

From acffceba0b8f61d6a5c972080df41f9e86743919 Mon Sep 17 00:00:00 2001
From: Phil Whipps <WhippsP@users.noreply.github.com>
Date: Thu, 18 Sep 2025 04:46:20 +1000
Subject: [PATCH 455/656] fix: Rev Fix Option 2 - Htmlized url regex (#9538)

* Update Rev Regex in settings.py

Removing single value revision numbers as that is against the naming standard (https://authors.ietf.org/naming-your-internet-draft#version) and causes issues with htmlized documents with -1 in the name (eg draft-ietf-oauth-v2-1)

* Reverse REGEX Change

* Update URLS REgex for REV

Directly insert Regex for REV rather than reference settings.URL_REGEXPS. This is to resolve issue https://github.com/ietf-tools/datatracker/issues/9533
---
 ietf/doc/urls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py
index 60255af856..6f1b698a9f 100644
--- a/ietf/doc/urls.py
+++ b/ietf/doc/urls.py
@@ -75,7 +75,7 @@
 # This block should really all be at the idealized docs.ietf.org service
     url(r'^html/(?P<name>bcp[0-9]+?)(\.txt|\.html)?/?$', RedirectView.as_view(url=settings.RFC_EDITOR_INFO_BASE_URL+"%(name)s", permanent=False)), 
     url(r'^html/(?P<name>std[0-9]+?)(\.txt|\.html)?/?$', RedirectView.as_view(url=settings.RFC_EDITOR_INFO_BASE_URL+"%(name)s", permanent=False)), 
-    url(r'^html/%(name)s(?:-%(rev)s)?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, views_doc.document_html),
+    url(r'^html/%(name)s(?:-(?P<rev>[0-9]{2}(-[0-9]{2})?))?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, views_doc.document_html),
 
     url(r'^id/%(name)s(?:-%(rev)s)?(?:\.(?P<ext>(txt|html|xml)))?/?$' % settings.URL_REGEXPS, views_doc.document_raw_id),
     url(r'^pdf/%(name)s(?:-%(rev)s)?(?:\.(?P<ext>[a-z]+))?/?$' % settings.URL_REGEXPS, views_doc.document_pdfized),

From 6b58aa4bd47fd5fe84750a0cc66dd38b8e801c72 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 18 Sep 2025 10:20:03 -0500
Subject: [PATCH 456/656] fix: edit only attachments actually attached to this
 liaison statement (#9548)

* fix: edit only attachments actually attached to this liaison statement

* chore: remove unused import

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 ietf/liaisons/tests.py | 29 +++++++++++++++++++++++------
 ietf/liaisons/views.py | 11 +++++++----
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py
index a1fbf77841..2f86f38789 100644
--- a/ietf/liaisons/tests.py
+++ b/ietf/liaisons/tests.py
@@ -939,17 +939,34 @@ def test_liaison_add_attachment(self):
         )
 
     def test_liaison_edit_attachment(self):
-
-        attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1')
-        url = urlreverse('ietf.liaisons.views.liaison_edit_attachment', kwargs=dict(object_id=attachment.statement_id,doc_id=attachment.document_id))
+        attachment = LiaisonStatementAttachmentFactory(document__name="liaiatt-1")
+        url = urlreverse(
+            "ietf.liaisons.views.liaison_edit_attachment",
+            kwargs=dict(
+                object_id=attachment.statement_id, doc_id=attachment.document_id
+            ),
+        )
         login_testing_unauthorized(self, "secretary", url)
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-        post_data = dict(title='New Title')
-        r = self.client.post(url,post_data)
+        post_data = dict(title="New Title")
+        r = self.client.post(url, post_data)
         attachment = LiaisonStatementAttachment.objects.get(pk=attachment.pk)
         self.assertEqual(r.status_code, 302)
-        self.assertEqual(attachment.document.title,'New Title')
+        self.assertEqual(attachment.document.title, "New Title")
+
+        # ensure attempts to edit attachments not attached to this liaison statement fail
+        other_attachment = LiaisonStatementAttachmentFactory(document__name="liaiatt-2")
+        url = urlreverse(
+            "ietf.liaisons.views.liaison_edit_attachment",
+            kwargs=dict(
+                object_id=attachment.statement_id, doc_id=other_attachment.document_id
+            ),
+        )
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
+        r = self.client.post(url, dict(title="New Title"))
+        self.assertEqual(r.status_code, 404)
 
     def test_liaison_delete_attachment(self):
         attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1')
diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py
index 9710149c90..f9136a8d14 100644
--- a/ietf/liaisons/views.py
+++ b/ietf/liaisons/views.py
@@ -7,15 +7,14 @@
 
 from django.contrib import messages
 from django.urls import reverse as urlreverse
-from django.core.exceptions import ValidationError
+from django.core.exceptions import ValidationError, ObjectDoesNotExist
 from django.core.validators import validate_email
 from django.db.models import Q, Prefetch
-from django.http import HttpResponse
+from django.http import Http404, HttpResponse
 from django.shortcuts import render, get_object_or_404, redirect
 
 import debug                            # pyflakes:ignore
 
-from ietf.doc.models import Document
 from ietf.ietfauth.utils import role_required, has_role
 from ietf.group.models import Group, Role
 from ietf.liaisons.models import (LiaisonStatement,LiaisonStatementEvent,
@@ -444,7 +443,11 @@ def liaison_edit(request, object_id):
 def liaison_edit_attachment(request, object_id, doc_id):
     '''Edit the Liaison Statement attachment title'''
     liaison = get_object_or_404(LiaisonStatement, pk=object_id)
-    doc = get_object_or_404(Document, pk=doc_id)
+    try:
+       doc = liaison.attachments.get(pk=doc_id)
+    except ObjectDoesNotExist:
+        raise Http404
+
     if not can_edit_liaison(request.user, liaison):
         permission_denied(request, "You are not authorized for this action.")
 

From 76f56ceabf4a101c7a8f72946778b7bb5b63f570 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 18 Sep 2025 10:20:30 -0500
Subject: [PATCH 457/656] fix: adjust anachronystic urls - doc_ids became
 numeric years ago. (#9549)

---
 ietf/liaisons/urls.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/liaisons/urls.py b/ietf/liaisons/urls.py
index a4afbfef5d..0fbd29425e 100644
--- a/ietf/liaisons/urls.py
+++ b/ietf/liaisons/urls.py
@@ -26,8 +26,8 @@
     url(r'^(?P<object_id>\d+)/$', views.liaison_detail),
     url(r'^(?P<object_id>\d+)/addcomment/$', views.add_comment),
     url(r'^(?P<object_id>\d+)/edit/$', views.liaison_edit),
-    url(r'^(?P<object_id>\d+)/edit-attachment/(?P<doc_id>[A-Za-z0-9._+-]+)$', views.liaison_edit_attachment),
-    url(r'^(?P<object_id>\d+)/delete-attachment/(?P<attach_id>[A-Za-z0-9._+-]+)$', views.liaison_delete_attachment),
+    url(r'^(?P<object_id>\d+)/edit-attachment/(?P<doc_id>[0-9]+)$', views.liaison_edit_attachment),
+    url(r'^(?P<object_id>\d+)/delete-attachment/(?P<attach_id>[0-9]+)$', views.liaison_delete_attachment),
     url(r'^(?P<object_id>\d+)/history/$', views.liaison_history),
     url(r'^(?P<object_id>\d+)/reply/$', views.liaison_reply),
     url(r'^(?P<object_id>\d+)/resend/$', views.liaison_resend),

From ad5823e0c6ebaa88ae6c949e1bdefeab951cb280 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 18 Sep 2025 10:22:31 -0500
Subject: [PATCH 458/656] fix: properly guard state transitions (#9554)

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 ietf/liaisons/tests.py |  3 +++
 ietf/liaisons/views.py | 32 +++++++++++++++++++-------------
 2 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py
index 2f86f38789..c3ff9dbe94 100644
--- a/ietf/liaisons/tests.py
+++ b/ietf/liaisons/tests.py
@@ -363,6 +363,9 @@ def test_approval_process(self):
         self.assertEqual(len(q('form button[name=approved]')), 0)
 
         # check the detail page / authorized
+        r = self.client.post(url, dict(dead="1"))
+        self.assertEqual(r.status_code, 403)
+        mailbox_before = len(outbox)
         self.client.login(username="ulm-liaiman", password="ulm-liaiman+password")
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py
index f9136a8d14..6a1e6e3def 100644
--- a/ietf/liaisons/views.py
+++ b/ietf/liaisons/views.py
@@ -7,7 +7,7 @@
 
 from django.contrib import messages
 from django.urls import reverse as urlreverse
-from django.core.exceptions import ValidationError, ObjectDoesNotExist
+from django.core.exceptions import ValidationError, ObjectDoesNotExist, PermissionDenied
 from django.core.validators import validate_email
 from django.db.models import Q, Prefetch
 from django.http import Http404, HttpResponse
@@ -404,22 +404,28 @@ def liaison_detail(request, object_id):
 
 
     if request.method == 'POST':
-        if request.POST.get('approved'):
-            liaison.change_state(state_id='approved',person=person)
-            liaison.change_state(state_id='posted',person=person)
-            send_liaison_by_email(request, liaison)
-            messages.success(request,'Liaison Statement Approved and Posted')
-        elif request.POST.get('dead'):
-            liaison.change_state(state_id='dead',person=person)
-            messages.success(request,'Liaison Statement Killed')
-        elif request.POST.get('resurrect'):
-            liaison.change_state(state_id='pending',person=person)
-            messages.success(request,'Liaison Statement Resurrected')
-        elif request.POST.get('do_action_taken') and can_take_care:
+        if request.POST.get('do_action_taken') and can_take_care:
             liaison.tags.remove('required')
             liaison.tags.add('taken')
             can_take_care = False
             messages.success(request,'Action handled')
+        else:
+            if can_edit:
+                if request.POST.get('approved'):
+                    liaison.change_state(state_id='approved',person=person)
+                    liaison.change_state(state_id='posted',person=person)
+                    send_liaison_by_email(request, liaison)
+                    messages.success(request,'Liaison Statement Approved and Posted')
+                elif request.POST.get('dead'):
+                    liaison.change_state(state_id='dead',person=person)
+                    messages.success(request,'Liaison Statement Killed')
+                elif request.POST.get('resurrect'):
+                    liaison.change_state(state_id='pending',person=person)
+                    messages.success(request,'Liaison Statement Resurrected')
+                else:
+                    pass
+            else:
+                raise PermissionDenied()
 
     relations_by = [i.target for i in liaison.source_of_set.filter(target__state__slug='posted')]
     relations_to = [i.source for i in liaison.target_of_set.filter(source__state__slug='posted')]

From e1c75d46161939acaf093bb50cf91af9a2cbb7ea Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 18 Sep 2025 10:32:26 -0500
Subject: [PATCH 459/656] fix: disable removing liaison attachments pending
 reimplementation (#9555)

---
 ietf/liaisons/tests.py | 16 ++++++++--------
 ietf/liaisons/views.py | 39 ++++++++++++++++++++++-----------------
 2 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py
index c3ff9dbe94..fd1c22be77 100644
--- a/ietf/liaisons/tests.py
+++ b/ietf/liaisons/tests.py
@@ -971,14 +971,14 @@ def test_liaison_edit_attachment(self):
         r = self.client.post(url, dict(title="New Title"))
         self.assertEqual(r.status_code, 404)
 
-    def test_liaison_delete_attachment(self):
-        attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1')
-        liaison = attachment.statement
-        url = urlreverse('ietf.liaisons.views.liaison_delete_attachment', kwargs=dict(object_id=liaison.pk,attach_id=attachment.pk))
-        login_testing_unauthorized(self, "secretary", url)
-        r = self.client.get(url)
-        self.assertEqual(r.status_code, 302)
-        self.assertEqual(liaison.liaisonstatementattachment_set.filter(removed=False).count(),0)
+    # def test_liaison_delete_attachment(self):
+    #     attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1')
+    #     liaison = attachment.statement
+    #     url = urlreverse('ietf.liaisons.views.liaison_delete_attachment', kwargs=dict(object_id=liaison.pk,attach_id=attachment.pk))
+    #     login_testing_unauthorized(self, "secretary", url)
+    #     r = self.client.get(url)
+    #     self.assertEqual(r.status_code, 302)
+    #     self.assertEqual(liaison.liaisonstatementattachment_set.filter(removed=False).count(),0)
 
     def test_in_response(self):
         '''A statement with purpose=in_response must have related statement specified'''
diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py
index 6a1e6e3def..6a6f579714 100644
--- a/ietf/liaisons/views.py
+++ b/ietf/liaisons/views.py
@@ -17,8 +17,7 @@
 
 from ietf.ietfauth.utils import role_required, has_role
 from ietf.group.models import Group, Role
-from ietf.liaisons.models import (LiaisonStatement,LiaisonStatementEvent,
-    LiaisonStatementAttachment)
+from ietf.liaisons.models import LiaisonStatement,LiaisonStatementEvent
 from ietf.liaisons.utils import (get_person_for_user, can_add_outgoing_liaison,
     can_add_incoming_liaison, can_edit_liaison,can_submit_liaison_required,
     can_add_liaison)
@@ -377,23 +376,29 @@ def liaison_history(request, object_id):
 
 def liaison_delete_attachment(request, object_id, attach_id):
     liaison = get_object_or_404(LiaisonStatement, pk=object_id)
-    attach = get_object_or_404(LiaisonStatementAttachment, pk=attach_id)
+
     if not can_edit_liaison(request.user, liaison):
         permission_denied(request, "You are not authorized for this action.")
-
-    # FIXME: this view should use POST instead of GET when deleting
-    attach.removed = True
-    attach.save()
-
-    # create event
-    LiaisonStatementEvent.objects.create(
-        type_id='modified',
-        by=get_person_for_user(request.user),
-        statement=liaison,
-        desc='Attachment Removed: {}'.format(attach.document.title)
-    )
-    messages.success(request, 'Attachment Deleted')
-    return redirect('ietf.liaisons.views.liaison_detail', object_id=liaison.pk)
+    else:
+        permission_denied(request, "This operation is temporarily unavailable. Ask the secretariat to mark the attachment as removed using the admin.")
+
+    # The following will be replaced with a different approach in the next generation of the liaison tool
+    # attach = get_object_or_404(LiaisonStatementAttachment, pk=attach_id)
+
+    # # FIXME: this view should use POST instead of GET when deleting
+    # attach.removed = True
+    # debug.say("Got here")
+    # attach.save()
+
+    # # create event
+    # LiaisonStatementEvent.objects.create(
+    #     type_id='modified',
+    #     by=get_person_for_user(request.user),
+    #     statement=liaison,
+    #     desc='Attachment Removed: {}'.format(attach.document.title)
+    # )
+    # messages.success(request, 'Attachment Deleted')
+    # return redirect('ietf.liaisons.views.liaison_detail', object_id=liaison.pk)
 
 def liaison_detail(request, object_id):
     liaison = get_object_or_404(LiaisonStatement, pk=object_id)

From 87e550c74ffef0f5b64b78a6a487321ebe923f11 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 19 Sep 2025 13:55:14 -0300
Subject: [PATCH 460/656] refactor: compare tokens using compare_digest (#9562)

* refactor: compare tokens using compare_digest

* test: test new helper

* refactor: const-time for auth_token check also
---
 ietf/submit/tests.py | 31 +++++++++++++++++++++++++++-
 ietf/submit/views.py | 49 +++++++++++++++++++++++++++++++++++---------
 2 files changed, 69 insertions(+), 11 deletions(-)

diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index 6b9002502b..ede63d2752 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -51,8 +51,9 @@
                                process_submission_xml, process_uploaded_submission, 
                                process_and_validate_submission, apply_yang_checker_to_draft, 
                                run_all_yang_model_checks)
+from ietf.submit.views import access_token_is_valid, auth_token_is_valid
 from ietf.utils import tool_version
-from ietf.utils.accesstoken import generate_access_token
+from ietf.utils.accesstoken import generate_access_token, generate_random_key
 from ietf.utils.mail import outbox, get_payload_text
 from ietf.utils.test_runner import TestBlobstoreManager
 from ietf.utils.test_utils import login_testing_unauthorized, TestCase
@@ -3500,3 +3501,31 @@ def test_submissionerror(self, mock_sanitize_message):
             mock_sanitize_message.call_args_list,
             [mock.call("hi"), mock.call("there")],
         )
+
+
+class HelperTests(TestCase):
+    def test_access_token_is_valid(self):
+        submission: Submission = SubmissionFactory()  # type: ignore
+        valid_token = submission.access_token()
+        access_key = submission.access_key  # accept this for backwards compat
+        invalid_token = "not the valid token"
+        self.assertTrue(access_token_is_valid(submission, valid_token))
+        self.assertTrue(access_token_is_valid(submission, access_key))
+        self.assertFalse(access_token_is_valid(submission, invalid_token))
+
+    def test_auth_token_is_valid(self):
+        auth_key = generate_random_key()
+        submission: Submission = SubmissionFactory(auth_key = auth_key)  # type: ignore
+        valid_token = generate_access_token(submission.auth_key)
+        auth_key = submission.auth_key  # accept this for backwards compat
+        invalid_token = "not the valid token"
+        self.assertTrue(auth_token_is_valid(submission, valid_token))
+        self.assertTrue(auth_token_is_valid(submission, auth_key))
+        self.assertFalse(auth_token_is_valid(submission, invalid_token))
+
+        submission.auth_key = ""
+        submission.save()
+        self.assertFalse(auth_token_is_valid(submission, valid_token))
+        self.assertFalse(auth_token_is_valid(submission, auth_key))
+        self.assertFalse(auth_token_is_valid(submission, invalid_token))
+        self.assertFalse(auth_token_is_valid(submission, ""))
diff --git a/ietf/submit/views.py b/ietf/submit/views.py
index 043b613016..8329a312bb 100644
--- a/ietf/submit/views.py
+++ b/ietf/submit/views.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 import re
 import datetime
+from secrets import compare_digest
 
 from typing import Optional, cast         # pyflakes:ignore
 from urllib.parse import urljoin
@@ -255,19 +256,48 @@ def search_submission(request):
     )
 
 
-def can_edit_submission(user, submission, access_token):
-    key_matched = access_token and submission.access_token() == access_token
-    if not key_matched: key_matched = submission.access_key == access_token # backwards-compat
-    return key_matched or has_role(user, "Secretariat")
+def access_token_is_valid(submission: Submission, access_token: str):
+    """Check whether access_token is valid for submission, in constant time"""
+    token_matched = compare_digest(submission.access_token(), access_token)
+    # also compare key directly for backwards compatibility
+    key_matched = compare_digest(submission.access_key, access_token)
+    return token_matched or key_matched
+
+
+def auth_token_is_valid(submission: Submission, auth_token: str):
+    """Check whether auth_token is valid for submission, in constant time"""
+    auth_key = submission.auth_key
+    if not auth_key:
+        # Make the same calls as the other branch to keep constant time, then
+        # return False because there is no auth key
+        compare_digest(generate_access_token("fake"), auth_token)
+        compare_digest("fake", auth_token)
+        return False
+    else:
+        token_matched = compare_digest(generate_access_token(auth_key), auth_token)
+        # also compare key directly for backwards compatibility
+        key_matched = compare_digest(auth_key, auth_token)
+        return token_matched or key_matched
+
+
+def can_edit_submission(user, submission: Submission, access_token: str | None):
+    if  has_role(user, "Secretariat"):
+        return True
+    elif not access_token:
+        return False
+    return access_token_is_valid(submission, access_token)
+
 
 def submission_status(request, submission_id, access_token=None):
     # type: (HttpRequest, str, Optional[str]) -> HttpResponse
     submission = get_object_or_404(Submission, pk=submission_id)
 
-    key_matched = access_token and submission.access_token() == access_token
-    if not key_matched: key_matched = submission.access_key == access_token # backwards-compat
-    if access_token and not key_matched:
-        raise Http404
+    if access_token:
+        key_matched = access_token_is_valid(submission, access_token)
+        if not key_matched:
+            raise Http404
+    else:
+        key_matched = False
 
     if submission.state.slug == "cancel":
         errors = {}
@@ -621,8 +651,7 @@ def edit_submission(request, submission_id, access_token=None):
 def confirm_submission(request, submission_id, auth_token):
     submission = get_object_or_404(Submission, pk=submission_id)
 
-    key_matched = submission.auth_key and auth_token == generate_access_token(submission.auth_key)
-    if not key_matched: key_matched = auth_token == submission.auth_key # backwards-compat
+    key_matched = submission.auth_key and auth_token_is_valid(submission, auth_token)
 
     if request.method == 'POST' and submission.state_id in ("auth", "aut-appr") and key_matched:
         # Set a temporary state 'confirmed' to avoid entering this code

From 4be83ce312dde9b434f86cff928daf5882809239 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 19 Sep 2025 11:58:49 -0500
Subject: [PATCH 461/656] fix: remove deprecated iesg docs view, link from ad
 dashboard to ad view of working groups (#9563)

* fix: remove deprecated iesg docs view

* fix: link from ad dashboard to ad view of working groups
---
 ietf/doc/tests.py                             | 11 ---
 ietf/doc/urls.py                              |  4 +-
 ietf/doc/views_search.py                      | 27 +-----
 ietf/templates/doc/ad_list.html               |  5 +-
 .../templates/doc/drafts_in_iesg_process.html | 83 -------------------
 5 files changed, 7 insertions(+), 123 deletions(-)
 delete mode 100644 ietf/templates/doc/drafts_in_iesg_process.html

diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index fa8c7fa4fc..16dcfb7754 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -449,17 +449,6 @@ def test_drafts_in_last_call(self):
         self.assertContains(r, draft.title)
         self.assertContains(r, escape(draft.action_holders.first().name))
 
-    def test_in_iesg_process(self):
-        doc_in_process = IndividualDraftFactory()
-        doc_in_process.action_holders.set([PersonFactory()])
-        doc_in_process.set_state(State.objects.get(type='draft-iesg', slug='lc'))
-        doc_not_in_process = IndividualDraftFactory()
-        r = self.client.get(urlreverse('ietf.doc.views_search.drafts_in_iesg_process'))
-        self.assertEqual(r.status_code, 200)
-        self.assertContains(r, doc_in_process.title)
-        self.assertContains(r, escape(doc_in_process.action_holders.first().name))
-        self.assertNotContains(r, doc_not_in_process.title)
-        
     def test_indexes(self):
         draft = IndividualDraftFactory()
         rfc = WgRfcFactory()
diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py
index 6f1b698a9f..7b444782d7 100644
--- a/ietf/doc/urls.py
+++ b/ietf/doc/urls.py
@@ -53,13 +53,13 @@
     url(r'^ad/?$', views_search.ad_workload),
     url(r'^ad/(?P<name>[^/]+)/?$', views_search.docs_for_ad),
     url(r'^ad2/(?P<name>[\w.-]+)/$', RedirectView.as_view(url='/doc/ad/%(name)s/', permanent=True)),
-    url(r'^for_iesg/?$', views_search.docs_for_iesg),
+    url(r'^for_iesg/?$', RedirectView.as_view(pattern_name='ietf.doc.views_search.docs_for_iesg', permanent=False)),
     url(r'^rfc-status-changes/?$', views_status_change.rfc_status_changes),
     url(r'^start-rfc-status-change/(?:%(name)s/)?$' % settings.URL_REGEXPS, views_status_change.start_rfc_status_change),
     url(r'^bof-requests/?$', views_bofreq.bof_requests),
     url(r'^bof-requests/new/$', views_bofreq.new_bof_request),
     url(r'^statement/new/$', views_statement.new_statement),
-    url(r'^iesg/?$', views_search.drafts_in_iesg_process),
+    url(r'^iesg/?$', views_search.docs_for_iesg),
     url(r'^email-aliases/?$', views_doc.email_aliases),
     url(r'^downref/?$', views_downref.downref_registry),
     url(r'^downref/add/?$', views_downref.downref_registry_add),
diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py
index 67ff0c2f21..2144c23e06 100644
--- a/ietf/doc/views_search.py
+++ b/ietf/doc/views_search.py
@@ -59,7 +59,7 @@
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import ( Document, DocHistory, State,
-    LastCallDocEvent, NewRevisionDocEvent, IESG_SUBSTATE_TAGS,
+    NewRevisionDocEvent, IESG_SUBSTATE_TAGS,
     IESG_BALLOT_ACTIVE_STATES, IESG_STATCHG_CONFLREV_ACTIVE_STATES,
     IESG_CHARTER_ACTIVE_STATES )
 from ietf.doc.fields import select2_id_doc_name_json
@@ -849,31 +849,6 @@ def drafts_in_last_call(request):
         'form':form, 'docs':results, 'meta':meta, 'pages':pages
     })
 
-def drafts_in_iesg_process(request):
-    states = State.objects.filter(type="draft-iesg").exclude(slug__in=('idexists', 'pub', 'dead', 'rfcqueue'))
-    title = "Documents in IESG process"
-
-    grouped_docs = []
-
-    for s in states.order_by("order"):
-        docs = Document.objects.filter(type="draft", states=s).distinct().order_by("time").select_related("ad", "group", "group__parent")
-        if docs:
-            if s.slug == "lc":
-                for d in docs:
-                    e = d.latest_event(LastCallDocEvent, type="sent_last_call")
-                    # If we don't have an event, use an arbitrary date in the past (but not datetime.datetime.min,
-                    # which causes problems with timezone conversions)
-                    d.lc_expires = e.expires if e else datetime.datetime(1950, 1, 1)
-                docs = list(docs)
-                docs.sort(key=lambda d: d.lc_expires)
-
-            grouped_docs.append((s, docs))
-
-    return render(request, 'doc/drafts_in_iesg_process.html', {
-            "grouped_docs": grouped_docs,
-            "title": title,
-            })
-
 def recent_drafts(request, days=7):
     slowcache = caches['slowpages']
     cache_key = f'recentdraftsview{days}'
diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html
index 7f7e95a873..cac709021e 100644
--- a/ietf/templates/doc/ad_list.html
+++ b/ietf/templates/doc/ad_list.html
@@ -33,7 +33,10 @@ <h1>IESG Dashboard</h1>
         are only shown to logged-in Area Directors.
     </div>
     {% endif %}
-    <p><a href="{% url 'ietf.doc.views_search.docs_for_iesg' %}">Documents in IESG Processing</a></p>
+    <p>
+        <a class="btn btn-primary" href="{% url 'ietf.doc.views_search.docs_for_iesg' %}">Documents in IESG Processing</a>
+        <a class="btn btn-primary" href="{% url 'ietf.iesg.views.working_groups' %}">IESG view of Working Groups</a>
+    </p>
     {% for dt in metadata %}
         <h2 class="mt-5" id="{{ dt.type.0 }}">{{ dt.type.1 }} State Counts</h2>
         <table class="table table-sm table-striped table-bordered tablesorter navskip">
diff --git a/ietf/templates/doc/drafts_in_iesg_process.html b/ietf/templates/doc/drafts_in_iesg_process.html
deleted file mode 100644
index d9b09e984e..0000000000
--- a/ietf/templates/doc/drafts_in_iesg_process.html
+++ /dev/null
@@ -1,83 +0,0 @@
-{% extends "base.html" %}
-{# Copyright The IETF Trust 2015, All Rights Reserved #}
-{% load origin %}
-{% load ietf_filters static %}
-{% load textfilters person_filters %}
-{% block pagehead %}
-    <link rel="stylesheet" href="{% static "ietf/css/list.css" %}">
-{% endblock %}
-{% block title %}{{ title }}{% endblock %}
-{% block content %}
-    {% origin %}
-    <h1>{{ title }}</h1>
-    <h2><i class="bi bi-exclamation-triangle"></i>This view is deprecated, and will soon redirect to a <a href="{% url 'ietf.doc.views_search.docs_for_iesg' %}">different representation</a></h2>
-    <table class="table table-sm table-striped tablesorter">
-        <thead>
-            <tr>
-                <th scope="col" data-sort="area">Area</th>
-                <th scope="col" data-sort="date">
-                    {% if state.slug == "lc" %}
-                        Expires at
-                    {% else %}
-                        Date
-                    {% endif %}
-                </th>
-                <th scope="col" data-sort="doc">Document</th>
-                <th scope="col" data-sort="level">Intended level</th>
-                <th scope="col" data-sort="ad">AD</th>
-            </tr>
-        </thead>
-        {% for state, docs in grouped_docs %}
-            <tbody>
-                <tr class="table-info">
-                    <th scope="col" colspan="5">{{ state.name }}</th>
-                </tr>
-            </tbody>
-            <tbody>
-                {% for doc in docs %}
-                    <tr>
-                        <td>
-                            {% if doc.area_acronym %}
-                                <a href="{% url "ietf.group.views.group_home" acronym=doc.area_acronym %}">{{ doc.area_acronym }}</a>
-                            {% endif %}
-                        </td>
-                        <td>
-                            {% if state.slug == "lc" %}
-                                {% if doc.lc_expires %}{{ doc.lc_expires|date:"Y-m-d" }}{% endif %}
-                            {% else %}
-                                {{ doc.time|date:"Y-m-d" }}
-                            {% endif %}
-                        </td>
-                        <td>
-                            <a href="{% url "ietf.doc.views_doc.document_main" doc.name %}">{{ doc.name }}</a>
-                            <br>
-                            <b>{{ doc.title }}</b>
-                            {% if doc.action_holders_enabled and doc.action_holders.exists %}
-                                <br>
-                                Action holder{{ doc.documentactionholder_set.all|pluralize }}:
-                                {% for action_holder in doc.documentactionholder_set.all %}
-                                    {% person_link action_holder.person title=action_holder.role_for_doc %}{% if action_holder|action_holder_badge %} {{ action_holder|action_holder_badge }}{% endif %}{% if not forloop.last %},{% endif %}
-                                {% endfor %}
-                            {% endif %}
-                            {% if doc.note %}
-                                <br>
-                                <i>Note: {{ doc.note|urlize_ietf_docs|linkify|linebreaksbr }}</i>
-                            {% endif %}
-                        </td>
-                        <td>
-                            {% if doc.intended_std_level %}
-                                {{ doc.intended_std_level.name }}
-                            {% else %}
-                                <span class="text-body-secondary">(None)</span>
-                            {% endif %}
-                        </td>
-                        <td>{% person_link doc.ad %}</td>
-                    </tr>
-                {% endfor %}
-            </tbody>
-        {% endfor %}
-    </table>
-{% endblock %}
-{% block js %}
-    <script src="{% static "ietf/js/list.js" %}"></script>
-{% endblock %}
\ No newline at end of file

From 5e1f46d05cc23faa95b741f9133f40fe58c1cd46 Mon Sep 17 00:00:00 2001
From: Eric Vyncke <eric@vyncke.org>
Date: Mon, 29 Sep 2025 15:47:23 +0200
Subject: [PATCH 462/656] feat: Distinguish I-Ds on WG plate from I-Ds on IESG
 plate (#9214)

* Add "Outside of the WG Internet-Draft" when IESG state != idexists

* No plural forms in the dividers

* Use different search_heading

* Use the right stream_id

* Adding tests_info coverage for prepare_group_documents

* fix: move identifying and sorting doxs with IESG into search utility.

* fix: improve ordering conditional

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/doc/utils_search.py |  9 ++++++++-
 ietf/group/tests_info.py | 22 +++++++++++++++++++++-
 ietf/group/views.py      |  1 -
 3 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py
index cfc8a872f8..a5f461f9bb 100644
--- a/ietf/doc/utils_search.py
+++ b/ietf/doc/utils_search.py
@@ -108,7 +108,10 @@ def fill_in_document_table_attributes(docs, have_telechat_date=False):
                 d.search_heading = "Withdrawn Internet-Draft"
                 d.expirable = False
             else:
-                d.search_heading = "%s Internet-Draft" % d.get_state()
+                if d.type_id == "draft" and d.stream_id == 'ietf' and d.get_state_slug('draft-iesg') != 'idexists': # values can be: ad-eval idexists approved rfcqueue dead iesg-eva
+                    d.search_heading = "%s with the IESG Internet-Draft" % d.get_state()
+                else:
+                    d.search_heading = "%s Internet-Draft" % d.get_state()
                 if state_slug == "active":
                     d.expirable = d.pk in expirable_pks
                 else:
@@ -221,6 +224,10 @@ def num(i):
 
         if d.type_id == "draft":
             res.append(num(["Active", "Expired", "Replaced", "Withdrawn", "RFC"].index(d.search_heading.split()[0])))
+            if "with the IESG" in d.search_heading:
+                res.append("1")
+            else:
+                res.append("0")
         else:
             res.append(d.type_id);
             res.append("-");
diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py
index eb85860ece..34f8500854 100644
--- a/ietf/group/tests_info.py
+++ b/ietf/group/tests_info.py
@@ -27,7 +27,7 @@
 
 from ietf.community.models import CommunityList
 from ietf.community.utils import reset_name_contains_index_for_rule
-from ietf.doc.factories import WgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory
+from ietf.doc.factories import WgDraftFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory
 from ietf.doc.models import Document, DocEvent, State
 from ietf.doc.storage_utils import retrieve_str
 from ietf.doc.utils_charter import charter_name_for_group
@@ -413,6 +413,7 @@ def test_group_documents(self):
             self.assertContains(r, draft3.name)
             for ah in draft3.action_holders.all():
                 self.assertContains(r, escape(ah.name))
+            self.assertContains(r, "Active with the IESG Internet-Draft") # draft3 is pub-req hence should have such a divider
             self.assertContains(r, 'for 173 days', count=1)  # the old_dah should be tagged
             self.assertContains(r, draft4.name)
             self.assertNotContains(r, draft5.name)
@@ -425,6 +426,25 @@ def test_group_documents(self):
         q = PyQuery(r.content)
         self.assertTrue(any([draft2.name in x.attrib['href'] for x in q('table td a.track-untrack-doc')]))
 
+        # Let's also check the IRTF stream
+        rg = GroupFactory(type_id='rg')
+        setup_default_community_list_for_group(rg)
+        rgDraft = RgDraftFactory(group=rg)
+        rgDraft4 = RgDraftFactory(group=rg)
+        rgDraft4.set_state(State.objects.get(slug='irsg-w'))
+        rgDraft7 = RgDraftFactory(group=rg)
+        rgDraft7.set_state(State.objects.get(type='draft-stream-%s' % rgDraft7.stream_id, slug='dead'))
+        for url in group_urlreverse_list(rg, 'ietf.group.views.group_documents'):
+            with self.settings(DOC_ACTION_HOLDER_MAX_AGE_DAYS=20):
+                r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            self.assertContains(r, rgDraft.name)
+            self.assertContains(r, rg.name)
+            self.assertContains(r, rg.acronym)
+            self.assertNotContains(r, draft3.name) # As draft3 is a WG draft, it should not be listed here
+            self.assertContains(r, rgDraft4.name)
+            self.assertNotContains(r, rgDraft7.name)
+
         # test the txt version too while we're at it
         for url in group_urlreverse_list(group, 'ietf.group.views.group_documents_txt'):
             r = self.client.get(url)
diff --git a/ietf/group/views.py b/ietf/group/views.py
index bc785ff81e..efe3eca15d 100644
--- a/ietf/group/views.py
+++ b/ietf/group/views.py
@@ -443,7 +443,6 @@ def prepare_group_documents(request, group, clist):
 
     return docs, meta, docs_related, meta_related
 
-
 def get_leadership(group_type):
     people = Person.objects.filter(
         role__name__slug="chair",

From ba8b73190df413c39deaa6b546ad2bc5405fd86c Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 30 Sep 2025 13:40:33 -0300
Subject: [PATCH 463/656] ci: DB persistence for blobdb, too

---
 k8s/settings_local.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index c1436e158b..c09bd70c86 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -114,15 +114,17 @@ def _multiline_to_list(s):
 
 # Configure persistent connections. A setting of 0 is Django's default.
 _conn_max_age = os.environ.get("DATATRACKER_DB_CONN_MAX_AGE", "0")
-# A string "none" means unlimited age.
-DATABASES["default"]["CONN_MAX_AGE"] = (
-    None if _conn_max_age.lower() == "none" else int(_conn_max_age)
-)
+for dbname in ["default", "blobdb"]:
+    # A string "none" means unlimited age.
+    DATABASES[dbname]["CONN_MAX_AGE"] = (
+        None if _conn_max_age.lower() == "none" else int(_conn_max_age)
+    )
 # Enable connection health checks if DATATRACKER_DB_CONN_HEALTH_CHECK is the string "true"
 _conn_health_checks = bool(
     os.environ.get("DATATRACKER_DB_CONN_HEALTH_CHECKS", "false").lower() == "true"
 )
-DATABASES["default"]["CONN_HEALTH_CHECKS"] = _conn_health_checks
+for dbname in ["default", "blobdb"]:
+    DATABASES[dbname]["CONN_HEALTH_CHECKS"] = _conn_health_checks
 
 # DATATRACKER_ADMINS is a newline-delimited list of addresses parseable by email.utils.parseaddr
 _admins_str = os.environ.get("DATATRACKER_ADMINS", None)

From d1cbdcb2afca5987706165a1928fece3da25a5ee Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 7 Oct 2025 14:54:08 -0300
Subject: [PATCH 464/656] chore: fix docker-compose comment (#9679)

Allows the commented-out options to work if uncommented.
---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 100119c464..8c6e0ea486 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -36,7 +36,7 @@ services:
     db:
         image: ghcr.io/ietf-tools/datatracker-db:latest
         #  build:
-        #      context: ..
+        #      context: .
         #      dockerfile: docker/db.Dockerfile
         restart: unless-stopped
         volumes:

From a8e8b9e95bedececcda0f54bedbe4de8f69d90a2 Mon Sep 17 00:00:00 2001
From: Absit Iniuria <absniuria@nours.page>
Date: Tue, 7 Oct 2025 19:02:37 +0100
Subject: [PATCH 465/656] feat: split liaison_statement_posted mailtrigger into
 outgoing and incoming (#9553)

* fix: add new fixtures and mt slugs

* fix: edit mt reverse func

* chore: edit multiline and hash comments

* fix: adjust migration

* chore: remove stray whitespace

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/liaisons/mails.py                        |  5 +-
 ietf/liaisons/tests.py                        | 39 +++++-----
 ietf/liaisons/views.py                        | 76 ++-----------------
 ..._statement_incoming_and_outgoing_posted.py | 72 ++++++++++++++++++
 ietf/mailtrigger/utils.py                     | 71 +++++++++++++++++
 ietf/name/fixtures/names.json                 | 45 ++++++++++-
 6 files changed, 217 insertions(+), 91 deletions(-)
 create mode 100644 ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py

diff --git a/ietf/liaisons/mails.py b/ietf/liaisons/mails.py
index 8708c8a078..878aada576 100644
--- a/ietf/liaisons/mails.py
+++ b/ietf/liaisons/mails.py
@@ -14,7 +14,10 @@
 def send_liaison_by_email(request, liaison):
     subject = 'New Liaison Statement, "%s"' % (liaison.title)
     from_email = settings.LIAISON_UNIVERSAL_FROM
-    (to_email, cc) = gather_address_lists('liaison_statement_posted',liaison=liaison)
+    if liaison.is_outgoing():
+        (to_email, cc) = gather_address_lists('liaison_statement_posted_outgoing',liaison=liaison)
+    else:
+        (to_email, cc) = gather_address_lists('liaison_statement_posted_incoming',liaison=liaison)
     bcc = ['statements@ietf.org']
     body = render_to_string('liaisons/liaison_mail.txt', dict(liaison=liaison))
 
diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py
index fd1c22be77..5478f6c302 100644
--- a/ietf/liaisons/tests.py
+++ b/ietf/liaisons/tests.py
@@ -112,61 +112,61 @@ def test_help_pages(self):
 
 
 class UnitTests(TestCase):
-    def test_get_cc(self):
-        from ietf.liaisons.views import get_cc,EMAIL_ALIASES
+    def test_get_contacts_for_liaison_messages_for_group_primary(self):
+        from ietf.mailtrigger.utils import get_contacts_for_liaison_messages_for_group_primary,EMAIL_ALIASES
 
         # test IETF
-        cc = get_cc(Group.objects.get(acronym='ietf'))
+        cc = get_contacts_for_liaison_messages_for_group_primary(Group.objects.get(acronym='ietf'))
         self.assertTrue(EMAIL_ALIASES['IESG'] in cc)
         self.assertTrue(EMAIL_ALIASES['IETFCHAIR'] in cc)
         # test IAB
-        cc = get_cc(Group.objects.get(acronym='iab'))
+        cc = get_contacts_for_liaison_messages_for_group_primary(Group.objects.get(acronym='iab'))
         self.assertTrue(EMAIL_ALIASES['IAB'] in cc)
         self.assertTrue(EMAIL_ALIASES['IABCHAIR'] in cc)
         # test an Area
         area = Group.objects.filter(type='area').first()
-        cc = get_cc(area)
+        cc = get_contacts_for_liaison_messages_for_group_primary(area)
         self.assertTrue(EMAIL_ALIASES['IETFCHAIR'] in cc)
         self.assertTrue(contacts_from_roles([area.ad_role()]) in cc)
         # test a Working Group
         wg = Group.objects.filter(type='wg').first()
-        cc = get_cc(wg)
+        cc = get_contacts_for_liaison_messages_for_group_primary(wg)
         self.assertTrue(contacts_from_roles([wg.parent.ad_role()]) in cc)
         self.assertTrue(contacts_from_roles([wg.get_chair()]) in cc)
         # test an SDO
         sdo = RoleFactory(name_id='liaiman',group__type_id='sdo',).group
-        cc = get_cc(sdo)
+        cc = get_contacts_for_liaison_messages_for_group_primary(sdo)
         self.assertTrue(contacts_from_roles([sdo.role_set.filter(name='liaiman').first()]) in cc)
         # test a cc_contact role
         cc_contact_role = RoleFactory(name_id='liaison_cc_contact', group=sdo)
-        cc = get_cc(sdo)
+        cc = get_contacts_for_liaison_messages_for_group_primary(sdo)
         self.assertIn(contact_email_from_role(cc_contact_role), cc)
 
-    def test_get_contacts_for_group(self):
-        from ietf.liaisons.views import get_contacts_for_group, EMAIL_ALIASES
+    def test_get_contacts_for_liaison_messages_for_group_secondary(self):
+        from ietf.mailtrigger.utils import get_contacts_for_liaison_messages_for_group_secondary,EMAIL_ALIASES
 
-        # test explicit
+        # test explicit group contacts
         sdo = GroupFactory(type_id='sdo')
         contact_email = RoleFactory(name_id='liaison_contact', group=sdo).email.address
-        contacts = get_contacts_for_group(sdo)
+        contacts = get_contacts_for_liaison_messages_for_group_secondary(sdo)
         self.assertIsNotNone(contact_email)
         self.assertIn(contact_email, contacts)
         # test area
         area = Group.objects.filter(type='area').first()
-        contacts = get_contacts_for_group(area)
+        contacts = get_contacts_for_liaison_messages_for_group_secondary(area)
         self.assertTrue(area.ad_role().email.address in contacts)
         # test wg
         wg = Group.objects.filter(type='wg').first()
-        contacts = get_contacts_for_group(wg)
+        contacts = get_contacts_for_liaison_messages_for_group_secondary(wg)
         self.assertTrue(wg.get_chair().email.address in contacts)
         # test ietf
-        contacts = get_contacts_for_group(Group.objects.get(acronym='ietf'))
+        contacts = get_contacts_for_liaison_messages_for_group_secondary(Group.objects.get(acronym='ietf'))
         self.assertTrue(EMAIL_ALIASES['IETFCHAIR'] in contacts)
         # test iab
-        contacts = get_contacts_for_group(Group.objects.get(acronym='iab'))
+        contacts = get_contacts_for_liaison_messages_for_group_secondary(Group.objects.get(acronym='iab'))
         self.assertTrue(EMAIL_ALIASES['IABCHAIR'] in contacts)
         # test iesg
-        contacts = get_contacts_for_group(Group.objects.get(acronym='iesg'))
+        contacts = get_contacts_for_liaison_messages_for_group_secondary(Group.objects.get(acronym='iesg'))
         self.assertTrue(EMAIL_ALIASES['IESG'] in contacts)
 
     def test_needs_approval(self):
@@ -786,8 +786,11 @@ def test_add_incoming_liaison(self):
         self.assertTrue("Liaison Statement" in outbox[-1]["Subject"])
     
         self.assertTrue('to_contacts@' in outbox[-1]['To'])
+        self.assertTrue(submitter.email_address(), outbox[-1]['To'])
         self.assertTrue('cc@' in outbox[-1]['Cc'])
 
+
+
     def test_add_outgoing_liaison(self):
         RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman')
         wg = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group
@@ -867,6 +870,8 @@ def test_add_outgoing_liaison(self):
         self.assertEqual(len(outbox), mailbox_before + 1)
         self.assertTrue("Liaison Statement" in outbox[-1]["Subject"])
         self.assertTrue('aread@' in outbox[-1]['To'])
+        self.assertTrue(submitter.email_address(), outbox[-1]['Cc'])
+        
 
     def test_add_outgoing_liaison_unapproved_post_only(self):
         RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman')
diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py
index 6a6f579714..f54a023357 100644
--- a/ietf/liaisons/views.py
+++ b/ietf/liaisons/views.py
@@ -27,14 +27,6 @@
 from ietf.name.models import LiaisonStatementTagName
 from ietf.utils.response import permission_denied
 
-EMAIL_ALIASES = {
-    "IETFCHAIR": "The IETF Chair <chair@ietf.org>",
-    "IESG": "The IESG <iesg@ietf.org>",
-    "IAB": "The IAB <iab@iab.org>",
-    "IABCHAIR": "The IAB Chair <iab-chair@iab.org>",
-}
-
-
 # -------------------------------------------------
 # Helper Functions
 # -------------------------------------------------
@@ -94,64 +86,6 @@ def contacts_from_roles(roles):
     emails = [ contact_email_from_role(r) for r in roles ]
     return ','.join(emails)
 
-def get_cc(group):
-    '''Returns list of emails to use as CC for group.  Simplified refactor of IETFHierarchy
-    get_cc() and get_from_cc()
-    '''
-    emails = []
-
-    # role based CCs
-    if group.acronym in ('ietf','iesg'):
-        emails.append(EMAIL_ALIASES['IESG'])
-        emails.append(EMAIL_ALIASES['IETFCHAIR'])
-    elif group.acronym in ('iab'):
-        emails.append(EMAIL_ALIASES['IAB'])
-        emails.append(EMAIL_ALIASES['IABCHAIR'])
-    elif group.type_id == 'area':
-        emails.append(EMAIL_ALIASES['IETFCHAIR'])
-        ad_roles = group.role_set.filter(name='ad')
-        emails.extend([ contact_email_from_role(r) for r in ad_roles ])
-    elif group.type_id == 'wg':
-        ad_roles = group.parent.role_set.filter(name='ad')
-        emails.extend([ contact_email_from_role(r) for r in ad_roles ])
-        chair_roles = group.role_set.filter(name='chair')
-        emails.extend([ contact_email_from_role(r) for r in chair_roles ])
-        if group.list_email:
-            emails.append('{} Discussion List <{}>'.format(group.name,group.list_email))
-    elif group.type_id == 'sdo':
-        liaiman_roles = group.role_set.filter(name='liaiman')
-        emails.extend([ contact_email_from_role(r) for r in liaiman_roles ])
-
-    # explicit CCs
-    liaison_cc_roles = group.role_set.filter(name='liaison_cc_contact')
-    emails.extend([ contact_email_from_role(r) for r in liaison_cc_roles ])
-
-    return emails
-
-def get_contacts_for_group(group):
-    '''Returns default contacts for groups as a comma separated string'''
-    # use explicit default contacts if defined
-    explicit_contacts = contacts_from_roles(group.role_set.filter(name='liaison_contact'))
-    if explicit_contacts:
-        return explicit_contacts
-
-    # otherwise construct based on group type
-    contacts = []
-    if group.type_id == 'area':
-        roles = group.role_set.filter(name='ad')
-        contacts.append(contacts_from_roles(roles))
-    elif group.type_id == 'wg':
-        roles = group.role_set.filter(name='chair')
-        contacts.append(contacts_from_roles(roles))
-    elif group.acronym == 'ietf':
-        contacts.append(EMAIL_ALIASES['IETFCHAIR'])
-    elif group.acronym == 'iab':
-        contacts.append(EMAIL_ALIASES['IABCHAIR'])
-    elif group.acronym == 'iesg':
-        contacts.append(EMAIL_ALIASES['IESG'])
-
-    return ','.join(contacts)
-
 def get_details_tabs(stmt, selected):
     return [
         t + (t[0].lower() == selected.lower(),)
@@ -207,6 +141,8 @@ def post_only(group,person):
 # -------------------------------------------------
 @can_submit_liaison_required
 def ajax_get_liaison_info(request):
+    from ietf.mailtrigger.utils import get_contacts_for_liaison_messages_for_group_primary,get_contacts_for_liaison_messages_for_group_secondary
+
     '''Returns dictionary of info to update entry form given the groups
     that have been selected
     '''
@@ -229,14 +165,14 @@ def ajax_get_liaison_info(request):
     result = {'response_contacts':[],'to_contacts': [], 'cc': [], 'needs_approval': False, 'post_only': False, 'full_list': []}
 
     for group in from_groups:
-        cc.extend(get_cc(group))
+        cc.extend(get_contacts_for_liaison_messages_for_group_primary(group))
         does_need_approval.append(needs_approval(group,person))
         can_post_only.append(post_only(group,person))
-        response_contacts.append(get_contacts_for_group(group))
+        response_contacts.append(get_contacts_for_liaison_messages_for_group_secondary(group))
 
     for group in to_groups:
-        cc.extend(get_cc(group))
-        to_contacts.append(get_contacts_for_group(group))
+        cc.extend(get_contacts_for_liaison_messages_for_group_primary(group))
+        to_contacts.append(get_contacts_for_liaison_messages_for_group_secondary(group))
 
     # if there are from_groups and any need approval
     if does_need_approval:
diff --git a/ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py b/ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py
new file mode 100644
index 0000000000..189a783a2e
--- /dev/null
+++ b/ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py
@@ -0,0 +1,72 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    Mailtrigger = apps.get_model("mailtrigger", "MailTrigger")
+    Recipient = apps.get_model("mailtrigger", "Recipient")
+    recipients_to = Recipient.objects.get(pk="liaison_to_contacts")
+    recipients_cc = list(
+        Recipient.objects.filter(
+            slug__in=(
+                "liaison_cc",
+                "liaison_coordinators",
+                "liaison_response_contacts",
+                "liaison_technical_contacts",
+            )
+        )
+    )
+    recipient_from = Recipient.objects.get(pk="liaison_from_contact")
+
+    liaison_posted_outgoing = Mailtrigger.objects.create(
+        slug="liaison_statement_posted_outgoing",
+        desc="Recipients for a message when a new outgoing liaison statement is posted",
+    )
+    liaison_posted_outgoing.to.add(recipients_to)
+    liaison_posted_outgoing.cc.add(*recipients_cc)
+    liaison_posted_outgoing.cc.add(recipient_from)
+
+    liaison_posted_incoming = Mailtrigger.objects.create(
+        slug="liaison_statement_posted_incoming",
+        desc="Recipients for a message when a new incoming liaison statement is posted",
+    )
+    liaison_posted_incoming.to.add(recipients_to)
+    liaison_posted_incoming.cc.add(*recipients_cc)
+
+    Mailtrigger.objects.filter(slug=("liaison_statement_posted")).delete()
+
+
+def reverse(apps, schema_editor):
+    Mailtrigger = apps.get_model("mailtrigger", "MailTrigger")
+    Recipient = apps.get_model("mailtrigger", "Recipient")
+
+    Mailtrigger.objects.filter(
+        slug__in=(
+            "liaison_statement_posted_outgoing",
+            "liaison_statement_posted_incoming",
+        )
+    ).delete()
+
+    liaison_statement_posted = Mailtrigger.objects.create(
+        slug="liaison_statement_posted",
+        desc="Recipients for a message when a new liaison statement is posted",
+    )
+
+    liaison_to_contacts = Recipient.objects.get(slug="liaison_to_contacts")
+    recipients_ccs = Recipient.objects.filter(
+        slug__in=(
+            "liaison_cc",
+            "liaison_coordinators",
+            "liaison_response_contacts",
+            "liaison_technical_contacts",
+        )
+    )
+    liaison_statement_posted.to.add(liaison_to_contacts)
+    liaison_statement_posted.cc.add(*recipients_ccs)
+
+
+class Migration(migrations.Migration):
+    dependencies = [("mailtrigger", "0007_historicalrecipient_historicalmailtrigger")]
+
+    operations = [migrations.RunPython(forward, reverse)]
diff --git a/ietf/mailtrigger/utils.py b/ietf/mailtrigger/utils.py
index 9915eae3fd..bcdaf5e44e 100644
--- a/ietf/mailtrigger/utils.py
+++ b/ietf/mailtrigger/utils.py
@@ -9,6 +9,14 @@
 from ietf.utils.mail import excludeaddrs
 
 
+EMAIL_ALIASES = {
+    "IETFCHAIR": "The IETF Chair <chair@ietf.org>",
+    "IESG": "The IESG <iesg@ietf.org>",
+    "IAB": "The IAB <iab@iab.org>",
+    "IABCHAIR": "The IAB Chair <iab-chair@iab.org>",
+}
+
+
 class AddrLists(namedtuple("AddrLists", ["to", "cc"])):
     __slots__ = ()
 
@@ -66,6 +74,69 @@ def get_mailtrigger(slug, create_from_slug_if_not_exists, desc_if_not_exists):
     return mailtrigger
 
 
+def get_contacts_for_liaison_messages_for_group_primary(group):
+    from ietf.liaisons.views import contact_email_from_role
+
+    '''Returns list of emails to use in liaison message for group
+    '''
+    emails = []
+
+    # role based emails
+    if group.acronym in ('ietf','iesg'):
+        emails.append(EMAIL_ALIASES['IESG'])
+        emails.append(EMAIL_ALIASES['IETFCHAIR'])
+    elif group.acronym in ('iab'):
+        emails.append(EMAIL_ALIASES['IAB'])
+        emails.append(EMAIL_ALIASES['IABCHAIR'])
+    elif group.type_id == 'area':
+        emails.append(EMAIL_ALIASES['IETFCHAIR'])
+        ad_roles = group.role_set.filter(name='ad')
+        emails.extend([ contact_email_from_role(r) for r in ad_roles ])
+    elif group.type_id == 'wg':
+        ad_roles = group.parent.role_set.filter(name='ad')
+        emails.extend([ contact_email_from_role(r) for r in ad_roles ])
+        chair_roles = group.role_set.filter(name='chair')
+        emails.extend([ contact_email_from_role(r) for r in chair_roles ])
+        if group.list_email:
+            emails.append('{} Discussion List <{}>'.format(group.name,group.list_email))
+    elif group.type_id == 'sdo':
+        liaiman_roles = group.role_set.filter(name='liaiman')
+        emails.extend([ contact_email_from_role(r) for r in liaiman_roles ])
+
+    # explicit CCs
+    liaison_cc_roles = group.role_set.filter(name='liaison_cc_contact')
+    emails.extend([ contact_email_from_role(r) for r in liaison_cc_roles ])
+
+    return emails
+
+
+def get_contacts_for_liaison_messages_for_group_secondary(group):
+    from ietf.liaisons.views import contacts_from_roles
+
+    '''Returns default contacts for groups as a comma separated string'''
+    # use explicit default contacts if defined
+    explicit_contacts = contacts_from_roles(group.role_set.filter(name='liaison_contact'))
+    if explicit_contacts:
+        return explicit_contacts
+
+    # otherwise construct based on group type
+    contacts = []
+    if group.type_id == 'area':
+        roles = group.role_set.filter(name='ad')
+        contacts.append(contacts_from_roles(roles))
+    elif group.type_id == 'wg':
+        roles = group.role_set.filter(name='chair')
+        contacts.append(contacts_from_roles(roles))
+    elif group.acronym == 'ietf':
+        contacts.append(EMAIL_ALIASES['IETFCHAIR'])
+    elif group.acronym == 'iab':
+        contacts.append(EMAIL_ALIASES['IABCHAIR'])
+    elif group.acronym == 'iesg':
+        contacts.append(EMAIL_ALIASES['IESG'])
+
+    return ','.join(contacts)
+
+
 def gather_relevant_expansions(**kwargs):
     def starts_with(prefix):
         return MailTrigger.objects.filter(slug__startswith=prefix).values_list(
diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json
index c94e15a459..58deb01f0c 100644
--- a/ietf/name/fixtures/names.json
+++ b/ietf/name/fixtures/names.json
@@ -2638,11 +2638,24 @@
       "order": 0,
       "slug": "historic",
       "type": "statement",
-      "used": true
+      "used": false
     },
     "model": "doc.state",
     "pk": 182
   },
+  {
+    "fields": {
+      "desc": "The statement is no longer active",
+      "name": "Inactive",
+      "next_states": [],
+      "order": 0,
+      "slug": "inactive",
+      "type": "statement",
+      "used": true
+    },
+    "model": "doc.state",
+    "pk": 183
+  },
   {
     "fields": {
       "label": "State"
@@ -5520,13 +5533,31 @@
         "liaison_response_contacts",
         "liaison_technical_contacts"
       ],
-      "desc": "Recipient for a message when a new liaison statement is posted",
+      "desc": "Recipients for a message when a new incoming liaison statement is posted",
       "to": [
+        "liaison_from_contact",
         "liaison_to_contacts"
       ]
     },
     "model": "mailtrigger.mailtrigger",
-    "pk": "liaison_statement_posted"
+    "pk": "liaison_statement_posted_incoming"
+  },
+  {
+    "fields": {
+      "cc": [
+        "liaison_cc",
+        "liaison_coordinators",
+        "liaison_from_contact",
+        "liaison_response_contacts",
+        "liaison_technical_contacts"
+      ],
+      "desc": "Recipients for a message when a new outgoing liaison statement is posted",
+      "to": [
+        "liaison_to_contacts"
+      ]
+    },
+    "model": "mailtrigger.mailtrigger",
+    "pk": "liaison_statement_posted_outgoing"
   },
   {
     "fields": {
@@ -7068,6 +7099,14 @@
     "model": "mailtrigger.recipient",
     "pk": "liaison_coordinators"
   },
+  {
+    "fields": {
+      "desc": "Email address of the formal sender of the statement",
+      "template": "{{liaison.from_contact}}"
+    },
+    "model": "mailtrigger.recipient",
+    "pk": "liaison_from_contact"
+  },
   {
     "fields": {
       "desc": "The assigned liaison manager for an external group ",

From 8fbbc55ec3cb87f528953da33e8c7194c2b75afd Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 7 Oct 2025 15:13:16 -0300
Subject: [PATCH 466/656] fix: keep day visible in timeslot editor (#9653)

---
 ietf/templates/meeting/timeslot_edit.html | 44 ++++++++++++-----------
 1 file changed, 24 insertions(+), 20 deletions(-)

diff --git a/ietf/templates/meeting/timeslot_edit.html b/ietf/templates/meeting/timeslot_edit.html
index 11691ba6dd..3259dba9da 100644
--- a/ietf/templates/meeting/timeslot_edit.html
+++ b/ietf/templates/meeting/timeslot_edit.html
@@ -11,20 +11,22 @@
     {% endcomment %}
     .timeslot-edit { overflow: auto; height: max(30rem, calc(100vh - 25rem));}
     .tstable { width: 100%; border-collapse: separate; } {# "separate" to ensure sticky cells keep their borders #}
-.tstable thead  { position: sticky; top: 0; z-index: 3; background-color: white;}
-.tstable th:first-child, .tstable td:first-child {
-    background-color: white; {# needs to match the lighter of the striped-table colors! #}
-position: sticky;
-left: 0;
-    z-index: 2; {# render above other cells / borders but below thead (z-index 3, above) #}
-}
-.tstable tbody > tr:nth-of-type(odd) > th:first-child {
-    background-color: rgb(249, 249, 249); {# needs to match the darker of the striped-table colors! #}
-}
-.tstable th { white-space: nowrap;}
-.tstable td { white-space: nowrap;}
-.capacity { font-size:80%; font-weight: normal;}
-a.new-timeslot-link { color: lightgray; font-size: large;}
+    .tstable tr th:first-child { min-width: 25rem; max-width: 25rem; overflow: hidden; } 
+    .tstable thead  { position: sticky; top: 0; z-index: 3; background-color: white;}
+    .tstable thead th span.day { position: sticky; left: 25.5rem; }
+    .tstable th:first-child, .tstable td:first-child {
+        background-color: white; {# needs to match the lighter of the striped-table colors! #}
+        position: sticky;
+        left: 0;
+        z-index: 2; {# render above other cells / borders but below thead (z-index 3, above) #}
+    }
+    .tstable tbody > tr:nth-of-type(odd) > th:first-child {
+        background-color: rgb(249, 249, 249); {# needs to match the darker of the striped-table colors! #}
+    }
+    .tstable th { white-space: nowrap;}
+    .tstable td { white-space: nowrap;}
+    .capacity { font-size:80%; font-weight: normal;}
+    a.new-timeslot-link { color: lightgray; font-size: large;}
 {% endblock %}
 {% block content %}
     {% origin %}
@@ -84,12 +86,14 @@ <h1>
                                 <th scope="col"></th>
                                 {% for day in time_slices %}
                                     <th scope="col" class="day-label" colspan="{{ date_slices|colWidth:day }}">
-                                        {{ day|date:'D' }} ({{ day }})
-                                        <i class="bi bi-trash delete-button"
-                                              title="Delete all on this day"
-                                              data-delete-scope="day"
-                                              data-date-id="{{ day.isoformat }}">
-                                        </i>
+                                        <span class="day">
+                                            {{ day|date:'D' }} ({{ day }})
+                                            <i class="bi bi-trash delete-button"
+                                               title="Delete all on this day"
+                                               data-delete-scope="day"
+                                               data-date-id="{{ day.isoformat }}">
+                                            </i>
+                                        </span>
                                     </th>
                                 {% endfor %}
                             {% endif %}

From 24101bb8ca85cfb3a5c47d7f9ed283cc6fb5bc0e Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 9 Oct 2025 13:49:40 -0500
Subject: [PATCH 467/656] feat: json snapshots of ipr statements (#9684)

---
 ietf/ipr/tests.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++
 ietf/ipr/urls.py  |  1 +
 ietf/ipr/utils.py | 24 ++++++++++++++++++--
 ietf/ipr/views.py |  7 +++++-
 4 files changed, 86 insertions(+), 3 deletions(-)

diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py
index 74fa540126..4146fbd4c1 100644
--- a/ietf/ipr/tests.py
+++ b/ietf/ipr/tests.py
@@ -3,6 +3,7 @@
 
 
 import datetime
+import json
 from unittest import mock
 import re
 
@@ -15,6 +16,8 @@
 from django.urls import reverse as urlreverse
 from django.utils import timezone
 
+from django.db.models import Max
+
 import debug                            # pyflakes:ignore
 
 from ietf.api.views import EmailIngestionError
@@ -45,6 +48,7 @@
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.message.factories import MessageFactory
 from ietf.message.models import Message
+from ietf.person.factories import PersonFactory
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized
 from ietf.utils.text import text_to_dict
@@ -1113,3 +1117,56 @@ def test_patent_details_required_unless_blanket(self):
             val = self.data.pop(pf)
             self.assertTrue(HolderIprDisclosureForm(data=self.data).is_valid())
             self.data[pf] = val
+
+class JsonSnapshotTests(TestCase):
+    def test_json_snapshot(self):
+        h = HolderIprDisclosureFactory()
+        url = urlreverse("ietf.ipr.views.json_snapshot", kwargs=dict(id=h.id))
+        login_testing_unauthorized(self, "secretary", url)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        dump = json.loads(r.content)
+        self.assertCountEqual(
+            [o["model"] for o in dump],
+            ["ipr.holderiprdisclosure", "ipr.iprdisclosurebase", "person.person"],
+        )
+        h.docs.add(WgRfcFactory())
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        dump = json.loads(r.content)
+        self.assertCountEqual(
+            [o["model"] for o in dump],
+            [
+                "ipr.holderiprdisclosure",
+                "ipr.iprdisclosurebase",
+                "ipr.iprdocrel",
+                "person.person",
+            ],
+        )
+        IprEventFactory(
+            disclosure=h,
+            message=MessageFactory(by=PersonFactory()),
+            in_reply_to=MessageFactory(),
+        )
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        dump = json.loads(r.content)
+        self.assertCountEqual(
+            [o["model"] for o in dump],
+            [
+                "ipr.holderiprdisclosure",
+                "ipr.iprdisclosurebase",
+                "ipr.iprdocrel",
+                "ipr.iprevent",
+                "message.message",
+                "message.message",
+                "person.person",
+                "person.person",
+                "person.person",
+                "person.person",
+            ],
+        )
+        no_such_ipr_id = IprDisclosureBase.objects.aggregate(Max("id"))["id__max"] + 1
+        url = urlreverse("ietf.ipr.views.json_snapshot", kwargs=dict(id=no_such_ipr_id))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
diff --git a/ietf/ipr/urls.py b/ietf/ipr/urls.py
index 84ed04a66b..2c8a26c624 100644
--- a/ietf/ipr/urls.py
+++ b/ietf/ipr/urls.py
@@ -21,6 +21,7 @@
     url(r'^(?P<id>\d+)/notify/(?P<type>update|posted)/$', views.notify),
     url(r'^(?P<id>\d+)/post/$', views.post),
     url(r'^(?P<id>\d+)/state/$', views.state),
+    url(r'^(?P<id>\d+)/json-snapshot/$', views.json_snapshot),
     url(r'^update/$', RedirectView.as_view(url=reverse_lazy('ietf.ipr.views.showlist'), permanent=True)),
     url(r'^update/(?P<id>\d+)/$', views.update),
     url(r'^new-(?P<_type>(specific|generic|general|third-party))/$', views.new),
diff --git a/ietf/ipr/utils.py b/ietf/ipr/utils.py
index 7e569a1d1d..bcbb052260 100644
--- a/ietf/ipr/utils.py
+++ b/ietf/ipr/utils.py
@@ -1,11 +1,16 @@
-# Copyright The IETF Trust 2014-2020, All Rights Reserved
+# Copyright The IETF Trust 2014-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
+import json
+import debug # pyflakes:ignore
+
 from textwrap import dedent
 
+from django.core import serializers
+
 from ietf.ipr.mail import process_response_email, UndeliverableIprResponseError
 
-import debug                            # pyflakes:ignore
+from ietf.ipr.models import IprDocRel
 
 def get_genitive(name):
     """Return the genitive form of name"""
@@ -85,3 +90,18 @@ def ingest_response_email(message: bytes):
             email_original_message=message,
             email_attach_traceback=True,
         ) from err
+
+def json_dump_disclosure(disclosure):
+    objs = set()
+    objs.add(disclosure)
+    objs.add(disclosure.iprdisclosurebase_ptr)
+    objs.add(disclosure.by)
+    objs.update(IprDocRel.objects.filter(disclosure=disclosure))
+    objs.update(disclosure.iprevent_set.all())
+    objs.update([i.by for i in disclosure.iprevent_set.all()])
+    objs.update([i.message for i in disclosure.iprevent_set.all() if i.message ])
+    objs.update([i.message.by for i in disclosure.iprevent_set.all() if i.message ])
+    objs.update([i.in_reply_to for i in disclosure.iprevent_set.all() if i.in_reply_to ])
+    objs.update([i.in_reply_to.by for i in disclosure.iprevent_set.all() if i.in_reply_to ])
+    objs = sorted(list(objs),key=lambda o:o.__class__.__name__)
+    return json.dumps(json.loads(serializers.serialize("json",objs)),indent=4)
diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py
index 08979a3972..8eb3affbc0 100644
--- a/ietf/ipr/views.py
+++ b/ietf/ipr/views.py
@@ -32,7 +32,7 @@
     NonDocSpecificIprDisclosure, IprDocRel,
     RelatedIpr,IprEvent)
 from ietf.ipr.utils import (get_genitive, get_ipr_summary,
-    iprs_from_docs, related_docs)
+    iprs_from_docs, json_dump_disclosure, related_docs)
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.message.models import Message
 from ietf.message.utils import infer_message
@@ -901,3 +901,8 @@ def update(request, id):
     child = ipr.get_child()
     type = class_to_type[child.__class__.__name__]
     return new(request, type, updates=id)
+
+@role_required("Secretariat")
+def json_snapshot(request, id):
+    obj = get_object_or_404(IprDisclosureBase,id=id).get_child()
+    return HttpResponse(json_dump_disclosure(obj),content_type="application/json")

From 9d2fa7a32c6dae35de28c3a5f62ca5d762baef3c Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 9 Oct 2025 17:04:13 -0500
Subject: [PATCH 468/656] feat: track deleted ipr disclosures (#9691)

* feat: track deleted ipr disclosures

* fix: unique constraint on removed_id
---
 ietf/ipr/admin.py                             | 23 ++++++++++++---
 .../migrations/0005_removediprdisclosure.py   | 28 +++++++++++++++++++
 .../migrations/0006_already_removed_ipr.py    | 24 ++++++++++++++++
 ietf/ipr/models.py                            |  6 +++-
 ietf/ipr/resources.py                         | 19 +++++++++++--
 ietf/ipr/tests.py                             | 22 ++++++++++++++-
 ietf/ipr/views.py                             | 11 ++++++--
 ietf/templates/ipr/deleted.html               | 16 +++++++++++
 8 files changed, 139 insertions(+), 10 deletions(-)
 create mode 100644 ietf/ipr/migrations/0005_removediprdisclosure.py
 create mode 100644 ietf/ipr/migrations/0006_already_removed_ipr.py
 create mode 100644 ietf/templates/ipr/deleted.html

diff --git a/ietf/ipr/admin.py b/ietf/ipr/admin.py
index afc1952d72..1a8a908dcd 100644
--- a/ietf/ipr/admin.py
+++ b/ietf/ipr/admin.py
@@ -1,13 +1,22 @@
-# Copyright The IETF Trust 2010-2020, All Rights Reserved
+# Copyright The IETF Trust 2010-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
 from django import forms
 from django.contrib import admin
 from ietf.name.models import DocRelationshipName
-from ietf.ipr.models import (IprDisclosureBase, IprDocRel, IprEvent,
-    RelatedIpr, HolderIprDisclosure, ThirdPartyIprDisclosure, GenericIprDisclosure, 
-    NonDocSpecificIprDisclosure, LegacyMigrationIprEvent)
+from ietf.ipr.models import (
+    IprDisclosureBase,
+    IprDocRel,
+    IprEvent,
+    RelatedIpr,
+    HolderIprDisclosure,
+    RemovedIprDisclosure,
+    ThirdPartyIprDisclosure,
+    GenericIprDisclosure,
+    NonDocSpecificIprDisclosure,
+    LegacyMigrationIprEvent,
+)
 
 # ------------------------------------------------------
 # ModelAdmins
@@ -110,3 +119,9 @@ class LegacyMigrationIprEventAdmin(admin.ModelAdmin):
     list_filter = ['time', 'type', 'response_due']
     raw_id_fields = ['by', 'disclosure', 'message', 'in_reply_to']
 admin.site.register(LegacyMigrationIprEvent, LegacyMigrationIprEventAdmin)
+
+class RemovedIprDisclosureAdmin(admin.ModelAdmin):
+    pass
+
+
+admin.site.register(RemovedIprDisclosure, RemovedIprDisclosureAdmin)
diff --git a/ietf/ipr/migrations/0005_removediprdisclosure.py b/ietf/ipr/migrations/0005_removediprdisclosure.py
new file mode 100644
index 0000000000..400a264579
--- /dev/null
+++ b/ietf/ipr/migrations/0005_removediprdisclosure.py
@@ -0,0 +1,28 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ipr", "0004_holderiprdisclosure_is_blanket_disclosure"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RemovedIprDisclosure",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("removed_id", models.PositiveBigIntegerField(unique=True)),
+                ("reason", models.TextField()),
+            ],
+        ),
+    ]
diff --git a/ietf/ipr/migrations/0006_already_removed_ipr.py b/ietf/ipr/migrations/0006_already_removed_ipr.py
new file mode 100644
index 0000000000..0e2dbc63eb
--- /dev/null
+++ b/ietf/ipr/migrations/0006_already_removed_ipr.py
@@ -0,0 +1,24 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    RemovedIprDisclosure = apps.get_model("ipr", "RemovedIprDisclosure")
+    for id in (6544, 6068):
+        RemovedIprDisclosure.objects.create(
+            removed_id=id,
+            reason="This IPR disclosure was removed as objectively false.",
+        )
+
+
+def reverse(apps, schema_editor):
+    RemovedIprDisclosure = apps.get_model("ipr", "RemovedIprDisclosure")
+    RemovedIprDisclosure.objects.all().delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ipr", "0005_removediprdisclosure"),
+    ]
+
+    operations = [migrations.RunPython(forward, reverse)]
diff --git a/ietf/ipr/models.py b/ietf/ipr/models.py
index 2d81eb4b42..ea148c2704 100644
--- a/ietf/ipr/models.py
+++ b/ietf/ipr/models.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2007-2023, All Rights Reserved
+# Copyright The IETF Trust 2007-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -270,3 +270,7 @@ class LegacyMigrationIprEvent(IprEvent):
     """A subclass of IprEvent specifically for capturing contents of legacy_url_0,
     the text of a disclosure submitted by email"""
     pass
+
+class RemovedIprDisclosure(models.Model):
+    removed_id = models.PositiveBigIntegerField(unique=True)
+    reason = models.TextField()
diff --git a/ietf/ipr/resources.py b/ietf/ipr/resources.py
index 0d8421cdec..c4d2c436e6 100644
--- a/ietf/ipr/resources.py
+++ b/ietf/ipr/resources.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2015-2019, All Rights Reserved
+# Copyright The IETF Trust 2015-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 # Autogenerated by the mkresources management command 2015-03-21 14:05 PDT
 
@@ -11,7 +11,7 @@
 
 from ietf import api
 
-from ietf.ipr.models import ( IprDisclosureBase, IprDocRel, HolderIprDisclosure, ThirdPartyIprDisclosure,
+from ietf.ipr.models import ( IprDisclosureBase, IprDocRel, HolderIprDisclosure, RemovedIprDisclosure, ThirdPartyIprDisclosure,
     RelatedIpr, NonDocSpecificIprDisclosure, GenericIprDisclosure, IprEvent, LegacyMigrationIprEvent )
 
 from ietf.person.resources import PersonResource
@@ -295,3 +295,18 @@ class Meta:
         }
 api.ipr.register(LegacyMigrationIprEventResource())
 
+
+
+class RemovedIprDisclosureResource(ModelResource):
+    class Meta:
+        queryset = RemovedIprDisclosure.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'removediprdisclosure'
+        ordering = ['id', ]
+        filtering = { 
+            "id": ALL,
+            "removed_id": ALL,
+            "reason": ALL,
+        }
+api.ipr.register(RemovedIprDisclosureResource())
diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py
index 4146fbd4c1..53a599e2de 100644
--- a/ietf/ipr/tests.py
+++ b/ietf/ipr/tests.py
@@ -41,7 +41,7 @@
 from ietf.ipr.forms import DraftForm, HolderIprDisclosureForm
 from ietf.ipr.mail import (process_response_email, get_reply_to, get_update_submitter_emails,
                            get_pseudo_submitter, get_holders, get_update_cc_addrs, UndeliverableIprResponseError)
-from ietf.ipr.models import (IprDisclosureBase, GenericIprDisclosure, HolderIprDisclosure,
+from ietf.ipr.models import (IprDisclosureBase, GenericIprDisclosure, HolderIprDisclosure, RemovedIprDisclosure,
                              ThirdPartyIprDisclosure, IprEvent)
 from ietf.ipr.templatetags.ipr_filters import no_revisions_message
 from ietf.ipr.utils import get_genitive, get_ipr_summary, ingest_response_email
@@ -129,6 +129,26 @@ def test_showlist(self):
             self.assertContains(r, "removed as objectively false")
             ipr.delete()
 
+    def test_show_delete(self):
+        ipr = HolderIprDisclosureFactory()
+        removed = RemovedIprDisclosure.objects.create(
+            removed_id=ipr.pk, reason="Removed for reasons"
+        )
+        url = urlreverse("ietf.ipr.views.show", kwargs=dict(id=removed.removed_id))
+        r = self.client.get(url)
+        self.assertContains(r, "Removed for reasons")
+        q = PyQuery(r.content)
+        self.assertEqual(len(q("#deletion_warning")), 0)
+        self.client.login(username="secretary", password="secretary+password")
+        r = self.client.get(url)
+        self.assertContains(r, "Removed for reasons")
+        q = PyQuery(r.content)
+        self.assertEqual(len(q("#deletion_warning")), 1)
+        ipr.delete()
+        r = self.client.get(url)
+        self.assertContains(r, "Removed for reasons")
+        q = PyQuery(r.content)
+        self.assertEqual(len(q("#deletion_warning")), 0)
 
     def test_show_posted(self):
         ipr = HolderIprDisclosureFactory()
diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py
index 8eb3affbc0..665c99dc43 100644
--- a/ietf/ipr/views.py
+++ b/ietf/ipr/views.py
@@ -28,7 +28,7 @@
     AddCommentForm, AddEmailForm, NotifyForm, StateForm, NonDocSpecificIprDisclosureForm,
     GenericIprDisclosureForm)
 from ietf.ipr.models import (IprDisclosureStateName, IprDisclosureBase,
-    HolderIprDisclosure, GenericIprDisclosure, ThirdPartyIprDisclosure,
+    HolderIprDisclosure, GenericIprDisclosure, RemovedIprDisclosure, ThirdPartyIprDisclosure,
     NonDocSpecificIprDisclosure, IprDocRel,
     RelatedIpr,IprEvent)
 from ietf.ipr.utils import (get_genitive, get_ipr_summary,
@@ -817,7 +817,14 @@ def get_details_tabs(ipr, selected):
 
 def show(request, id):
     """View of individual declaration"""
-    ipr = get_object_or_404(IprDisclosureBase, id=id).get_child()
+    ipr = IprDisclosureBase.objects.filter(id=id)
+    removed = RemovedIprDisclosure.objects.filter(removed_id=id)
+    if removed.exists():
+        return render(request, "ipr/deleted.html", {"removed": removed.get(), "ipr": ipr})
+    if not ipr.exists():
+        raise Http404
+    else:
+        ipr = ipr.get().get_child()
     if not has_role(request.user, 'Secretariat'):
         if ipr.state.slug in ['removed', 'removed_objfalse']:
             return render(request, "ipr/removed.html", {
diff --git a/ietf/templates/ipr/deleted.html b/ietf/templates/ipr/deleted.html
new file mode 100644
index 0000000000..24f696ebca
--- /dev/null
+++ b/ietf/templates/ipr/deleted.html
@@ -0,0 +1,16 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2015-2023, All Rights Reserved #}
+{% load ietf_filters origin %}
+{% block title %}Removed IPR Disclosure{% endblock %}
+{% block content %}
+    {% origin %}
+    <h1>Removed IPR disclosure</h1>
+    <p class="alert alert-info my-3">
+        {{ removed.reason }}
+    </p>
+    {% if user|has_role:"Secretariat" and ipr.exists %}
+    <p class="alert alert-warn my-3" id="deletion_warning">
+        This disclosure has not yet been deleted and parts of its content is available through, e.g, the history view and the /api/v1 views.
+    </p>
+    {% endif %}
+    {% endblock %}
\ No newline at end of file

From ed6b061cfe4279328dbb7b914f5f7f76644521f9 Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Fri, 10 Oct 2025 05:43:24 -0700
Subject: [PATCH 469/656] chore: merge feat/sreq to main (#9697)

* refactor: move session request tool to ietf.meeting and restyle (#9617)

* refactor: move session request tool to ietf.meeting and restyle to match standard Datatracker

* fix: add redirect for old session request url

* fix: move stripe javascript to js file

* fix: update copyright lines for modified files

* fix: rename javascripts and expand redirects

* fix: don't show inactive constraints label when there are none (#9680)

* chore: remove unused utility

* fix: add test for secr main menu page (#9693)

* fix: don't show inactive constraints label when there are none

* fix: add test for secr main menu page

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/meeting/forms.py                         | 333 +++++++-
 .../templatetags/ams_filters.py               |   2 +
 .../tests_session_requests.py}                | 314 ++++---
 ietf/meeting/tests_views.py                   |   4 +-
 ietf/meeting/urls.py                          |  14 +-
 .../views_session_request.py}                 | 803 ++++++++++--------
 ietf/secr/meetings/views.py                   |   4 +-
 ietf/secr/sreq/__init__.py                    |   0
 ietf/secr/sreq/forms.py                       | 333 --------
 ietf/secr/sreq/templatetags/__init__.py       |   0
 ietf/secr/sreq/urls.py                        |  20 -
 ietf/secr/telechat/tests.py                   |  21 +
 ietf/secr/templates/includes/activities.html  |  23 -
 .../includes/buttons_next_cancel.html         |   6 -
 .../includes/buttons_submit_cancel.html       |   6 -
 .../templates/includes/sessions_footer.html   |   5 -
 .../includes/sessions_request_form.html       | 130 ---
 .../includes/sessions_request_view.html       |  73 --
 .../sessions_request_view_formset.html        |  32 -
 .../sessions_request_view_session_set.html    |  32 -
 ietf/secr/templates/index.html                |   6 +-
 ietf/secr/templates/sreq/confirm.html         |  57 --
 ietf/secr/templates/sreq/edit.html            |  39 -
 ietf/secr/templates/sreq/locked.html          |  30 -
 ietf/secr/templates/sreq/main.html            |  65 --
 ietf/secr/templates/sreq/new.html             |  43 -
 ietf/secr/templates/sreq/tool_status.html     |  42 -
 ietf/secr/templates/sreq/view.html            |  55 --
 ietf/secr/urls.py                             |  13 +-
 ietf/secr/utils/group.py                      |  50 --
 ietf/settings.py                              |   1 -
 ietf/static/js/custom_striped.js              |  16 +
 ietf/{secr => }/static/js/session_form.js     |   2 +-
 .../js/session_request.js}                    |  12 +-
 ietf/templates/base/menu.html                 |   4 +-
 ietf/templates/group/meetings-row.html        |   3 +-
 ietf/templates/group/meetings.html            |   3 +-
 .../meeting/important_dates_for_meeting.ics   |   5 +-
 ietf/templates/meeting/requests.html          |   4 +-
 .../session_approval_notification.txt         |   5 +-
 .../meeting}/session_cancel_notification.txt  |   1 +
 .../meeting/session_details_form.html         |  64 +-
 .../session_not_meeting_notification.txt}     |   1 +
 .../meeting/session_request_confirm.html      |  38 +
 .../meeting/session_request_form.html         | 206 +++++
 .../meeting/session_request_info.txt          |  26 +
 .../meeting/session_request_list.html         |  65 ++
 .../meeting/session_request_locked.html       |  21 +
 .../meeting}/session_request_notification.txt |   3 +-
 .../meeting/session_request_status.html       |  28 +
 .../meeting/session_request_view.html         |  59 ++
 .../meeting/session_request_view_formset.html |  49 ++
 .../session_request_view_session_set.html     |  47 +
 .../meeting/session_request_view_table.html   | 146 ++++
 package.json                                  |   5 +-
 55 files changed, 1728 insertions(+), 1641 deletions(-)
 rename ietf/{secr/sreq => meeting}/templatetags/ams_filters.py (96%)
 rename ietf/{secr/sreq/tests.py => meeting/tests_session_requests.py} (84%)
 rename ietf/{secr/sreq/views.py => meeting/views_session_request.py} (80%)
 delete mode 100644 ietf/secr/sreq/__init__.py
 delete mode 100644 ietf/secr/sreq/forms.py
 delete mode 100644 ietf/secr/sreq/templatetags/__init__.py
 delete mode 100644 ietf/secr/sreq/urls.py
 delete mode 100644 ietf/secr/templates/includes/activities.html
 delete mode 100644 ietf/secr/templates/includes/buttons_next_cancel.html
 delete mode 100644 ietf/secr/templates/includes/buttons_submit_cancel.html
 delete mode 100755 ietf/secr/templates/includes/sessions_footer.html
 delete mode 100755 ietf/secr/templates/includes/sessions_request_form.html
 delete mode 100644 ietf/secr/templates/includes/sessions_request_view.html
 delete mode 100644 ietf/secr/templates/includes/sessions_request_view_formset.html
 delete mode 100644 ietf/secr/templates/includes/sessions_request_view_session_set.html
 delete mode 100755 ietf/secr/templates/sreq/confirm.html
 delete mode 100755 ietf/secr/templates/sreq/edit.html
 delete mode 100755 ietf/secr/templates/sreq/locked.html
 delete mode 100755 ietf/secr/templates/sreq/main.html
 delete mode 100755 ietf/secr/templates/sreq/new.html
 delete mode 100755 ietf/secr/templates/sreq/tool_status.html
 delete mode 100644 ietf/secr/templates/sreq/view.html
 delete mode 100644 ietf/secr/utils/group.py
 create mode 100644 ietf/static/js/custom_striped.js
 rename ietf/{secr => }/static/js/session_form.js (92%)
 rename ietf/{secr/static/js/sessions.js => static/js/session_request.js} (90%)
 rename ietf/{secr/templates/sreq => templates/meeting}/session_approval_notification.txt (56%)
 rename ietf/{secr/templates/sreq => templates/meeting}/session_cancel_notification.txt (71%)
 rename ietf/{secr/templates/sreq/not_meeting_notification.txt => templates/meeting/session_not_meeting_notification.txt} (83%)
 create mode 100644 ietf/templates/meeting/session_request_confirm.html
 create mode 100644 ietf/templates/meeting/session_request_form.html
 create mode 100644 ietf/templates/meeting/session_request_info.txt
 create mode 100644 ietf/templates/meeting/session_request_list.html
 create mode 100644 ietf/templates/meeting/session_request_locked.html
 rename ietf/{secr/templates/sreq => templates/meeting}/session_request_notification.txt (56%)
 create mode 100644 ietf/templates/meeting/session_request_status.html
 create mode 100644 ietf/templates/meeting/session_request_view.html
 create mode 100644 ietf/templates/meeting/session_request_view_formset.html
 create mode 100644 ietf/templates/meeting/session_request_view_session_set.html
 create mode 100644 ietf/templates/meeting/session_request_view_table.html

diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py
index b6b1a1591f..e5b1697f86 100644
--- a/ietf/meeting/forms.py
+++ b/ietf/meeting/forms.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2016-2023, All Rights Reserved
+# Copyright The IETF Trust 2016-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -15,19 +15,24 @@
 from django.core import validators
 from django.core.exceptions import ValidationError
 from django.forms import BaseInlineFormSet
+from django.template.defaultfilters import pluralize
 from django.utils.functional import cached_property
+from django.utils.safestring import mark_safe
 
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import Document, State, NewRevisionDocEvent
 from ietf.group.models import Group
 from ietf.group.utils import groups_managed_by
-from ietf.meeting.models import Session, Meeting, Schedule, COUNTRIES, TIMEZONES, TimeSlot, Room
+from ietf.meeting.models import (Session, Meeting, Schedule, COUNTRIES, TIMEZONES, TimeSlot, Room,
+    Constraint, ResourceAssociation)
 from ietf.meeting.helpers import get_next_interim_number, make_materials_directories
 from ietf.meeting.helpers import is_interim_meeting_approved, get_next_agenda_name
 from ietf.message.models import Message
-from ietf.name.models import TimeSlotTypeName, SessionPurposeName
+from ietf.name.models import TimeSlotTypeName, SessionPurposeName, TimerangeName, ConstraintName
+from ietf.person.fields import SearchablePersonsField
 from ietf.person.models import Person
+from ietf.utils import log
 from ietf.utils.fields import (
     DatepickerDateField,
     DatepickerSplitDateTimeWidget,
@@ -35,9 +40,14 @@
     ModelMultipleChoiceField,
     MultiEmailField,
 )
+from ietf.utils.html import clean_text_field
 from ietf.utils.validators import ( validate_file_size, validate_mime_type,
     validate_file_extension, validate_no_html_frame)
 
+NUM_SESSION_CHOICES = (('', '--Please select'), ('1', '1'), ('2', '2'))
+SESSION_TIME_RELATION_CHOICES = (('', 'No preference'),) + Constraint.TIME_RELATION_CHOICES
+JOINT_FOR_SESSION_CHOICES = (('1', 'First session'), ('2', 'Second session'), ('3', 'Third session'), )
+
 # -------------------------------------------------
 # Helpers
 # -------------------------------------------------
@@ -74,6 +84,27 @@ def duration_string(duration):
     return string
 
 
+def allowed_conflicting_groups():
+    return Group.objects.filter(
+        type__in=['wg', 'ag', 'rg', 'rag', 'program', 'edwg'],
+        state__in=['bof', 'proposed', 'active'])
+
+
+def check_conflict(groups, source_group):
+    '''
+    Takes a string which is a list of group acronyms.  Checks that they are all active groups
+    '''
+    # convert to python list (allow space or comma separated lists)
+    items = groups.replace(',', ' ').split()
+    active_groups = allowed_conflicting_groups()
+    for group in items:
+        if group == source_group.acronym:
+            raise forms.ValidationError("Cannot declare a conflict with the same group: %s" % group)
+
+        if not active_groups.filter(acronym=group):
+            raise forms.ValidationError("Invalid or inactive group acronym: %s" % group)
+
+
 # -------------------------------------------------
 # Forms
 # -------------------------------------------------
@@ -753,6 +784,9 @@ def __init__(self, group, *args, **kwargs):
         self.fields['purpose'].queryset = SessionPurposeName.objects.filter(pk__in=session_purposes)
         if not group.features.acts_like_wg:
             self.fields['requested_duration'].durations = [datetime.timedelta(minutes=m) for m in range(30, 241, 30)]
+        # add bootstrap classes
+        self.fields['purpose'].widget.attrs.update({'class': 'form-select'})
+        self.fields['type'].widget.attrs.update({'class': 'form-select', 'aria-label': 'session type'})
 
     class Meta:
         model = Session
@@ -837,3 +871,296 @@ def sessiondetailsformset_factory(min_num=1, max_num=3):
         max_num=max_num,
         extra=max_num,  # only creates up to max_num total
     )
+
+
+class SessionRequestStatusForm(forms.Form):
+    message = forms.CharField(widget=forms.Textarea(attrs={'rows': '3', 'cols': '80'}), strip=False)
+
+
+class NameModelMultipleChoiceField(ModelMultipleChoiceField):
+    def label_from_instance(self, name):
+        return name.desc
+
+
+class SessionRequestForm(forms.Form):
+    num_session = forms.ChoiceField(
+        choices=NUM_SESSION_CHOICES,
+        label="Number of sessions")
+    # session fields are added in __init__()
+    session_time_relation = forms.ChoiceField(
+        choices=SESSION_TIME_RELATION_CHOICES,
+        required=False,
+        label="Time between two sessions")
+    attendees = forms.IntegerField(label="Number of Attendees")
+    # FIXME: it would cleaner to have these be
+    # ModelMultipleChoiceField, and just customize the widgetry, that
+    # way validation comes for free (applies to this CharField and the
+    # constraints dynamically instantiated in __init__())
+    joint_with_groups = forms.CharField(max_length=255, required=False)
+    joint_with_groups_selector = forms.ChoiceField(choices=[], required=False)  # group select widget for prev field
+    joint_for_session = forms.ChoiceField(choices=JOINT_FOR_SESSION_CHOICES, required=False)
+    comments = forms.CharField(
+        max_length=200,
+        label='Special Requests',
+        help_text='i.e. restrictions on meeting times / days, etc.  (limit 200 characters)',
+        required=False)
+    third_session = forms.BooleanField(
+        required=False,
+        help_text="Help")
+    resources = forms.MultipleChoiceField(
+        widget=forms.CheckboxSelectMultiple,
+        required=False,
+        label='Resources Requested')
+    bethere = SearchablePersonsField(
+        label="Participants who must be present",
+        required=False,
+        help_text=mark_safe('<span class="fw-bold text-danger">Do not include Area Directors and WG Chairs; the system already tracks their availability.</span>'))
+    timeranges = NameModelMultipleChoiceField(
+        widget=forms.CheckboxSelectMultiple,
+        required=False,
+        label=mark_safe('Times during which this WG can <strong>not</strong> meet:<br>Please explain any selections in Special Requests below.'),
+        queryset=TimerangeName.objects.all())
+    adjacent_with_wg = forms.ChoiceField(
+        required=False,
+        label=mark_safe('Plan session adjacent with another WG:<br>(Immediately before or after another WG, no break in between, in the same room.)'))
+    send_notifications = forms.BooleanField(label="Send notification emails?", required=False, initial=False)
+
+    def __init__(self, group, meeting, data=None, *args, **kwargs):
+        self.hidden = kwargs.pop('hidden', False)
+        self.notifications_optional = kwargs.pop('notifications_optional', False)
+
+        self.group = group
+        formset_class = sessiondetailsformset_factory(max_num=3 if group.features.acts_like_wg else 50)
+        self.session_forms = formset_class(group=self.group, meeting=meeting, data=data)
+        super().__init__(data=data, *args, **kwargs)
+        if not self.notifications_optional:
+            self.fields['send_notifications'].widget = forms.HiddenInput()
+
+        # Allow additional sessions for non-wg-like groups
+        if not self.group.features.acts_like_wg:
+            self.fields['num_session'].choices = ((n, str(n)) for n in range(1, 51))
+
+        self._add_widget_class(self.fields['third_session'].widget, 'form-check-input')
+        self.fields['comments'].widget = forms.Textarea(attrs={'rows': '3', 'cols': '65'})
+
+        other_groups = list(allowed_conflicting_groups().exclude(pk=group.pk).values_list('acronym', 'acronym').order_by('acronym'))
+        self.fields['adjacent_with_wg'].choices = [('', '--No preference')] + other_groups
+        group_acronym_choices = [('', '--Select WG(s)')] + other_groups
+        self.fields['joint_with_groups_selector'].choices = group_acronym_choices
+
+        # Set up constraints for the meeting
+        self._wg_field_data = []
+        for constraintname in meeting.group_conflict_types.all():
+            # two fields for each constraint: a CharField for the group list and a selector to add entries
+            constraint_field = forms.CharField(max_length=255, required=False)
+            constraint_field.widget.attrs['data-slug'] = constraintname.slug
+            constraint_field.widget.attrs['data-constraint-name'] = str(constraintname).title()
+            constraint_field.widget.attrs['aria-label'] = f'{constraintname.slug}_input'
+            self._add_widget_class(constraint_field.widget, 'wg_constraint')
+            self._add_widget_class(constraint_field.widget, 'form-control')
+
+            selector_field = forms.ChoiceField(choices=group_acronym_choices, required=False)
+            selector_field.widget.attrs['data-slug'] = constraintname.slug  # used by onchange handler
+            self._add_widget_class(selector_field.widget, 'wg_constraint_selector')
+            self._add_widget_class(selector_field.widget, 'form-control')
+
+            cfield_id = 'constraint_{}'.format(constraintname.slug)
+            cselector_id = 'wg_selector_{}'.format(constraintname.slug)
+            # keep an eye out for field name conflicts
+            log.assertion('cfield_id not in self.fields')
+            log.assertion('cselector_id not in self.fields')
+            self.fields[cfield_id] = constraint_field
+            self.fields[cselector_id] = selector_field
+            self._wg_field_data.append((constraintname, cfield_id, cselector_id))
+
+        # Show constraints that are not actually used by the meeting so these don't get lost
+        self._inactive_wg_field_data = []
+        inactive_cnames = ConstraintName.objects.filter(
+            is_group_conflict=True  # Only collect group conflicts...
+        ).exclude(
+            meeting=meeting  # ...that are not enabled for this meeting...
+        ).filter(
+            constraint__source=group,  # ...but exist for this group...
+            constraint__meeting=meeting,  # ... at this meeting.
+        ).distinct()
+
+        for inactive_constraint_name in inactive_cnames:
+            field_id = 'delete_{}'.format(inactive_constraint_name.slug)
+            self.fields[field_id] = forms.BooleanField(required=False, label='Delete this conflict', help_text='Delete this inactive conflict?')
+            self._add_widget_class(self.fields[field_id].widget, 'form-control')
+            constraints = group.constraint_source_set.filter(meeting=meeting, name=inactive_constraint_name)
+            self._inactive_wg_field_data.append(
+                (inactive_constraint_name,
+                 ' '.join([c.target.acronym for c in constraints]),
+                 field_id)
+            )
+
+        self.fields['joint_with_groups_selector'].widget.attrs['onchange'] = "document.form_post.joint_with_groups.value=document.form_post.joint_with_groups.value + ' ' + this.options[this.selectedIndex].value; return 1;"
+        self.fields["resources"].choices = [(x.pk, x.desc) for x in ResourceAssociation.objects.filter(name__used=True).order_by('name__order')]
+
+        if self.hidden:
+            # replace all the widgets to start...
+            for key in list(self.fields.keys()):
+                self.fields[key].widget = forms.HiddenInput()
+            # re-replace a couple special cases
+            self.fields['resources'].widget = forms.MultipleHiddenInput()
+            self.fields['timeranges'].widget = forms.MultipleHiddenInput()
+            # and entirely replace bethere - no need to support searching if input is hidden
+            self.fields['bethere'] = ModelMultipleChoiceField(
+                widget=forms.MultipleHiddenInput, required=False,
+                queryset=Person.objects.all(),
+            )
+
+    def wg_constraint_fields(self):
+        """Iterates over wg constraint fields
+
+        Intended for use in the template.
+        """
+        for cname, cfield_id, cselector_id in self._wg_field_data:
+            yield cname, self[cfield_id], self[cselector_id]
+
+    def wg_constraint_count(self):
+        """How many wg constraints are there?"""
+        return len(self._wg_field_data)
+
+    def wg_constraint_field_ids(self):
+        """Iterates over wg constraint field IDs"""
+        for cname, cfield_id, _ in self._wg_field_data:
+            yield cname, cfield_id
+
+    def inactive_wg_constraints(self):
+        for cname, value, field_id in self._inactive_wg_field_data:
+            yield cname, value, self[field_id]
+
+    def inactive_wg_constraint_count(self):
+        return len(self._inactive_wg_field_data)
+
+    def inactive_wg_constraint_field_ids(self):
+        """Iterates over wg constraint field IDs"""
+        for cname, _, field_id in self._inactive_wg_field_data:
+            yield cname, field_id
+
+    @staticmethod
+    def _add_widget_class(widget, new_class):
+        """Add a new class, taking care in case some already exist"""
+        existing_classes = widget.attrs.get('class', '').split()
+        widget.attrs['class'] = ' '.join(existing_classes + [new_class])
+
+    def _join_conflicts(self, cleaned_data, slugs):
+        """Concatenate constraint fields from cleaned data into a single list"""
+        conflicts = []
+        for cname, cfield_id, _ in self._wg_field_data:
+            if cname.slug in slugs and cfield_id in cleaned_data:
+                groups = cleaned_data[cfield_id]
+                # convert to python list (allow space or comma separated lists)
+                items = groups.replace(',', ' ').split()
+                conflicts.extend(items)
+        return conflicts
+
+    def _validate_duplicate_conflicts(self, cleaned_data):
+        """Validate that no WGs appear in more than one constraint that does not allow duplicates
+
+        Raises ValidationError
+        """
+        # Only the older constraints (conflict, conflic2, conflic3) need to be mutually exclusive.
+        all_conflicts = self._join_conflicts(cleaned_data, ['conflict', 'conflic2', 'conflic3'])
+        seen = []
+        duplicated = []
+        errors = []
+        for c in all_conflicts:
+            if c not in seen:
+                seen.append(c)
+            elif c not in duplicated:  # only report once
+                duplicated.append(c)
+                errors.append(forms.ValidationError('%s appears in conflicts more than once' % c))
+        return errors
+
+    def clean_joint_with_groups(self):
+        groups = self.cleaned_data['joint_with_groups']
+        check_conflict(groups, self.group)
+        return groups
+
+    def clean_comments(self):
+        return clean_text_field(self.cleaned_data['comments'])
+
+    def clean_bethere(self):
+        bethere = self.cleaned_data["bethere"]
+        if bethere:
+            extra = set(
+                Person.objects.filter(
+                    role__group=self.group, role__name__in=["chair", "ad"]
+                )
+                & bethere
+            )
+            if extra:
+                extras = ", ".join(e.name for e in extra)
+                raise forms.ValidationError(
+                    (
+                        f"Please remove the following person{pluralize(len(extra))}, the system "
+                        f"tracks their availability due to their role{pluralize(len(extra))}: {extras}."
+                    )
+                )
+        return bethere
+
+    def clean_send_notifications(self):
+        return True if not self.notifications_optional else self.cleaned_data['send_notifications']
+
+    def is_valid(self):
+        return super().is_valid() and self.session_forms.is_valid()
+
+    def clean(self):
+        super(SessionRequestForm, self).clean()
+        self.session_forms.clean()
+
+        data = self.cleaned_data
+
+        # Validate the individual conflict fields
+        for _, cfield_id, _ in self._wg_field_data:
+            try:
+                check_conflict(data[cfield_id], self.group)
+            except forms.ValidationError as e:
+                self.add_error(cfield_id, e)
+
+        # Skip remaining tests if individual field tests had errors,
+        if self.errors:
+            return data
+
+        # error if conflicts contain disallowed dupes
+        for error in self._validate_duplicate_conflicts(data):
+            self.add_error(None, error)
+
+        # Verify expected number of session entries are present
+        num_sessions_with_data = len(self.session_forms.forms_to_keep)
+        num_sessions_expected = -1
+        try:
+            num_sessions_expected = int(data.get('num_session', ''))
+        except ValueError:
+            self.add_error('num_session', 'Invalid value for number of sessions')
+        if num_sessions_with_data < num_sessions_expected:
+            self.add_error('num_session', 'Must provide data for all sessions')
+
+        # if default (empty) option is selected, cleaned_data won't include num_session key
+        if num_sessions_expected != 2 and num_sessions_expected is not None:
+            if data.get('session_time_relation'):
+                self.add_error(
+                    'session_time_relation',
+                    forms.ValidationError('Time between sessions can only be used when two sessions are requested.')
+                )
+
+        joint_session = data.get('joint_for_session', '')
+        if joint_session != '':
+            joint_session = int(joint_session)
+            if joint_session > num_sessions_with_data:
+                self.add_error(
+                    'joint_for_session',
+                    forms.ValidationError(
+                        f'Session {joint_session} can not be the joint session, the session has not been requested.'
+                    )
+                )
+
+        return data
+
+    @property
+    def media(self):
+        # get media for our formset
+        return super().media + self.session_forms.media + forms.Media(js=('ietf/js/session_form.js',))
diff --git a/ietf/secr/sreq/templatetags/ams_filters.py b/ietf/meeting/templatetags/ams_filters.py
similarity index 96%
rename from ietf/secr/sreq/templatetags/ams_filters.py
rename to ietf/meeting/templatetags/ams_filters.py
index 3ef872232a..a8175a81d6 100644
--- a/ietf/secr/sreq/templatetags/ams_filters.py
+++ b/ietf/meeting/templatetags/ams_filters.py
@@ -1,3 +1,5 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
 from django import template
 from ietf.person.models import Person
 
diff --git a/ietf/secr/sreq/tests.py b/ietf/meeting/tests_session_requests.py
similarity index 84%
rename from ietf/secr/sreq/tests.py
rename to ietf/meeting/tests_session_requests.py
index 847b993e1c..0cb092d2f8 100644
--- a/ietf/secr/sreq/tests.py
+++ b/ietf/meeting/tests_session_requests.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2013-2022, All Rights Reserved
+# Copyright The IETF Trust 2013-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -15,30 +15,15 @@
 from ietf.name.models import ConstraintName, TimerangeName
 from ietf.person.factories import PersonFactory
 from ietf.person.models import Person
-from ietf.secr.sreq.forms import SessionForm
+from ietf.meeting.forms import SessionRequestForm
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text, send_mail
 from ietf.utils.timezone import date_today
 
 
 from pyquery import PyQuery
 
-SECR_USER='secretary'
+SECR_USER = 'secretary'
 
-class SreqUrlTests(TestCase):
-    def test_urls(self):
-        MeetingFactory(type_id='ietf',date=date_today())
-
-        self.client.login(username="secretary", password="secretary+password")
-
-        r = self.client.get("/secr/")
-        self.assertEqual(r.status_code, 200)
-
-        r = self.client.get("/secr/sreq/")
-        self.assertEqual(r.status_code, 200)
-
-        testgroup=GroupFactory()
-        r = self.client.get("/secr/sreq/%s/new/" % testgroup.acronym)
-        self.assertEqual(r.status_code, 200)
 
 class SessionRequestTestCase(TestCase):
     def test_main(self):
@@ -46,7 +31,7 @@ def test_main(self):
         SessionFactory.create_batch(2, meeting=meeting, status_id='sched')
         SessionFactory.create_batch(2, meeting=meeting, status_id='disappr')
         # Several unscheduled groups come from make_immutable_base_data
-        url = reverse('ietf.secr.sreq.views.main')
+        url = reverse('ietf.meeting.views_session_request.list_view')
         self.client.login(username="secretary", password="secretary+password")
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
@@ -62,27 +47,27 @@ def test_approve(self):
         mars = GroupFactory(parent=area, acronym='mars')
         # create session waiting for approval
         session = SessionFactory(meeting=meeting, group=mars, status_id='apprw')
-        url = reverse('ietf.secr.sreq.views.approve', kwargs={'acronym':'mars'})
+        url = reverse('ietf.meeting.views_session_request.approve_request', kwargs={'acronym': 'mars'})
         self.client.login(username="ad", password="ad+password")
         r = self.client.get(url)
-        self.assertRedirects(r,reverse('ietf.secr.sreq.views.view', kwargs={'acronym':'mars'}))
+        self.assertRedirects(r, reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'}))
         self.assertEqual(SchedulingEvent.objects.filter(session=session).order_by('-id')[0].status_id, 'appr')
-        
+
     def test_cancel(self):
         meeting = MeetingFactory(type_id='ietf', date=date_today())
         ad = Person.objects.get(user__username='ad')
         area = RoleFactory(name_id='ad', person=ad, group__type_id='area').group
         session = SessionFactory(meeting=meeting, group__parent=area, group__acronym='mars', status_id='sched')
-        url = reverse('ietf.secr.sreq.views.cancel', kwargs={'acronym':'mars'})
+        url = reverse('ietf.meeting.views_session_request.cancel_request', kwargs={'acronym': 'mars'})
         self.client.login(username="ad", password="ad+password")
         r = self.client.get(url)
-        self.assertRedirects(r,reverse('ietf.secr.sreq.views.main'))
+        self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view'))
         self.assertEqual(SchedulingEvent.objects.filter(session=session).order_by('-id')[0].status_id, 'deleted')
 
     def test_cancel_notification_msg(self):
         to = "<iesg-secretary@ietf.org>"
         subject = "Dummy subject"
-        template = "sreq/session_cancel_notification.txt"
+        template = "meeting/session_cancel_notification.txt"
         meeting = MeetingFactory(type_id="ietf", date=date_today())
         requester = PersonFactory(name="James O'Rourke", user__username="jimorourke")
         context = {"meeting": meeting, "requester": requester}
@@ -113,9 +98,9 @@ def test_edit(self):
         group4 = GroupFactory()
         iabprog = GroupFactory(type_id='program')
 
-        SessionFactory(meeting=meeting,group=mars,status_id='sched')
+        SessionFactory(meeting=meeting, group=mars, status_id='sched')
 
-        url = reverse('ietf.secr.sreq.views.edit', kwargs={'acronym':'mars'})
+        url = reverse('ietf.meeting.views_session_request.edit_request', kwargs={'acronym': 'mars'})
         self.client.login(username="marschairman", password="marschairman+password")
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
@@ -123,9 +108,9 @@ def test_edit(self):
         comments = 'need lights'
         mars_sessions = meeting.session_set.filter(group__acronym='mars')
         empty_outbox()
-        post_data = {'num_session':'2',
+        post_data = {'num_session': '2',
                      'attendees': attendees,
-                     'constraint_chair_conflict':iabprog.acronym,
+                     'constraint_chair_conflict': iabprog.acronym,
                      'session_time_relation': 'subsequent-days',
                      'adjacent_with_wg': group2.acronym,
                      'joint_with_groups': group3.acronym + ' ' + group4.acronym,
@@ -135,7 +120,7 @@ def test_edit(self):
                      'session_set-INITIAL_FORMS': '1',
                      'session_set-MIN_NUM_FORMS': '1',
                      'session_set-MAX_NUM_FORMS': '3',
-                     'session_set-0-id':mars_sessions[0].pk,
+                     'session_set-0-id': mars_sessions[0].pk,
                      'session_set-0-name': mars_sessions[0].name,
                      'session_set-0-short': mars_sessions[0].short,
                      'session_set-0-purpose': mars_sessions[0].purpose_id,
@@ -169,7 +154,7 @@ def test_edit(self):
                      'session_set-2-DELETE': 'on',
                      'submit': 'Continue'}
         r = self.client.post(url, post_data, HTTP_HOST='example.com')
-        redirect_url = reverse('ietf.secr.sreq.views.view', kwargs={'acronym': 'mars'})
+        redirect_url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'})
         self.assertRedirects(r, redirect_url)
 
         # Check whether updates were stored in the database
@@ -204,17 +189,17 @@ def test_edit(self):
         # Edit again, changing the joint sessions and clearing some fields. The behaviour of
         # edit is different depending on whether previous joint sessions were recorded.
         empty_outbox()
-        post_data = {'num_session':'2',
-                     'attendees':attendees,
-                     'constraint_chair_conflict':'',
-                     'comments':'need lights',
+        post_data = {'num_session': '2',
+                     'attendees': attendees,
+                     'constraint_chair_conflict': '',
+                     'comments': 'need lights',
                      'joint_with_groups': group2.acronym,
                      'joint_for_session': '1',
                      'session_set-TOTAL_FORMS': '3',  # matches what view actually sends, even with only 2 filled in
                      'session_set-INITIAL_FORMS': '2',
                      'session_set-MIN_NUM_FORMS': '1',
                      'session_set-MAX_NUM_FORMS': '3',
-                     'session_set-0-id':sessions[0].pk,
+                     'session_set-0-id': sessions[0].pk,
                      'session_set-0-name': sessions[0].name,
                      'session_set-0-short': sessions[0].short,
                      'session_set-0-purpose': sessions[0].purpose_id,
@@ -270,7 +255,6 @@ def test_edit(self):
         r = self.client.get(redirect_url)
         self.assertContains(r, 'First session with: {}'.format(group2.acronym))
 
-
     def test_edit_constraint_bethere(self):
         meeting = MeetingFactory(type_id='ietf', date=date_today())
         mars = RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars').group
@@ -282,7 +266,7 @@ def test_edit_constraint_bethere(self):
             name_id='bethere',
         )
         self.assertEqual(session.people_constraints.count(), 1)
-        url = reverse('ietf.secr.sreq.views.edit', kwargs=dict(acronym='mars'))
+        url = reverse('ietf.meeting.views_session_request.edit_request', kwargs=dict(acronym='mars'))
         self.client.login(username='marschairman', password='marschairman+password')
         attendees = '10'
         ad = Person.objects.get(user__username='ad')
@@ -290,8 +274,8 @@ def test_edit_constraint_bethere(self):
             'num_session': '1',
             'attendees': attendees,
             'bethere': str(ad.pk),
-            'constraint_chair_conflict':'',
-            'comments':'',
+            'constraint_chair_conflict': '',
+            'comments': '',
             'joint_with_groups': '',
             'joint_for_session': '',
             'delete_conflict': 'on',
@@ -299,7 +283,7 @@ def test_edit_constraint_bethere(self):
             'session_set-INITIAL_FORMS': '1',
             'session_set-MIN_NUM_FORMS': '1',
             'session_set-MAX_NUM_FORMS': '3',
-            'session_set-0-id':session.pk,
+            'session_set-0-id': session.pk,
             'session_set-0-name': session.name,
             'session_set-0-short': session.short,
             'session_set-0-purpose': session.purpose_id,
@@ -313,8 +297,8 @@ def test_edit_constraint_bethere(self):
             'session_set-1-id': '',
             'session_set-1-name': '',
             'session_set-1-short': '',
-            'session_set-1-purpose':'regular',
-            'session_set-1-type':'regular',
+            'session_set-1-purpose': 'regular',
+            'session_set-1-type': 'regular',
             'session_set-1-requested_duration': '',
             'session_set-1-on_agenda': 'True',
             'session_set-1-attendees': attendees,
@@ -333,7 +317,7 @@ def test_edit_constraint_bethere(self):
             'submit': 'Save',
         }
         r = self.client.post(url, post_data, HTTP_HOST='example.com')
-        redirect_url = reverse('ietf.secr.sreq.views.view', kwargs={'acronym': 'mars'})
+        redirect_url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'})
         self.assertRedirects(r, redirect_url)
         self.assertEqual([pc.person for pc in session.people_constraints.all()], [ad])
 
@@ -350,7 +334,7 @@ def test_edit_inactive_conflicts(self):
             target=other_group,
         )
 
-        url = reverse('ietf.secr.sreq.views.edit', kwargs=dict(acronym='mars'))
+        url = reverse('ietf.meeting.views_session_request.edit_request', kwargs=dict(acronym='mars'))
         self.client.login(username='marschairman', password='marschairman+password')
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
@@ -360,17 +344,17 @@ def test_edit_inactive_conflicts(self):
         found = q('input#id_delete_conflict[type="checkbox"]')
         self.assertEqual(len(found), 1)
         delete_checkbox = found[0]
-        # check that the label on the checkbox is correct
-        self.assertIn('Delete this conflict', delete_checkbox.tail)
+        self.assertIn('Delete this conflict', delete_checkbox.label.text)
         # check that the target is displayed correctly in the UI
-        self.assertIn(other_group.acronym, delete_checkbox.find('../input[@type="text"]').value)
+        row = found.parent().parent()
+        self.assertIn(other_group.acronym, row.find('input[@type="text"]').val())
 
         attendees = '10'
         post_data = {
             'num_session': '1',
             'attendees': attendees,
-            'constraint_chair_conflict':'',
-            'comments':'',
+            'constraint_chair_conflict': '',
+            'comments': '',
             'joint_with_groups': '',
             'joint_for_session': '',
             'delete_conflict': 'on',
@@ -378,7 +362,7 @@ def test_edit_inactive_conflicts(self):
             'session_set-INITIAL_FORMS': '1',
             'session_set-MIN_NUM_FORMS': '1',
             'session_set-MAX_NUM_FORMS': '3',
-            'session_set-0-id':session.pk,
+            'session_set-0-id': session.pk,
             'session_set-0-name': session.name,
             'session_set-0-short': session.short,
             'session_set-0-purpose': session.purpose_id,
@@ -392,28 +376,28 @@ def test_edit_inactive_conflicts(self):
             'submit': 'Save',
         }
         r = self.client.post(url, post_data, HTTP_HOST='example.com')
-        redirect_url = reverse('ietf.secr.sreq.views.view', kwargs={'acronym': 'mars'})
+        redirect_url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'})
         self.assertRedirects(r, redirect_url)
         self.assertEqual(len(mars.constraint_source_set.filter(name_id='conflict')), 0)
 
     def test_tool_status(self):
         MeetingFactory(type_id='ietf', date=date_today())
-        url = reverse('ietf.secr.sreq.views.tool_status')
+        url = reverse('ietf.meeting.views_session_request.status')
         self.client.login(username="secretary", password="secretary+password")
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-        r = self.client.post(url, {'message':'locked', 'submit':'Lock'})
-        self.assertRedirects(r,reverse('ietf.secr.sreq.views.main'))
+        r = self.client.post(url, {'message': 'locked', 'submit': 'Lock'})
+        self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view'))
 
     def test_new_req_constraint_types(self):
         """Configurable constraint types should be handled correctly in a new request
 
-        Relies on SessionForm representing constraint values with element IDs
+        Relies on SessionRequestForm representing constraint values with element IDs
         like id_constraint_<ConstraintName slug>
         """
         meeting = MeetingFactory(type_id='ietf', date=date_today())
         RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars')
-        url = reverse('ietf.secr.sreq.views.new', kwargs=dict(acronym='mars'))
+        url = reverse('ietf.meeting.views_session_request.new_request', kwargs=dict(acronym='mars'))
         self.client.login(username="marschairman", password="marschairman+password")
 
         for expected in [
@@ -441,7 +425,7 @@ def test_edit_req_constraint_types(self):
                        add_to_schedule=False)
         RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars')
 
-        url = reverse('ietf.secr.sreq.views.edit', kwargs=dict(acronym='mars'))
+        url = reverse('ietf.meeting.views_session_request.edit_request', kwargs=dict(acronym='mars'))
         self.client.login(username='marschairman', password='marschairman+password')
 
         for expected in [
@@ -460,6 +444,7 @@ def test_edit_req_constraint_types(self):
                 ['id_constraint_{}'.format(conf_name) for conf_name in expected],
             )
 
+
 class SubmitRequestCase(TestCase):
     def setUp(self):
         super(SubmitRequestCase, self).setUp()
@@ -476,15 +461,15 @@ def test_submit_request(self):
         group3 = GroupFactory(parent=area)
         group4 = GroupFactory(parent=area)
         session_count_before = Session.objects.filter(meeting=meeting, group=group).count()
-        url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym})
-        confirm_url = reverse('ietf.secr.sreq.views.confirm',kwargs={'acronym':group.acronym})
-        main_url = reverse('ietf.secr.sreq.views.main')
+        url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym})
+        confirm_url = reverse('ietf.meeting.views_session_request.confirm', kwargs={'acronym': group.acronym})
+        main_url = reverse('ietf.meeting.views_session_request.list_view')
         attendees = '10'
         comments = 'need projector'
-        post_data = {'num_session':'1',
-                     'attendees':attendees,
-                     'constraint_chair_conflict':'',
-                     'comments':comments,
+        post_data = {'num_session': '1',
+                     'attendees': attendees,
+                     'constraint_chair_conflict': '',
+                     'comments': comments,
                      'adjacent_with_wg': group2.acronym,
                      'timeranges': ['thursday-afternoon-early', 'thursday-afternoon-late'],
                      'joint_with_groups': group3.acronym + ' ' + group4.acronym,
@@ -506,7 +491,7 @@ def test_submit_request(self):
                      'session_set-0-DELETE': '',
                      'submit': 'Continue'}
         self.client.login(username="secretary", password="secretary+password")
-        r = self.client.post(url,post_data)
+        r = self.client.post(url, post_data)
         self.assertEqual(r.status_code, 200)
 
         # Verify the contents of the confirm view
@@ -515,13 +500,13 @@ def test_submit_request(self):
         self.assertContains(r, 'First session with: {} {}'.format(group3.acronym, group4.acronym))
 
         post_data['submit'] = 'Submit'
-        r = self.client.post(confirm_url,post_data)
+        r = self.client.post(confirm_url, post_data)
         self.assertRedirects(r, main_url)
         session_count_after = Session.objects.filter(meeting=meeting, group=group, type='regular').count()
         self.assertEqual(session_count_after, session_count_before + 1)
 
         # test that second confirm does not add sessions
-        r = self.client.post(confirm_url,post_data)
+        r = self.client.post(confirm_url, post_data)
         self.assertRedirects(r, main_url)
         session_count_after = Session.objects.filter(meeting=meeting, group=group, type='regular').count()
         self.assertEqual(session_count_after, session_count_before + 1)
@@ -535,42 +520,6 @@ def test_submit_request(self):
         )
         self.assertEqual(set(list(session.joint_with_groups.all())), set([group3, group4]))
 
-    def test_submit_request_invalid(self):
-        MeetingFactory(type_id='ietf', date=date_today())
-        ad = Person.objects.get(user__username='ad')
-        area = RoleFactory(name_id='ad', person=ad, group__type_id='area').group
-        group = GroupFactory(parent=area)
-        url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym})
-        attendees = '10'
-        comments = 'need projector'
-        post_data = {
-            'num_session':'2',
-            'attendees':attendees,
-            'constraint_chair_conflict':'',
-            'comments':comments,
-            'session_set-TOTAL_FORMS': '1',
-            'session_set-INITIAL_FORMS': '1',
-            'session_set-MIN_NUM_FORMS': '1',
-            'session_set-MAX_NUM_FORMS': '3',
-            # no 'session_set-0-id' to create a new session
-            'session_set-0-name': '',
-            'session_set-0-short': '',
-            'session_set-0-purpose': 'regular',
-            'session_set-0-type': 'regular',
-            'session_set-0-requested_duration': '3600',
-            'session_set-0-on_agenda': True,
-            'session_set-0-remote_instructions': '',
-            'session_set-0-attendees': attendees,
-            'session_set-0-comments': comments,
-            'session_set-0-DELETE': '',
-        }
-        self.client.login(username="secretary", password="secretary+password")
-        r = self.client.post(url,post_data)
-        self.assertEqual(r.status_code, 200)
-        q = PyQuery(r.content)
-        self.assertEqual(len(q('#session-request-form')),1)
-        self.assertContains(r, 'Must provide data for all sessions')
-
     def test_submit_request_check_constraints(self):
         m1 = MeetingFactory(type_id='ietf', date=date_today() - datetime.timedelta(days=100))
         MeetingFactory(type_id='ietf', date=date_today(),
@@ -597,7 +546,7 @@ def test_submit_request_check_constraints(self):
 
         self.client.login(username="secretary", password="secretary+password")
 
-        url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym})
+        url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym})
         r = self.client.get(url + '?previous')
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
@@ -607,11 +556,11 @@ def test_submit_request_check_constraints(self):
 
         attendees = '10'
         comments = 'need projector'
-        post_data = {'num_session':'1',
-                     'attendees':attendees,
+        post_data = {'num_session': '1',
+                     'attendees': attendees,
                      'constraint_chair_conflict': group.acronym,
-                     'comments':comments,
-                     'session_set-TOTAL_FORMS': '1',
+                     'comments': comments,
+                     'session_set-TOTAL_FORMS': '3',
                      'session_set-INITIAL_FORMS': '1',
                      'session_set-MIN_NUM_FORMS': '1',
                      'session_set-MAX_NUM_FORMS': '3',
@@ -626,11 +575,31 @@ def test_submit_request_check_constraints(self):
                      'session_set-0-attendees': attendees,
                      'session_set-0-comments': comments,
                      'session_set-0-DELETE': '',
+                     'session_set-1-name': '',
+                     'session_set-1-short': '',
+                     'session_set-1-purpose': session.purpose_id,
+                     'session_set-1-type': session.type_id,
+                     'session_set-1-requested_duration': '',
+                     'session_set-1-on_agenda': session.on_agenda,
+                     'session_set-1-remote_instructions': '',
+                     'session_set-1-attendees': attendees,
+                     'session_set-1-comments': '',
+                     'session_set-1-DELETE': 'on',
+                     'session_set-2-name': '',
+                     'session_set-2-short': '',
+                     'session_set-2-purpose': session.purpose_id,
+                     'session_set-2-type': session.type_id,
+                     'session_set-2-requested_duration': '',
+                     'session_set-2-on_agenda': session.on_agenda,
+                     'session_set-2-remote_instructions': '',
+                     'session_set-2-attendees': attendees,
+                     'session_set-2-comments': '',
+                     'session_set-2-DELETE': 'on',
                      'submit': 'Continue'}
-        r = self.client.post(url,post_data)
+        r = self.client.post(url, post_data)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('#session-request-form')),1)
+        self.assertEqual(len(q('#session-request-form')), 1)
         self.assertContains(r, "Cannot declare a conflict with the same group")
 
     def test_request_notification(self):
@@ -645,18 +614,18 @@ def test_request_notification(self):
         RoleFactory(name_id='chair', group=group, person__user__username='ameschairman')
         resource = ResourceAssociation.objects.create(name_id='project')
         # Bit of a test data hack - the fixture now has no used resources to pick from
-        resource.name.used=True
+        resource.name.used = True
         resource.name.save()
 
-        url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym})
-        confirm_url = reverse('ietf.secr.sreq.views.confirm',kwargs={'acronym':group.acronym})
+        url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym})
+        confirm_url = reverse('ietf.meeting.views_session_request.confirm', kwargs={'acronym': group.acronym})
         len_before = len(outbox)
         attendees = '10'
-        post_data = {'num_session':'2',
-                     'attendees':attendees,
-                     'bethere':str(ad.pk),
-                     'constraint_chair_conflict':group4.acronym,
-                     'comments':'',
+        post_data = {'num_session': '2',
+                     'attendees': attendees,
+                     'bethere': str(ad.pk),
+                     'constraint_chair_conflict': group4.acronym,
+                     'comments': '',
                      'resources': resource.pk,
                      'session_time_relation': 'subsequent-days',
                      'adjacent_with_wg': group2.acronym,
@@ -692,23 +661,23 @@ def test_request_notification(self):
                      'submit': 'Continue'}
         self.client.login(username="ameschairman", password="ameschairman+password")
         # submit
-        r = self.client.post(url,post_data)
+        r = self.client.post(url, post_data)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertTrue('Confirm' in str(q("title")), r.context['form'].errors)
         # confirm
         post_data['submit'] = 'Submit'
-        r = self.client.post(confirm_url,post_data)
-        self.assertRedirects(r, reverse('ietf.secr.sreq.views.main'))
-        self.assertEqual(len(outbox),len_before+1)
+        r = self.client.post(confirm_url, post_data)
+        self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view'))
+        self.assertEqual(len(outbox), len_before + 1)
         notification = outbox[-1]
         notification_payload = get_payload_text(notification)
-        sessions = Session.objects.filter(meeting=meeting,group=group)
+        sessions = Session.objects.filter(meeting=meeting, group=group)
         self.assertEqual(len(sessions), 2)
         session = sessions[0]
 
-        self.assertEqual(session.resources.count(),1)
-        self.assertEqual(session.people_constraints.count(),1)
+        self.assertEqual(session.resources.count(), 1)
+        self.assertEqual(session.people_constraints.count(), 1)
         self.assertEqual(session.constraints().get(name='time_relation').time_relation, 'subsequent-days')
         self.assertEqual(session.constraints().get(name='wg_adjacent').target.acronym, group2.acronym)
         self.assertEqual(
@@ -731,7 +700,7 @@ def test_request_notification(self):
     def test_request_notification_msg(self):
         to = "<iesg-secretary@ietf.org>"
         subject = "Dummy subject"
-        template = "sreq/session_request_notification.txt"
+        template = "meeting/session_request_notification.txt"
         header = "A new"
         meeting = MeetingFactory(type_id="ietf", date=date_today())
         requester = PersonFactory(name="James O'Rourke", user__username="jimorourke")
@@ -767,19 +736,19 @@ def test_request_notification_third_session(self):
         RoleFactory(name_id='chair', group=group, person__user__username='ameschairman')
         resource = ResourceAssociation.objects.create(name_id='project')
         # Bit of a test data hack - the fixture now has no used resources to pick from
-        resource.name.used=True
+        resource.name.used = True
         resource.name.save()
 
-        url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym})
-        confirm_url = reverse('ietf.secr.sreq.views.confirm',kwargs={'acronym':group.acronym})
+        url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym})
+        confirm_url = reverse('ietf.meeting.views_session_request.confirm', kwargs={'acronym': group.acronym})
         len_before = len(outbox)
         attendees = '10'
-        post_data = {'num_session':'2',
+        post_data = {'num_session': '2',
                      'third_session': 'true',
-                     'attendees':attendees,
-                     'bethere':str(ad.pk),
-                     'constraint_chair_conflict':group4.acronym,
-                     'comments':'',
+                     'attendees': attendees,
+                     'bethere': str(ad.pk),
+                     'constraint_chair_conflict': group4.acronym,
+                     'comments': '',
                      'resources': resource.pk,
                      'session_time_relation': 'subsequent-days',
                      'adjacent_with_wg': group2.acronym,
@@ -826,23 +795,23 @@ def test_request_notification_third_session(self):
                      'submit': 'Continue'}
         self.client.login(username="ameschairman", password="ameschairman+password")
         # submit
-        r = self.client.post(url,post_data)
+        r = self.client.post(url, post_data)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertTrue('Confirm' in str(q("title")), r.context['form'].errors)
         # confirm
         post_data['submit'] = 'Submit'
-        r = self.client.post(confirm_url,post_data)
-        self.assertRedirects(r, reverse('ietf.secr.sreq.views.main'))
-        self.assertEqual(len(outbox),len_before+1)
+        r = self.client.post(confirm_url, post_data)
+        self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view'))
+        self.assertEqual(len(outbox), len_before + 1)
         notification = outbox[-1]
         notification_payload = get_payload_text(notification)
-        sessions = Session.objects.filter(meeting=meeting,group=group)
+        sessions = Session.objects.filter(meeting=meeting, group=group)
         self.assertEqual(len(sessions), 3)
         session = sessions[0]
 
-        self.assertEqual(session.resources.count(),1)
-        self.assertEqual(session.people_constraints.count(),1)
+        self.assertEqual(session.resources.count(), 1)
+        self.assertEqual(session.people_constraints.count(), 1)
         self.assertEqual(session.constraints().get(name='time_relation').time_relation, 'subsequent-days')
         self.assertEqual(session.constraints().get(name='wg_adjacent').target.acronym, group2.acronym)
         self.assertEqual(
@@ -861,16 +830,17 @@ def test_request_notification_third_session(self):
         self.assertIn('1 Hour, 1 Hour, 1 Hour', notification_payload)
         self.assertIn('The third session requires your approval', notification_payload)
 
+
 class LockAppTestCase(TestCase):
     def setUp(self):
         super().setUp()
-        self.meeting = MeetingFactory(type_id='ietf', date=date_today(),session_request_lock_message='locked')
+        self.meeting = MeetingFactory(type_id='ietf', date=date_today(), session_request_lock_message='locked')
         self.group = GroupFactory(acronym='mars')
         RoleFactory(name_id='chair', group=self.group, person__user__username='marschairman')
-        SessionFactory(group=self.group,meeting=self.meeting)
+        SessionFactory(group=self.group, meeting=self.meeting)
 
     def test_edit_request(self):
-        url = reverse('ietf.secr.sreq.views.edit',kwargs={'acronym':self.group.acronym})
+        url = reverse('ietf.meeting.views_session_request.edit_request', kwargs={'acronym': self.group.acronym})
         self.client.login(username="secretary", password="secretary+password")
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
@@ -882,48 +852,49 @@ def test_edit_request(self):
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertEqual(len(q(':disabled[name="submit"]')), 1)
-    
+
     def test_view_request(self):
-        url = reverse('ietf.secr.sreq.views.view',kwargs={'acronym':self.group.acronym})
+        url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': self.group.acronym})
         self.client.login(username="secretary", password="secretary+password")
-        r = self.client.get(url,follow=True)
+        r = self.client.get(url, follow=True)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertEqual(len(q(':enabled[name="edit"]')), 1)  # secretary can edit
         chair = self.group.role_set.filter(name_id='chair').first().person.user.username
         self.client.login(username=chair, password=f'{chair}+password')
-        r = self.client.get(url,follow=True)
+        r = self.client.get(url, follow=True)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
         self.assertEqual(len(q(':disabled[name="edit"]')), 1)  # chair cannot edit
 
     def test_new_request(self):
-        url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':self.group.acronym})
-        
+        url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': self.group.acronym})
+
         # try as WG Chair
         self.client.login(username="marschairman", password="marschairman+password")
         r = self.client.get(url, follow=True)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('#session-request-form')),0)
-        
+        self.assertEqual(len(q('#session-request-form')), 0)
+
         # try as Secretariat
         self.client.login(username="secretary", password="secretary+password")
-        r = self.client.get(url,follow=True)
+        r = self.client.get(url, follow=True)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertEqual(len(q('#session-request-form')),1)
-    
+        self.assertEqual(len(q('#session-request-form')), 1)
+
+
 class NotMeetingCase(TestCase):
     def test_not_meeting(self):
-        MeetingFactory(type_id='ietf',date=date_today())
+        MeetingFactory(type_id='ietf', date=date_today())
         group = GroupFactory(acronym='mars')
-        url = reverse('ietf.secr.sreq.views.no_session',kwargs={'acronym':group.acronym}) 
+        url = reverse('ietf.meeting.views_session_request.no_session', kwargs={'acronym': group.acronym})
         self.client.login(username="secretary", password="secretary+password")
 
         empty_outbox()
 
-        r = self.client.get(url,follow=True)
+        r = self.client.get(url, follow=True)
         # If the view invoked by that get throws an exception (such as an integrity error),
         # the traceback from this test will talk about a TransactionManagementError and
         # yell about executing queries before the end of an 'atomic' block
@@ -932,14 +903,15 @@ def test_not_meeting(self):
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, 'A message was sent to notify not having a session')
 
-        r = self.client.get(url,follow=True)
+        r = self.client.get(url, follow=True)
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, 'is already marked as not meeting')
 
-        self.assertEqual(len(outbox),1)
+        self.assertEqual(len(outbox), 1)
         self.assertTrue('Not having a session' in outbox[0]['Subject'])
         self.assertTrue('session-request@' in outbox[0]['To'])
 
+
 class RetrievePreviousCase(TestCase):
     pass
 
@@ -949,7 +921,7 @@ class RetrievePreviousCase(TestCase):
     # test access by unauthorized
 
 
-class SessionFormTest(TestCase):
+class SessionRequestFormTest(TestCase):
     def setUp(self):
         super().setUp()
         self.meeting = MeetingFactory(type_id='ietf')
@@ -1014,19 +986,19 @@ def setUp(self):
             'session_set-2-comments': '',
             'session_set-2-DELETE': '',
         }
-        
+
     def test_valid(self):
         # Test with three sessions
-        form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting)
+        form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting)
         self.assertTrue(form.is_valid())
-        
+
         # Test with two sessions
         self.valid_form_data.update({
             'third_session': '',
             'session_set-TOTAL_FORMS': '2',
             'joint_for_session': '2'
         })
-        form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting)
+        form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting)
         self.assertTrue(form.is_valid())
 
         # Test with one session
@@ -1036,9 +1008,9 @@ def test_valid(self):
             'joint_for_session': '1',
             'session_time_relation': '',
         })
-        form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting)
+        form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting)
         self.assertTrue(form.is_valid())
-    
+
     def test_invalid_groups(self):
         new_form_data = {
             'constraint_chair_conflict': 'doesnotexist',
@@ -1057,7 +1029,7 @@ def test_valid_group_appears_in_multiple_conflicts(self):
             'constraint_tech_overlap': self.group2.acronym,
         }
         self.valid_form_data.update(new_form_data)
-        form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting)
+        form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting)
         self.assertTrue(form.is_valid())
 
     def test_invalid_group_appears_in_multiple_conflicts(self):
@@ -1116,7 +1088,7 @@ def test_invalid_joint_for_session(self):
                              'joint_for_session': [
                                  'Session 2 can not be the joint session, the session has not been requested.']
                          })
-    
+
     def test_invalid_missing_session_length(self):
         form = self._invalid_test_helper({
             'session_set-TOTAL_FORMS': '2',
@@ -1156,6 +1128,6 @@ def test_invalid_missing_session_length(self):
 
     def _invalid_test_helper(self, new_form_data):
         form_data = dict(self.valid_form_data, **new_form_data)
-        form = SessionForm(data=form_data, group=self.group1, meeting=self.meeting)
+        form = SessionRequestForm(data=form_data, group=self.group1, meeting=self.meeting)
         self.assertFalse(form.is_valid())
         return form
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index bd3ab772fc..b1bbc62907 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2009-2024, All Rights Reserved
+# Copyright The IETF Trust 2009-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 import datetime
 import io
@@ -7554,7 +7554,7 @@ def test_meeting_requests(self):
         )
         def _sreq_edit_link(sess):
             return urlreverse(
-                'ietf.secr.sreq.views.edit',
+                'ietf.meeting.views_session_request.edit_request',
                 kwargs={
                     'num': meeting.number,
                     'acronym': sess.group.acronym,
diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py
index 18b123b4d8..af36a6656c 100644
--- a/ietf/meeting/urls.py
+++ b/ietf/meeting/urls.py
@@ -1,10 +1,10 @@
-# Copyright The IETF Trust 2007-2024, All Rights Reserved
+# Copyright The IETF Trust 2007-2025, All Rights Reserved
 
 from django.conf import settings
 from django.urls import include
 from django.views.generic import RedirectView
 
-from ietf.meeting import views, views_proceedings
+from ietf.meeting import views, views_proceedings, views_session_request
 from ietf.utils.urls import url
 
 class AgendaRedirectView(RedirectView):
@@ -108,6 +108,8 @@ def get_redirect_url(self, *args, **kwargs):
     url(r'^important-dates.(?P<output_format>ics)$', views.important_dates),
     url(r'^proceedings/meetinghosts/edit/', views_proceedings.edit_meetinghosts),
     url(r'^proceedings/meetinghosts/(?P<host_id>\d+)/logo/$', views_proceedings.meetinghost_logo),
+    url(r'^session/request/%(acronym)s/edit/$' % settings.URL_REGEXPS, views_session_request.edit_request),
+    url(r'^session/request/%(acronym)s/view/$' % settings.URL_REGEXPS, views_session_request.view_request),
 ]
 
 urlpatterns = [
@@ -127,6 +129,13 @@ def get_redirect_url(self, *args, **kwargs):
     url(r'^upcoming/?$', views.upcoming),
     url(r'^upcoming\.ics/?$', views.upcoming_ical),
     url(r'^upcoming\.json/?$', views.upcoming_json),
+    url(r'^session/request/$', views_session_request.list_view),
+    url(r'^session/request/%(acronym)s/new/$' % settings.URL_REGEXPS, views_session_request.new_request),
+    url(r'^session/request/%(acronym)s/approve/$' % settings.URL_REGEXPS, views_session_request.approve_request),
+    url(r'^session/request/%(acronym)s/no_session/$' % settings.URL_REGEXPS, views_session_request.no_session),
+    url(r'^session/request/%(acronym)s/cancel/$' % settings.URL_REGEXPS, views_session_request.cancel_request),
+    url(r'^session/request/%(acronym)s/confirm/$' % settings.URL_REGEXPS, views_session_request.confirm),
+    url(r'^session/request/status/$', views_session_request.status),
     url(r'^session/(?P<session_id>\d+)/agenda_materials$', views.session_materials),
     url(r'^session/(?P<session_id>\d+)/cancel/?', views.cancel_session),
     url(r'^session/(?P<session_id>\d+)/edit/?', views.edit_session),
@@ -140,4 +149,3 @@ def get_redirect_url(self, *args, **kwargs):
     url(r'^(?P<num>\d+)/', include(safe_for_all_meeting_types)),
     url(r'^(?P<num>interim-[a-z0-9-]+)/', include(safe_for_all_meeting_types)),
 ]
-
diff --git a/ietf/secr/sreq/views.py b/ietf/meeting/views_session_request.py
similarity index 80%
rename from ietf/secr/sreq/views.py
rename to ietf/meeting/views_session_request.py
index eb93168e1c..a1ef74f1b8 100644
--- a/ietf/secr/sreq/views.py
+++ b/ietf/meeting/views_session_request.py
@@ -1,29 +1,26 @@
-# Copyright The IETF Trust 2013-2022, All Rights Reserved
+# Copyright The IETF Trust 2007-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
-
 import datetime
 import inflect
 from collections import defaultdict, OrderedDict
 
 from django.conf import settings
 from django.contrib import messages
+from django.core.exceptions import ObjectDoesNotExist
 from django.db.models import Q
 from django.shortcuts import render, get_object_or_404, redirect
 from django.http import Http404
 
-import debug                            # pyflakes:ignore
-
 from ietf.group.models import Group, GroupFeatures
 from ietf.ietfauth.utils import has_role, role_required
-from ietf.meeting.models import Meeting, Session, Constraint, ResourceAssociation, SchedulingEvent
 from ietf.meeting.helpers import get_meeting
+from ietf.meeting.models import Session, Meeting, Constraint, ResourceAssociation, SchedulingEvent
 from ietf.meeting.utils import add_event_info_to_session_qs
-from ietf.name.models import SessionStatusName, ConstraintName
-from ietf.secr.sreq.forms import (SessionForm, ToolStatusForm, allowed_conflicting_groups,
+from ietf.meeting.forms import (SessionRequestStatusForm, SessionRequestForm, allowed_conflicting_groups,
     JOINT_FOR_SESSION_CHOICES)
+from ietf.name.models import SessionStatusName, ConstraintName
 from ietf.secr.utils.decorators import check_permissions
-from ietf.secr.utils.group import get_my_groups
 from ietf.utils.mail import send_mail
 from ietf.mailtrigger.utils import gather_address_lists
 
@@ -31,12 +28,25 @@
 # Globals
 # -------------------------------------------------
 # TODO: This needs to be replaced with something that pays attention to groupfeatures
-AUTHORIZED_ROLES=('WG Chair','WG Secretary','RG Chair','IAB Group Chair','Area Director','Secretariat','Team Chair','IRTF Chair','Program Chair','Program Lead','Program Secretary', 'EDWG Chair')
+AUTHORIZED_ROLES = (
+    'WG Chair',
+    'WG Secretary',
+    'RG Chair',
+    'IAB Group Chair',
+    'Area Director',
+    'Secretariat',
+    'Team Chair',
+    'IRTF Chair',
+    'Program Chair',
+    'Program Lead',
+    'Program Secretary',
+    'EDWG Chair')
 
 # -------------------------------------------------
 # Helper Functions
 # -------------------------------------------------
 
+
 def check_app_locked(meeting=None):
     '''
     This function returns True if the application is locked to non-secretariat users.
@@ -45,6 +55,54 @@ def check_app_locked(meeting=None):
         meeting = get_meeting(days=14)
     return bool(meeting.session_request_lock_message)
 
+
+def get_lock_message(meeting=None):
+    '''
+    Returns the message to display to non-secretariat users when the tool is locked.
+    '''
+    if not meeting:
+        meeting = get_meeting(days=14)
+    return meeting.session_request_lock_message
+
+
+def get_my_groups(user, conclude=False):
+    '''
+    Takes a Django user object (from request)
+    Returns a list of groups the user has access to.  Rules are as follows
+    secretariat - has access to all groups
+    area director - has access to all groups in their area
+    wg chair or secretary - has access to their own group
+    chair of irtf has access to all irtf groups
+
+    If user=None than all groups are returned.
+    concluded=True means include concluded groups.  Need this to upload materials for groups
+    after they've been concluded.  it happens.
+    '''
+    my_groups = set()
+    states = ['bof', 'proposed', 'active']
+    if conclude:
+        states.extend(['conclude', 'bof-conc'])
+
+    all_groups = Group.objects.filter(type__features__has_meetings=True, state__in=states).order_by('acronym')
+    if user is None or has_role(user, 'Secretariat'):
+        return all_groups
+
+    try:
+        person = user.person
+    except ObjectDoesNotExist:
+        return list()
+
+    for group in all_groups:
+        if group.role_set.filter(person=person, name__in=('chair', 'secr', 'ad')):
+            my_groups.add(group)
+            continue
+        if group.parent and group.parent.role_set.filter(person=person, name__in=('ad', 'chair')):
+            my_groups.add(group)
+            continue
+
+    return list(my_groups)
+
+
 def get_initial_session(sessions, prune_conflicts=False):
     '''
     This function takes a queryset of sessions ordered by 'id' for consistency.  It returns
@@ -97,13 +155,43 @@ def valid_conflict(conflict):
             initial['joint_for_session_display'] = dict(JOINT_FOR_SESSION_CHOICES)[initial['joint_for_session']]
     return initial
 
-def get_lock_message(meeting=None):
+
+def inbound_session_conflicts_as_string(group, meeting):
     '''
-    Returns the message to display to non-secretariat users when the tool is locked.
+    Takes a Group object and Meeting object and returns a string of other groups which have
+    a conflict with this one
     '''
-    if not meeting:
-        meeting = get_meeting(days=14)
-    return meeting.session_request_lock_message
+    constraints = group.constraint_target_set.filter(meeting=meeting, name__is_group_conflict=True)
+    group_set = set(constraints.values_list('source__acronym', flat=True))  # set to de-dupe
+    group_list = sorted(group_set)  # give a consistent order
+    return ', '.join(group_list)
+
+
+def get_outbound_conflicts(form: SessionRequestForm):
+    """extract wg conflict constraint data from a SessionForm"""
+    outbound_conflicts = []
+    for conflictname, cfield_id in form.wg_constraint_field_ids():
+        conflict_groups = form.cleaned_data[cfield_id]
+        if len(conflict_groups) > 0:
+            outbound_conflicts.append(dict(name=conflictname, groups=conflict_groups))
+    return outbound_conflicts
+
+
+def save_conflicts(group, meeting, conflicts, name):
+    '''
+    This function takes a Group, Meeting a string which is a list of Groups acronyms (conflicts),
+    and the constraint name (conflict|conflic2|conflic3) and creates Constraint records
+    '''
+    constraint_name = ConstraintName.objects.get(slug=name)
+    acronyms = conflicts.replace(',',' ').split()
+    for acronym in acronyms:
+        target = Group.objects.get(acronym=acronym)
+
+        constraint = Constraint(source=group,
+                                target=target,
+                                meeting=meeting,
+                                name=constraint_name)
+        constraint.save()
 
 
 def get_requester_text(person, group):
@@ -129,22 +217,6 @@ def get_requester_text(person, group):
         )
 
 
-def save_conflicts(group, meeting, conflicts, name):
-    '''
-    This function takes a Group, Meeting a string which is a list of Groups acronyms (conflicts),
-    and the constraint name (conflict|conflic2|conflic3) and creates Constraint records
-    '''
-    constraint_name = ConstraintName.objects.get(slug=name)
-    acronyms = conflicts.replace(',',' ').split()
-    for acronym in acronyms:
-        target = Group.objects.get(acronym=acronym)
-
-        constraint = Constraint(source=group,
-                                target=target,
-                                meeting=meeting,
-                                name=constraint_name)
-        constraint.save()
-
 def send_notification(group, meeting, login, sreq_data, session_data, action):
     '''
     This function generates email notifications for various session request activities.
@@ -152,10 +224,10 @@ def send_notification(group, meeting, login, sreq_data, session_data, action):
     session_data is an array of data from individual session subforms
     action argument is a string [new|update].
     '''
-    (to_email, cc_list) = gather_address_lists('session_requested',group=group,person=login)
+    (to_email, cc_list) = gather_address_lists('session_requested', group=group, person=login)
     from_email = (settings.SESSION_REQUEST_FROM_EMAIL)
     subject = '%s - New Meeting Session Request for IETF %s' % (group.acronym, meeting.number)
-    template = 'sreq/session_request_notification.txt'
+    template = 'meeting/session_request_notification.txt'
 
     # send email
     context = {}
@@ -164,7 +236,7 @@ def send_notification(group, meeting, login, sreq_data, session_data, action):
     context['meeting'] = meeting
     context['login'] = login
     context['header'] = 'A new'
-    context['requester'] = get_requester_text(login,group)
+    context['requester'] = get_requester_text(login, group)
 
     # update overrides
     if action == 'update':
@@ -174,10 +246,10 @@ def send_notification(group, meeting, login, sreq_data, session_data, action):
     # if third session requested approval is required
     # change headers TO=ADs, CC=session-request, submitter and cochairs
     if len(session_data) > 2:
-        (to_email, cc_list) = gather_address_lists('session_requested_long',group=group,person=login)
+        (to_email, cc_list) = gather_address_lists('session_requested_long', group=group, person=login)
         subject = '%s - Request for meeting session approval for IETF %s' % (group.acronym, meeting.number)
-        template = 'sreq/session_approval_notification.txt'
-        #status_text = 'the %s Directors for approval' % group.parent
+        template = 'meeting/session_approval_notification.txt'
+        # status_text = 'the %s Directors for approval' % group.parent
 
     context['session_lengths'] = [sd['requested_duration'] for sd in session_data]
 
@@ -189,103 +261,188 @@ def send_notification(group, meeting, login, sreq_data, session_data, action):
               context,
               cc=cc_list)
 
-def inbound_session_conflicts_as_string(group, meeting):
-    '''
-    Takes a Group object and Meeting object and returns a string of other groups which have
-    a conflict with this one
-    '''
-    constraints = group.constraint_target_set.filter(meeting=meeting, name__is_group_conflict=True)
-    group_set = set(constraints.values_list('source__acronym', flat=True))  # set to de-dupe
-    group_list = sorted(group_set)  # give a consistent order
-    return ', '.join(group_list)
+
+def session_changed(session):
+    latest_event = SchedulingEvent.objects.filter(session=session).order_by('-time', '-id').first()
+
+    if latest_event and latest_event.status_id == "schedw" and session.meeting.schedule is not None:
+        # send an email to iesg-secretariat to alert to change
+        pass
+
+
+def status_slug_for_new_session(session, session_number):
+    if session.group.features.acts_like_wg and session_number == 2:
+        return 'apprw'
+    return 'schedw'
 
 # -------------------------------------------------
 # View Functions
 # -------------------------------------------------
-@check_permissions
-def approve(request, acronym):
+
+
+@role_required(*AUTHORIZED_ROLES)
+def list_view(request):
     '''
-    This view approves the third session.  For use by ADs or Secretariat.
+    Display list of groups the user has access to.
     '''
     meeting = get_meeting(days=14)
-    group = get_object_or_404(Group, acronym=acronym)
 
-    session = add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group=group)).filter(current_status='apprw').first()
-    if session is None:
-        raise Http404
+    # check for locked flag
+    is_locked = check_app_locked()
+    if is_locked and not has_role(request.user, 'Secretariat'):
+        message = get_lock_message()
+        return render(request, 'meeting/session_request_locked.html', {
+            'message': message,
+            'meeting': meeting})
 
-    if has_role(request.user,'Secretariat') or group.parent.role_set.filter(name='ad',person=request.user.person):
-        SchedulingEvent.objects.create(
-            session=session,
-            status=SessionStatusName.objects.get(slug='appr'),
-            by=request.user.person,
-        )
-        session_changed(session)
+    scheduled_groups = []
+    unscheduled_groups = []
 
-        messages.success(request, 'Third session approved')
-        return redirect('ietf.secr.sreq.views.view', acronym=acronym)
-    else:
-        # if an unauthorized user gets here return error
-        messages.error(request, 'Not authorized to approve the third session')
-        return redirect('ietf.secr.sreq.views.view', acronym=acronym)
+    group_types = GroupFeatures.objects.filter(has_meetings=True).values_list('type', flat=True)
 
-@check_permissions
-def cancel(request, acronym):
-    '''
-    This view cancels a session request and sends a notification.
-    To cancel, or withdraw the request set status = deleted.
-    "canceled" status is used by the secretariat.
+    my_groups = [g for g in get_my_groups(request.user, conclude=True) if g.type_id in group_types]
 
-    NOTE: this function can also be called after a session has been
-    scheduled during the period when the session request tool is
-    reopened.  In this case be sure to clear the timeslot assignment as well.
+    sessions_by_group = defaultdict(list)
+    for s in add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group__in=my_groups)).filter(current_status__in=['schedw', 'apprw', 'appr', 'sched']):
+        sessions_by_group[s.group_id].append(s)
+
+    for group in my_groups:
+        group.meeting_sessions = sessions_by_group.get(group.pk, [])
+
+        if group.pk in sessions_by_group:
+            # include even if concluded as we need to to see that the
+            # sessions are there
+            scheduled_groups.append(group)
+        else:
+            if group.state_id not in ['conclude', 'bof-conc']:
+                # too late for unscheduled if concluded
+                unscheduled_groups.append(group)
+
+    # warn if there are no associated groups
+    if not scheduled_groups and not unscheduled_groups:
+        messages.warning(request, 'The account %s is not associated with any groups.  If you have multiple Datatracker accounts you may try another or report a problem to %s' % (request.user, settings.SECRETARIAT_ACTION_EMAIL))
+
+    # add session status messages for use in template
+    for group in scheduled_groups:
+        if not group.features.acts_like_wg or (len(group.meeting_sessions) < 3):
+            group.status_message = group.meeting_sessions[0].current_status
+        else:
+            group.status_message = 'First two sessions: %s, Third session: %s' % (group.meeting_sessions[0].current_status, group.meeting_sessions[2].current_status)
+
+    # add not meeting indicators for use in template
+    for group in unscheduled_groups:
+        if any(s.current_status == 'notmeet' for s in group.meeting_sessions):
+            group.not_meeting = True
+
+    return render(request, 'meeting/session_request_list.html', {
+        'is_locked': is_locked,
+        'meeting': meeting,
+        'scheduled_groups': scheduled_groups,
+        'unscheduled_groups': unscheduled_groups},
+    )
+
+
+@role_required('Secretariat')
+def status(request):
+    '''
+    This view handles locking and unlocking of the session request tool to the public.
     '''
     meeting = get_meeting(days=14)
-    group = get_object_or_404(Group, acronym=acronym)
-    sessions = Session.objects.filter(meeting=meeting,group=group).order_by('id')
-    login = request.user.person
+    is_locked = check_app_locked(meeting=meeting)
 
-    # delete conflicts
-    Constraint.objects.filter(meeting=meeting,source=group).delete()
+    if request.method == 'POST':
+        button_text = request.POST.get('submit', '')
+        if button_text == 'Back':
+            return redirect('ietf.meeting.views_session_request.list_view')
 
-    # mark sessions as deleted
-    for session in sessions:
-        SchedulingEvent.objects.create(
-            session=session,
-            status=SessionStatusName.objects.get(slug='deleted'),
-            by=request.user.person,
-        )
-        session_changed(session)
+        form = SessionRequestStatusForm(request.POST)
 
-        # clear schedule assignments if already scheduled
-        session.timeslotassignments.all().delete()
+        if button_text == 'Lock':
+            if form.is_valid():
+                meeting.session_request_lock_message = form.cleaned_data['message']
+                meeting.save()
+                messages.success(request, 'Session Request Tool is now Locked')
+                return redirect('ietf.meeting.views_session_request.list_view')
 
-    # send notifitcation
-    (to_email, cc_list) = gather_address_lists('session_request_cancelled',group=group,person=login)
-    from_email = (settings.SESSION_REQUEST_FROM_EMAIL)
-    subject = '%s - Cancelling a meeting request for IETF %s' % (group.acronym, meeting.number)
-    send_mail(request, to_email, from_email, subject, 'sreq/session_cancel_notification.txt',
-              {'requester':get_requester_text(login,group),
-               'meeting':meeting}, cc=cc_list)
+        elif button_text == 'Unlock':
+            meeting.session_request_lock_message = ''
+            meeting.save()
+            messages.success(request, 'Session Request Tool is now Unlocked')
+            return redirect('ietf.meeting.views_session_request.list_view')
 
-    messages.success(request, 'The %s Session Request has been cancelled' % group.acronym)
-    return redirect('ietf.secr.sreq.views.main')
+    else:
+        if is_locked:
+            message = get_lock_message()
+            initial = {'message': message}
+            form = SessionRequestStatusForm(initial=initial)
+        else:
+            form = SessionRequestStatusForm()
 
+    return render(request, 'meeting/session_request_status.html', {
+        'is_locked': is_locked,
+        'form': form},
+    )
 
-def status_slug_for_new_session(session, session_number):
-    if session.group.features.acts_like_wg and session_number == 2:
-        return 'apprw'
-    return 'schedw'
 
+@check_permissions
+def new_request(request, acronym):
+    '''
+    This view gathers details for a new session request.  The user proceeds to confirm()
+    to create the request.
+    '''
+    group = get_object_or_404(Group, acronym=acronym)
+    if len(group.features.session_purposes) == 0:
+        raise Http404(f'Cannot request sessions for group "{acronym}"')
+    meeting = get_meeting(days=14)
+    session_conflicts = dict(inbound=inbound_session_conflicts_as_string(group, meeting))
 
-def get_outbound_conflicts(form: SessionForm):
-    """extract wg conflict constraint data from a SessionForm"""
-    outbound_conflicts = []
-    for conflictname, cfield_id in form.wg_constraint_field_ids():
-        conflict_groups = form.cleaned_data[cfield_id]
-        if len(conflict_groups) > 0:
-            outbound_conflicts.append(dict(name=conflictname, groups=conflict_groups))
-    return outbound_conflicts
+    # check if app is locked
+    is_locked = check_app_locked()
+    if is_locked and not has_role(request.user, 'Secretariat'):
+        messages.warning(request, "The Session Request Tool is closed")
+        return redirect('ietf.meeting.views_session_request.list_view')
+
+    if request.method == 'POST':
+        button_text = request.POST.get('submit', '')
+        if button_text == 'Cancel':
+            return redirect('ietf.meeting.views_session_request.list_view')
+
+        form = SessionRequestForm(group, meeting, request.POST, notifications_optional=has_role(request.user, "Secretariat"))
+        if form.is_valid():
+            return confirm(request, acronym)
+
+    # the "previous" querystring causes the form to be returned
+    # pre-populated with data from last meeeting's session request
+    elif request.method == 'GET' and 'previous' in request.GET:
+        latest_session = add_event_info_to_session_qs(Session.objects.filter(meeting__type_id='ietf', group=group)).exclude(current_status__in=['notmeet', 'deleted', 'canceled',]).order_by('-meeting__date').first()
+        if latest_session:
+            previous_meeting = Meeting.objects.get(number=latest_session.meeting.number)
+            previous_sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=previous_meeting, group=group)).exclude(current_status__in=['notmeet', 'deleted']).order_by('id')
+            if not previous_sessions:
+                messages.warning(request, 'This group did not meet at %s' % previous_meeting)
+                return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym)
+            else:
+                messages.info(request, 'Fetched session info from %s' % previous_meeting)
+        else:
+            messages.warning(request, 'Did not find any previous meeting')
+            return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym)
+
+        initial = get_initial_session(previous_sessions, prune_conflicts=True)
+        if 'resources' in initial:
+            initial['resources'] = [x.pk for x in initial['resources']]
+        form = SessionRequestForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat"))
+
+    else:
+        initial = {}
+        form = SessionRequestForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat"))
+
+    return render(request, 'meeting/session_request_form.html', {
+        'meeting': meeting,
+        'form': form,
+        'group': group,
+        'is_create': True,
+        'session_conflicts': session_conflicts},
+    )
 
 
 @role_required(*AUTHORIZED_ROLES)
@@ -295,11 +452,11 @@ def confirm(request, acronym):
     to confirm for submission.
     '''
     # FIXME: this should be using form.is_valid/form.cleaned_data - invalid input will make it crash
-    group = get_object_or_404(Group,acronym=acronym)
+    group = get_object_or_404(Group, acronym=acronym)
     if len(group.features.session_purposes) == 0:
         raise Http404(f'Cannot request sessions for group "{acronym}"')
     meeting = get_meeting(days=14)
-    form = SessionForm(group, meeting, request.POST, hidden=True, notifications_optional=has_role(request.user, "Secretariat"))
+    form = SessionRequestForm(group, meeting, request.POST, hidden=True, notifications_optional=has_role(request.user, "Secretariat"))
     form.is_valid()
 
     login = request.user.person
@@ -307,8 +464,8 @@ def confirm(request, acronym):
     # check if request already exists for this group
     if add_event_info_to_session_qs(Session.objects.filter(group=group, meeting=meeting)).filter(Q(current_status__isnull=True) | ~Q(current_status__in=['deleted', 'notmeet'])):
         messages.warning(request, 'Sessions for working group %s have already been requested once.' % group.acronym)
-        return redirect('ietf.secr.sreq.views.main')
-                
+        return redirect('ietf.meeting.views_session_request.list_view')
+
     session_data = form.data.copy()
     # use cleaned_data for the 'bethere' field so we get the Person instances
     session_data['bethere'] = form.cleaned_data['bethere'] if 'bethere' in form.cleaned_data else []
@@ -318,7 +475,7 @@ def confirm(request, acronym):
         session_data['joint_for_session_display'] = dict(JOINT_FOR_SESSION_CHOICES)[session_data['joint_for_session']]
     if form.cleaned_data.get('timeranges'):
         session_data['timeranges_display'] = [t.desc for t in form.cleaned_data['timeranges']]
-    session_data['resources'] = [ ResourceAssociation.objects.get(pk=pk) for pk in request.POST.getlist('resources') ]
+    session_data['resources'] = [ResourceAssociation.objects.get(pk=pk) for pk in request.POST.getlist('resources')]
 
     # extract wg conflict constraint data for the view / notifications
     outbound_conflicts = get_outbound_conflicts(form)
@@ -326,7 +483,7 @@ def confirm(request, acronym):
     button_text = request.POST.get('submit', '')
     if button_text == 'Cancel':
         messages.success(request, 'Session Request has been cancelled')
-        return redirect('ietf.secr.sreq.views.main')
+        return redirect('ietf.meeting.views_session_request.list_view')
 
     if request.method == 'POST' and button_text == 'Submit':
         # delete any existing session records with status = canceled or notmeet
@@ -344,10 +501,10 @@ def confirm(request, acronym):
             if 'resources' in form.data:
                 new_session.resources.set(session_data['resources'])
             jfs = form.data.get('joint_for_session', '-1')
-            if not jfs: # jfs might be ''
+            if not jfs:  # jfs might be ''
                 jfs = '-1'
             if int(jfs) == count + 1:  # count is zero-indexed
-                groups_split = form.cleaned_data.get('joint_with_groups').replace(',',' ').split()
+                groups_split = form.cleaned_data.get('joint_with_groups').replace(',', ' ').split()
                 joint = Group.objects.filter(acronym__in=groups_split)
                 new_session.joint_with_groups.set(joint)
             new_session.save()
@@ -388,36 +545,105 @@ def confirm(request, acronym):
                 'new',
             )
 
-        status_text = 'IETF Agenda to be scheduled'
-        messages.success(request, 'Your request has been sent to %s' % status_text)
-        return redirect('ietf.secr.sreq.views.main')
+        status_text = 'IETF Agenda to be scheduled'
+        messages.success(request, 'Your request has been sent to %s' % status_text)
+        return redirect('ietf.meeting.views_session_request.list_view')
+
+    # POST from request submission
+    session_conflicts = dict(
+        outbound=outbound_conflicts,  # each is a dict with name and groups as keys
+        inbound=inbound_session_conflicts_as_string(group, meeting),
+    )
+    if form.cleaned_data.get('third_session'):
+        messages.warning(request, 'Note: Your request for a third session must be approved by an area director before being submitted to agenda@ietf.org.  Click "Submit" below to email an approval request to the area directors')
+
+    return render(request, 'meeting/session_request_confirm.html', {
+        'form': form,
+        'session': session_data,
+        'group': group,
+        'meeting': meeting,
+        'session_conflicts': session_conflicts},
+    )
+
+
+@role_required(*AUTHORIZED_ROLES)
+def view_request(request, acronym, num=None):
+    '''
+    This view displays the session request info
+    '''
+    meeting = get_meeting(num, days=14)
+    group = get_object_or_404(Group, acronym=acronym)
+    query = Session.objects.filter(meeting=meeting, group=group)
+    status_is_null = Q(current_status__isnull=True)
+    status_allowed = ~Q(current_status__in=("canceled", "notmeet", "deleted"))
+    sessions = (
+        add_event_info_to_session_qs(query)
+        .filter(status_is_null | status_allowed)
+        .order_by("id")
+    )
+
+    # check if app is locked
+    is_locked = check_app_locked()
+    if is_locked:
+        messages.warning(request, "The Session Request Tool is closed")
+
+    # if there are no session requests yet, redirect to new session request page
+    if not sessions:
+        if is_locked:
+            return redirect('ietf.meeting.views_session_request.list_view')
+        else:
+            return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym)
+
+    activities = [{
+        'act_date': e.time.strftime('%b %d, %Y'),
+        'act_time': e.time.strftime('%H:%M:%S'),
+        'activity': e.status.name,
+        'act_by': e.by,
+    } for e in sessions[0].schedulingevent_set.select_related('status', 'by')]
+
+    # gather outbound conflicts
+    outbound_dict = OrderedDict()
+    for obc in group.constraint_source_set.filter(meeting=meeting, name__is_group_conflict=True):
+        if obc.name.slug not in outbound_dict:
+            outbound_dict[obc.name.slug] = []
+        outbound_dict[obc.name.slug].append(obc.target.acronym)
 
-    # POST from request submission
     session_conflicts = dict(
-        outbound=outbound_conflicts,  # each is a dict with name and groups as keys
         inbound=inbound_session_conflicts_as_string(group, meeting),
+        outbound=[dict(name=ConstraintName.objects.get(slug=slug), groups=' '.join(groups))
+                  for slug, groups in outbound_dict.items()],
     )
-    return render(request, 'sreq/confirm.html', {
-        'form': form,
-        'session': session_data,
+
+    show_approve_button = False
+
+    # if sessions include a 3rd session waiting approval and the user is a secretariat or AD of the group
+    # display approve button
+    if any(s.current_status == 'apprw' for s in sessions):
+        if has_role(request.user, 'Secretariat') or group.parent.role_set.filter(name='ad', person=request.user.person):
+            show_approve_button = True
+
+    # build session dictionary (like querydict from new session request form) for use in template
+    session = get_initial_session(sessions)
+
+    return render(request, 'meeting/session_request_view.html', {
+        'can_edit': (not is_locked) or has_role(request.user, 'Secretariat'),
+        'can_cancel': (not is_locked) or has_role(request.user, 'Secretariat'),
+        'session': session,  # legacy processed data
+        'sessions': sessions,  # actual session instances
+        'activities': activities,
+        'meeting': meeting,
         'group': group,
-        'session_conflicts': session_conflicts},
+        'session_conflicts': session_conflicts,
+        'show_approve_button': show_approve_button},
     )
-    
 
-def session_changed(session):
-    latest_event = SchedulingEvent.objects.filter(session=session).order_by('-time', '-id').first()
-
-    if latest_event and latest_event.status_id == "schedw" and session.meeting.schedule != None:
-        # send an email to iesg-secretariat to alert to change
-        pass
 
 @check_permissions
-def edit(request, acronym, num=None):
+def edit_request(request, acronym, num=None):
     '''
     This view allows the user to edit details of the session request
     '''
-    meeting = get_meeting(num,days=14)
+    meeting = get_meeting(num, days=14)
     group = get_object_or_404(Group, acronym=acronym)
     if len(group.features.session_purposes) == 0:
         raise Http404(f'Cannot request sessions for group "{acronym}"')
@@ -443,15 +669,15 @@ def edit(request, acronym, num=None):
     login = request.user.person
 
     first_session = Session()
-    if(len(sessions) > 0):
+    if (len(sessions) > 0):
         first_session = sessions[0]
 
     if request.method == 'POST':
         button_text = request.POST.get('submit', '')
         if button_text == 'Cancel':
-            return redirect('ietf.secr.sreq.views.view', acronym=acronym)
+            return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym)
 
-        form = SessionForm(group, meeting, request.POST, initial=initial, notifications_optional=has_role(request.user, "Secretariat"))
+        form = SessionRequestForm(group, meeting, request.POST, initial=initial, notifications_optional=has_role(request.user, "Secretariat"))
         if form.is_valid():
             if form.has_changed():
                 changed_session_forms = [sf for sf in form.session_forms.forms_to_keep if sf.has_changed()]
@@ -513,11 +739,11 @@ def edit(request, acronym, num=None):
 
                 if 'resources' in form.changed_data:
                     new_resource_ids = form.cleaned_data['resources']
-                    new_resources = [ ResourceAssociation.objects.get(pk=a)
-                                      for a in new_resource_ids]
+                    new_resources = [ResourceAssociation.objects.get(pk=a)
+                                     for a in new_resource_ids]
                     first_session.resources = new_resources
 
-                if 'bethere' in form.changed_data and set(form.cleaned_data['bethere'])!=set(initial['bethere']):
+                if 'bethere' in form.changed_data and set(form.cleaned_data['bethere']) != set(initial['bethere']):
                     first_session.constraints().filter(name='bethere').delete()
                     bethere_cn = ConstraintName.objects.get(slug='bethere')
                     for p in form.cleaned_data['bethere']:
@@ -539,7 +765,7 @@ def edit(request, acronym, num=None):
 
                 # deprecated
                 # log activity
-                #add_session_activity(group,'Session Request was updated',meeting,user)
+                # add_session_activity(group,'Session Request was updated',meeting,user)
 
                 # send notification
                 if form.cleaned_data.get("send_notifications"):
@@ -556,7 +782,7 @@ def edit(request, acronym, num=None):
                     )
 
             messages.success(request, 'Session Request updated')
-            return redirect('ietf.secr.sreq.views.view', acronym=acronym)
+            return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym)
 
     else:  # method is not POST
         # gather outbound conflicts for initial value
@@ -567,142 +793,46 @@ def edit(request, acronym, num=None):
             initial['constraint_{}'.format(slug)] = ' '.join(groups)
 
         if not sessions:
-            return redirect('ietf.secr.sreq.views.new', acronym=acronym)
-        form = SessionForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat"))
+            return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym)
+        form = SessionRequestForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat"))
 
-    return render(request, 'sreq/edit.html', {
-        'is_locked': is_locked and not has_role(request.user,'Secretariat'),
+    return render(request, 'meeting/session_request_form.html', {
+        'is_locked': is_locked and not has_role(request.user, 'Secretariat'),
         'meeting': meeting,
         'form': form,
         'group': group,
+        'is_create': False,
         'session_conflicts': session_conflicts},
     )
 
-@role_required(*AUTHORIZED_ROLES)
-def main(request):
-    '''
-    Display list of groups the user has access to.
-
-    Template variables
-    form: a select box populated with unscheduled groups
-    meeting: the current meeting
-    scheduled_sessions:
-    '''
-    # check for locked flag
-    is_locked = check_app_locked()
-
-    if is_locked and not has_role(request.user,'Secretariat'):
-        message = get_lock_message()
-        return render(request, 'sreq/locked.html', {
-        'message': message},
-    )
-
-    meeting = get_meeting(days=14)
-
-    scheduled_groups = []
-    unscheduled_groups = []
-
-    group_types = GroupFeatures.objects.filter(has_meetings=True).values_list('type', flat=True)
-
-    my_groups = [g for g in get_my_groups(request.user, conclude=True) if g.type_id in group_types]
-
-    sessions_by_group = defaultdict(list)
-    for s in add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group__in=my_groups)).filter(current_status__in=['schedw', 'apprw', 'appr', 'sched']):
-        sessions_by_group[s.group_id].append(s)
-
-    for group in my_groups:
-        group.meeting_sessions = sessions_by_group.get(group.pk, [])
-
-        if group.pk in sessions_by_group:
-            # include even if concluded as we need to to see that the
-            # sessions are there
-            scheduled_groups.append(group)
-        else:
-            if group.state_id not in ['conclude', 'bof-conc']:
-                # too late for unscheduled if concluded
-                unscheduled_groups.append(group)
-
-    # warn if there are no associated groups
-    if not scheduled_groups and not unscheduled_groups:
-        messages.warning(request, 'The account %s is not associated with any groups.  If you have multiple Datatracker accounts you may try another or report a problem to %s' % (request.user, settings.SECRETARIAT_ACTION_EMAIL))
-     
-    # add session status messages for use in template
-    for group in scheduled_groups:
-        if not group.features.acts_like_wg or (len(group.meeting_sessions) < 3):
-            group.status_message = group.meeting_sessions[0].current_status
-        else:
-            group.status_message = 'First two sessions: %s, Third session: %s' % (group.meeting_sessions[0].current_status, group.meeting_sessions[2].current_status)
-
-    # add not meeting indicators for use in template
-    for group in unscheduled_groups:
-        if any(s.current_status == 'notmeet' for s in group.meeting_sessions):
-            group.not_meeting = True
-
-    return render(request, 'sreq/main.html', {
-        'is_locked': is_locked,
-        'meeting': meeting,
-        'scheduled_groups': scheduled_groups,
-        'unscheduled_groups': unscheduled_groups},
-    )
 
 @check_permissions
-def new(request, acronym):
+def approve_request(request, acronym):
     '''
-    This view gathers details for a new session request.  The user proceeds to confirm()
-    to create the request.
+    This view approves the third session.  For use by ADs or Secretariat.
     '''
-    group = get_object_or_404(Group, acronym=acronym)
-    if len(group.features.session_purposes) == 0:
-        raise Http404(f'Cannot request sessions for group "{acronym}"')
     meeting = get_meeting(days=14)
-    session_conflicts = dict(inbound=inbound_session_conflicts_as_string(group, meeting))
-
-    # check if app is locked
-    is_locked = check_app_locked()
-    if is_locked and not has_role(request.user,'Secretariat'):
-        messages.warning(request, "The Session Request Tool is closed")
-        return redirect('ietf.secr.sreq.views.main')
-    
-    if request.method == 'POST':
-        button_text = request.POST.get('submit', '')
-        if button_text == 'Cancel':
-            return redirect('ietf.secr.sreq.views.main')
-
-        form = SessionForm(group, meeting, request.POST, notifications_optional=has_role(request.user, "Secretariat"))
-        if form.is_valid():
-            return confirm(request, acronym)
+    group = get_object_or_404(Group, acronym=acronym)
 
-    # the "previous" querystring causes the form to be returned
-    # pre-populated with data from last meeeting's session request
-    elif request.method == 'GET' and 'previous' in request.GET:
-        latest_session = add_event_info_to_session_qs(Session.objects.filter(meeting__type_id='ietf', group=group)).exclude(current_status__in=['notmeet', 'deleted', 'canceled',]).order_by('-meeting__date').first()
-        if latest_session:
-            previous_meeting = Meeting.objects.get(number=latest_session.meeting.number)
-            previous_sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=previous_meeting, group=group)).exclude(current_status__in=['notmeet', 'deleted']).order_by('id')
-            if not previous_sessions:
-                messages.warning(request, 'This group did not meet at %s' % previous_meeting)
-                return redirect('ietf.secr.sreq.views.new', acronym=acronym)
-            else:
-                messages.info(request, 'Fetched session info from %s' % previous_meeting)
-        else:
-            messages.warning(request, 'Did not find any previous meeting')
-            return redirect('ietf.secr.sreq.views.new', acronym=acronym)
+    session = add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group=group)).filter(current_status='apprw').first()
+    if session is None:
+        raise Http404
 
-        initial = get_initial_session(previous_sessions, prune_conflicts=True)
-        if 'resources' in initial:
-            initial['resources'] = [x.pk for x in initial['resources']]
-        form = SessionForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat"))
+    if has_role(request.user, 'Secretariat') or group.parent.role_set.filter(name='ad', person=request.user.person):
+        SchedulingEvent.objects.create(
+            session=session,
+            status=SessionStatusName.objects.get(slug='appr'),
+            by=request.user.person,
+        )
+        session_changed(session)
 
+        messages.success(request, 'Third session approved')
+        return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym)
     else:
-        initial={}
-        form = SessionForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat"))
+        # if an unauthorized user gets here return error
+        messages.error(request, 'Not authorized to approve the third session')
+        return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym)
 
-    return render(request, 'sreq/new.html', {
-        'meeting': meeting,
-        'form': form,
-        'group': group,
-        'session_conflicts': session_conflicts},
-    )
 
 @check_permissions
 def no_session(request, acronym):
@@ -722,7 +852,7 @@ def no_session(request, acronym):
     # skip if state is already notmeet
     if add_event_info_to_session_qs(Session.objects.filter(group=group, meeting=meeting)).filter(current_status='notmeet'):
         messages.info(request, 'The group %s is already marked as not meeting' % group.acronym)
-        return redirect('ietf.secr.sreq.views.main')
+        return redirect('ietf.meeting.views_session_request.list_view')
 
     session = Session.objects.create(
         group=group,
@@ -740,125 +870,62 @@ def no_session(request, acronym):
     session_changed(session)
 
     # send notification
-    (to_email, cc_list) = gather_address_lists('session_request_not_meeting',group=group,person=login)
+    (to_email, cc_list) = gather_address_lists('session_request_not_meeting', group=group, person=login)
     from_email = (settings.SESSION_REQUEST_FROM_EMAIL)
     subject = '%s - Not having a session at IETF %s' % (group.acronym, meeting.number)
-    send_mail(request, to_email, from_email, subject, 'sreq/not_meeting_notification.txt',
-              {'login':login,
-               'group':group,
-               'meeting':meeting}, cc=cc_list)
+    send_mail(request, to_email, from_email, subject, 'meeting/session_not_meeting_notification.txt',
+              {'login': login,
+               'group': group,
+               'meeting': meeting}, cc=cc_list)
 
     # deprecated?
     # log activity
-    #text = 'A message was sent to notify not having a session at IETF %d' % meeting.meeting_num
-    #add_session_activity(group,text,meeting,request.person)
+    # text = 'A message was sent to notify not having a session at IETF %d' % meeting.meeting_num
+    # add_session_activity(group,text,meeting,request.person)
 
     # redirect
     messages.success(request, 'A message was sent to notify not having a session at IETF %s' % meeting.number)
-    return redirect('ietf.secr.sreq.views.main')
-
-@role_required('Secretariat')
-def tool_status(request):
-    '''
-    This view handles locking and unlocking of the tool to the public.
-    '''
-    meeting = get_meeting(days=14)
-    is_locked = check_app_locked(meeting=meeting)
-    
-    if request.method == 'POST':
-        button_text = request.POST.get('submit', '')
-        if button_text == 'Back':
-            return redirect('ietf.secr.sreq.views.main')
-
-        form = ToolStatusForm(request.POST)
-
-        if button_text == 'Lock':
-            if form.is_valid():
-                meeting.session_request_lock_message = form.cleaned_data['message']
-                meeting.save()
-                messages.success(request, 'Session Request Tool is now Locked')
-                return redirect('ietf.secr.sreq.views.main')
-
-        elif button_text == 'Unlock':
-            meeting.session_request_lock_message = ''
-            meeting.save()
-            messages.success(request, 'Session Request Tool is now Unlocked')
-            return redirect('ietf.secr.sreq.views.main')
-
-    else:
-        if is_locked:
-            message = get_lock_message()
-            initial = {'message': message}
-            form = ToolStatusForm(initial=initial)
-        else:
-            form = ToolStatusForm()
+    return redirect('ietf.meeting.views_session_request.list_view')
 
-    return render(request, 'sreq/tool_status.html', {
-        'is_locked': is_locked,
-        'form': form},
-    )
 
-@role_required(*AUTHORIZED_ROLES)
-def view(request, acronym, num = None):
+@check_permissions
+def cancel_request(request, acronym):
     '''
-    This view displays the session request info
+    This view cancels a session request and sends a notification.
+    To cancel, or withdraw the request set status = deleted.
+    "canceled" status is used by the secretariat.
+
+    NOTE: this function can also be called after a session has been
+    scheduled during the period when the session request tool is
+    reopened.  In this case be sure to clear the timeslot assignment as well.
     '''
-    meeting = get_meeting(num,days=14)
+    meeting = get_meeting(days=14)
     group = get_object_or_404(Group, acronym=acronym)
-    sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group=group)).filter(Q(current_status__isnull=True) | ~Q(current_status__in=('canceled','notmeet','deleted'))).order_by('id')
-
-    # check if app is locked
-    is_locked = check_app_locked()
-    if is_locked:
-        messages.warning(request, "The Session Request Tool is closed")
-
-    # if there are no session requests yet, redirect to new session request page
-    if not sessions:
-        if is_locked:
-            return redirect('ietf.secr.sreq.views.main')
-        else:
-            return redirect('ietf.secr.sreq.views.new', acronym=acronym)
-
-    activities = [{
-        'act_date': e.time.strftime('%b %d, %Y'),
-        'act_time': e.time.strftime('%H:%M:%S'),
-        'activity': e.status.name,
-        'act_by': e.by,
-    } for e in sessions[0].schedulingevent_set.select_related('status', 'by')]
-
-    # gather outbound conflicts
-    outbound_dict = OrderedDict()
-    for obc in group.constraint_source_set.filter(meeting=meeting, name__is_group_conflict=True):
-        if obc.name.slug not in outbound_dict:
-            outbound_dict[obc.name.slug] = []
-        outbound_dict[obc.name.slug].append(obc.target.acronym)
-
-    session_conflicts = dict(
-        inbound=inbound_session_conflicts_as_string(group, meeting),
-        outbound=[dict(name=ConstraintName.objects.get(slug=slug), groups=' '.join(groups))
-                  for slug, groups in outbound_dict.items()],
-    )
+    sessions = Session.objects.filter(meeting=meeting, group=group).order_by('id')
+    login = request.user.person
 
-    show_approve_button = False
+    # delete conflicts
+    Constraint.objects.filter(meeting=meeting, source=group).delete()
 
-    # if sessions include a 3rd session waiting approval and the user is a secretariat or AD of the group
-    # display approve button
-    if any(s.current_status == 'apprw' for s in sessions):
-        if has_role(request.user,'Secretariat') or group.parent.role_set.filter(name='ad',person=request.user.person):
-            show_approve_button = True
+    # mark sessions as deleted
+    for session in sessions:
+        SchedulingEvent.objects.create(
+            session=session,
+            status=SessionStatusName.objects.get(slug='deleted'),
+            by=request.user.person,
+        )
+        session_changed(session)
 
-    # build session dictionary (like querydict from new session request form) for use in template
-    session = get_initial_session(sessions)
+        # clear schedule assignments if already scheduled
+        session.timeslotassignments.all().delete()
 
-    return render(request, 'sreq/view.html', {
-        'can_edit': (not is_locked) or has_role(request.user, 'Secretariat'),
-        'can_cancel': (not is_locked) or has_role(request.user, 'Secretariat'),
-        'session': session,  # legacy processed data
-        'sessions': sessions,  # actual session instances
-        'activities': activities,
-        'meeting': meeting,
-        'group': group,
-        'session_conflicts': session_conflicts,
-        'show_approve_button': show_approve_button},
-    )
+    # send notifitcation
+    (to_email, cc_list) = gather_address_lists('session_request_cancelled', group=group, person=login)
+    from_email = (settings.SESSION_REQUEST_FROM_EMAIL)
+    subject = '%s - Cancelling a meeting request for IETF %s' % (group.acronym, meeting.number)
+    send_mail(request, to_email, from_email, subject, 'meeting/session_cancel_notification.txt',
+              {'requester': get_requester_text(login, group),
+               'meeting': meeting}, cc=cc_list)
 
+    messages.success(request, 'The %s Session Request has been cancelled' % group.acronym)
+    return redirect('ietf.meeting.views_session_request.list_view')
diff --git a/ietf/secr/meetings/views.py b/ietf/secr/meetings/views.py
index 47f7b7ffa5..1f6f2f3297 100644
--- a/ietf/secr/meetings/views.py
+++ b/ietf/secr/meetings/views.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2007-2023, All Rights Reserved
+# Copyright The IETF Trust 2007-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 import datetime
@@ -20,12 +20,12 @@
 from ietf.meeting.helpers import make_materials_directories, populate_important_dates
 from ietf.meeting.models import Meeting, Session, Room, TimeSlot, SchedTimeSessAssignment, Schedule, SchedulingEvent
 from ietf.meeting.utils import add_event_info_to_session_qs
+from ietf.meeting.views_session_request import get_initial_session
 from ietf.name.models import SessionStatusName
 from ietf.group.models import Group, GroupEvent
 from ietf.secr.meetings.forms import ( BaseMeetingRoomFormSet, MeetingModelForm, MeetingSelectForm,
     MeetingRoomForm, MiscSessionForm, TimeSlotForm, RegularSessionEditForm,
     MeetingRoomOptionsForm )
-from ietf.secr.sreq.views import get_initial_session
 from ietf.secr.utils.meeting import get_session, get_timeslot
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.utils.timezone import make_aware
diff --git a/ietf/secr/sreq/__init__.py b/ietf/secr/sreq/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/ietf/secr/sreq/forms.py b/ietf/secr/sreq/forms.py
deleted file mode 100644
index 4a0f449b2a..0000000000
--- a/ietf/secr/sreq/forms.py
+++ /dev/null
@@ -1,333 +0,0 @@
-# Copyright The IETF Trust 2013-2022, All Rights Reserved
-# -*- coding: utf-8 -*-
-
-
-from django import forms
-from django.template.defaultfilters import pluralize
-
-import debug                            # pyflakes:ignore
-
-from ietf.name.models import TimerangeName, ConstraintName
-from ietf.group.models import Group
-from ietf.meeting.forms import sessiondetailsformset_factory
-from ietf.meeting.models import ResourceAssociation, Constraint
-from ietf.person.fields import SearchablePersonsField
-from ietf.person.models import Person
-from ietf.utils.fields import ModelMultipleChoiceField
-from ietf.utils.html import clean_text_field
-from ietf.utils import log
-
-# -------------------------------------------------
-# Globals
-# -------------------------------------------------
-
-NUM_SESSION_CHOICES = (('','--Please select'),('1','1'),('2','2'))
-SESSION_TIME_RELATION_CHOICES = (('', 'No preference'),) + Constraint.TIME_RELATION_CHOICES
-JOINT_FOR_SESSION_CHOICES = (('1', 'First session'), ('2', 'Second session'), ('3', 'Third session'), )
-
-# -------------------------------------------------
-# Helper Functions
-# -------------------------------------------------
-def allowed_conflicting_groups():
-    return Group.objects.filter(type__in=['wg', 'ag', 'rg', 'rag', 'program', 'edwg'], state__in=['bof', 'proposed', 'active'])
-
-def check_conflict(groups, source_group):
-    '''
-    Takes a string which is a list of group acronyms.  Checks that they are all active groups
-    '''
-    # convert to python list (allow space or comma separated lists)
-    items = groups.replace(',',' ').split()
-    active_groups = allowed_conflicting_groups()
-    for group in items:
-        if group == source_group.acronym:
-            raise forms.ValidationError("Cannot declare a conflict with the same group: %s" % group)
-
-        if not active_groups.filter(acronym=group):
-            raise forms.ValidationError("Invalid or inactive group acronym: %s" % group)
-
-# -------------------------------------------------
-# Forms
-# -------------------------------------------------
-
-class GroupSelectForm(forms.Form):
-    group = forms.ChoiceField()
-
-    def __init__(self,*args,**kwargs):
-        choices = kwargs.pop('choices')
-        super(GroupSelectForm, self).__init__(*args,**kwargs)
-        self.fields['group'].widget.choices = choices
-
-
-class NameModelMultipleChoiceField(ModelMultipleChoiceField):
-    def label_from_instance(self, name):
-        return name.desc
-
-
-class SessionForm(forms.Form):
-    num_session = forms.ChoiceField(choices=NUM_SESSION_CHOICES)
-    # session fields are added in __init__()
-    session_time_relation = forms.ChoiceField(choices=SESSION_TIME_RELATION_CHOICES, required=False)
-    attendees = forms.IntegerField()
-    # FIXME: it would cleaner to have these be
-    # ModelMultipleChoiceField, and just customize the widgetry, that
-    # way validation comes for free (applies to this CharField and the
-    # constraints dynamically instantiated in __init__())
-    joint_with_groups = forms.CharField(max_length=255,required=False)
-    joint_with_groups_selector = forms.ChoiceField(choices=[], required=False)  # group select widget for prev field
-    joint_for_session = forms.ChoiceField(choices=JOINT_FOR_SESSION_CHOICES, required=False)
-    comments = forms.CharField(max_length=200,required=False)
-    third_session = forms.BooleanField(required=False)
-    resources     = forms.MultipleChoiceField(widget=forms.CheckboxSelectMultiple,required=False)
-    bethere       = SearchablePersonsField(label="Must be present", required=False)
-    timeranges    = NameModelMultipleChoiceField(widget=forms.CheckboxSelectMultiple, required=False,
-                                                 queryset=TimerangeName.objects.all())
-    adjacent_with_wg = forms.ChoiceField(required=False)
-    send_notifications = forms.BooleanField(label="Send notification emails?", required=False, initial=False)
-
-    def __init__(self, group, meeting, data=None, *args, **kwargs):
-        self.hidden = kwargs.pop('hidden', False)
-        self.notifications_optional = kwargs.pop('notifications_optional', False)
-
-        self.group = group
-        formset_class = sessiondetailsformset_factory(max_num=3 if group.features.acts_like_wg else 50)
-        self.session_forms = formset_class(group=self.group, meeting=meeting, data=data)
-        super(SessionForm, self).__init__(data=data, *args, **kwargs)
-        if not self.notifications_optional:
-            self.fields['send_notifications'].widget = forms.HiddenInput()
-
-        # Allow additional sessions for non-wg-like groups
-        if not self.group.features.acts_like_wg:
-            self.fields['num_session'].choices = ((n, str(n)) for n in range(1, 51))
-
-        self.fields['comments'].widget = forms.Textarea(attrs={'rows':'3','cols':'65'})
-
-        other_groups = list(allowed_conflicting_groups().exclude(pk=group.pk).values_list('acronym', 'acronym').order_by('acronym'))
-        self.fields['adjacent_with_wg'].choices = [('', '--No preference')] + other_groups
-        group_acronym_choices = [('','--Select WG(s)')] + other_groups
-        self.fields['joint_with_groups_selector'].choices = group_acronym_choices
-
-        # Set up constraints for the meeting
-        self._wg_field_data = []
-        for constraintname in meeting.group_conflict_types.all():
-            # two fields for each constraint: a CharField for the group list and a selector to add entries
-            constraint_field = forms.CharField(max_length=255, required=False)
-            constraint_field.widget.attrs['data-slug'] = constraintname.slug
-            constraint_field.widget.attrs['data-constraint-name'] = str(constraintname).title()
-            self._add_widget_class(constraint_field.widget, 'wg_constraint')
-
-            selector_field = forms.ChoiceField(choices=group_acronym_choices, required=False)
-            selector_field.widget.attrs['data-slug'] = constraintname.slug  # used by onchange handler
-            self._add_widget_class(selector_field.widget, 'wg_constraint_selector')
-
-            cfield_id = 'constraint_{}'.format(constraintname.slug)
-            cselector_id = 'wg_selector_{}'.format(constraintname.slug)
-            # keep an eye out for field name conflicts
-            log.assertion('cfield_id not in self.fields')
-            log.assertion('cselector_id not in self.fields')
-            self.fields[cfield_id] = constraint_field
-            self.fields[cselector_id] = selector_field
-            self._wg_field_data.append((constraintname, cfield_id, cselector_id))
-
-        # Show constraints that are not actually used by the meeting so these don't get lost
-        self._inactive_wg_field_data = []
-        inactive_cnames = ConstraintName.objects.filter(
-            is_group_conflict=True  # Only collect group conflicts...
-        ).exclude(
-            meeting=meeting  # ...that are not enabled for this meeting...
-        ).filter(
-            constraint__source=group,  # ...but exist for this group...
-            constraint__meeting=meeting,  # ... at this meeting.
-        ).distinct()
-
-        for inactive_constraint_name in inactive_cnames:
-            field_id = 'delete_{}'.format(inactive_constraint_name.slug)
-            self.fields[field_id] = forms.BooleanField(required=False, label='Delete this conflict', help_text='Delete this inactive conflict?')
-            constraints = group.constraint_source_set.filter(meeting=meeting, name=inactive_constraint_name)
-            self._inactive_wg_field_data.append(
-                (inactive_constraint_name,
-                 ' '.join([c.target.acronym for c in constraints]),
-                 field_id)
-            )
-
-        self.fields['joint_with_groups_selector'].widget.attrs['onchange'] = "document.form_post.joint_with_groups.value=document.form_post.joint_with_groups.value + ' ' + this.options[this.selectedIndex].value; return 1;"
-        self.fields["resources"].choices = [(x.pk,x.desc) for x in ResourceAssociation.objects.filter(name__used=True).order_by('name__order') ]
-
-        if self.hidden:
-            # replace all the widgets to start...
-            for key in list(self.fields.keys()):
-                self.fields[key].widget = forms.HiddenInput()
-            # re-replace a couple special cases
-            self.fields['resources'].widget = forms.MultipleHiddenInput()
-            self.fields['timeranges'].widget = forms.MultipleHiddenInput()
-            # and entirely replace bethere - no need to support searching if input is hidden
-            self.fields['bethere'] = ModelMultipleChoiceField(
-                widget=forms.MultipleHiddenInput, required=False,
-                queryset=Person.objects.all(),
-            )
-
-    def wg_constraint_fields(self):
-        """Iterates over wg constraint fields
-
-        Intended for use in the template.
-        """
-        for cname, cfield_id, cselector_id in self._wg_field_data:
-            yield cname, self[cfield_id], self[cselector_id]
-
-    def wg_constraint_count(self):
-        """How many wg constraints are there?"""
-        return len(self._wg_field_data)
-
-    def wg_constraint_field_ids(self):
-        """Iterates over wg constraint field IDs"""
-        for cname, cfield_id, _ in self._wg_field_data:
-            yield cname, cfield_id
-
-    def inactive_wg_constraints(self):
-        for cname, value, field_id in self._inactive_wg_field_data:
-            yield cname, value, self[field_id]
-
-    def inactive_wg_constraint_count(self):
-        return len(self._inactive_wg_field_data)
-
-    def inactive_wg_constraint_field_ids(self):
-        """Iterates over wg constraint field IDs"""
-        for cname, _, field_id in self._inactive_wg_field_data:
-            yield cname, field_id
-
-    @staticmethod
-    def _add_widget_class(widget, new_class):
-        """Add a new class, taking care in case some already exist"""
-        existing_classes = widget.attrs.get('class', '').split()
-        widget.attrs['class'] = ' '.join(existing_classes + [new_class])
-
-    def _join_conflicts(self, cleaned_data, slugs):
-        """Concatenate constraint fields from cleaned data into a single list"""
-        conflicts = []
-        for cname, cfield_id, _ in self._wg_field_data:
-            if cname.slug in slugs and cfield_id in cleaned_data:
-                groups = cleaned_data[cfield_id]
-                # convert to python list (allow space or comma separated lists)
-                items = groups.replace(',',' ').split()
-                conflicts.extend(items)
-        return conflicts
-
-    def _validate_duplicate_conflicts(self, cleaned_data):
-        """Validate that no WGs appear in more than one constraint that does not allow duplicates
-
-        Raises ValidationError
-        """
-        # Only the older constraints (conflict, conflic2, conflic3) need to be mutually exclusive.
-        all_conflicts = self._join_conflicts(cleaned_data, ['conflict', 'conflic2', 'conflic3'])
-        seen = []
-        duplicated = []
-        errors = []
-        for c in all_conflicts:
-            if c not in seen:
-                seen.append(c)
-            elif c not in duplicated:  # only report once
-                duplicated.append(c)
-                errors.append(forms.ValidationError('%s appears in conflicts more than once' % c))
-        return errors
-
-    def clean_joint_with_groups(self):
-        groups = self.cleaned_data['joint_with_groups']
-        check_conflict(groups, self.group)
-        return groups
-
-    def clean_comments(self):
-        return clean_text_field(self.cleaned_data['comments'])
-
-    def clean_bethere(self):
-        bethere = self.cleaned_data["bethere"]
-        if bethere:
-            extra = set(
-                Person.objects.filter(
-                    role__group=self.group, role__name__in=["chair", "ad"]
-                )
-                & bethere
-            )
-            if extra:
-                extras = ", ".join(e.name for e in extra)
-                raise forms.ValidationError(
-                    (
-                        f"Please remove the following person{pluralize(len(extra))}, the system "
-                        f"tracks their availability due to their role{pluralize(len(extra))}: {extras}."
-                    )
-                )
-        return bethere
-
-    def clean_send_notifications(self):
-        return True if not self.notifications_optional else self.cleaned_data['send_notifications']
-
-    def is_valid(self):
-        return super().is_valid() and self.session_forms.is_valid()
-
-    def clean(self):
-        super(SessionForm, self).clean()
-        self.session_forms.clean()
-
-        data = self.cleaned_data
-
-        # Validate the individual conflict fields
-        for _, cfield_id, _ in self._wg_field_data:
-            try:
-                check_conflict(data[cfield_id], self.group)
-            except forms.ValidationError as e:
-                self.add_error(cfield_id, e)
-
-        # Skip remaining tests if individual field tests had errors,
-        if self.errors:
-            return data
-
-        # error if conflicts contain disallowed dupes
-        for error in self._validate_duplicate_conflicts(data):
-            self.add_error(None, error)
-
-        # Verify expected number of session entries are present
-        num_sessions_with_data = len(self.session_forms.forms_to_keep)
-        num_sessions_expected = -1
-        try:
-            num_sessions_expected = int(data.get('num_session', ''))
-        except ValueError:
-            self.add_error('num_session', 'Invalid value for number of sessions')
-        if num_sessions_with_data < num_sessions_expected:
-            self.add_error('num_session', 'Must provide data for all sessions')
-
-        # if default (empty) option is selected, cleaned_data won't include num_session key
-        if num_sessions_expected != 2 and num_sessions_expected is not None:
-            if data.get('session_time_relation'):
-                self.add_error(
-                    'session_time_relation',
-                    forms.ValidationError('Time between sessions can only be used when two sessions are requested.')
-                )
-
-        joint_session = data.get('joint_for_session', '')
-        if joint_session != '':
-            joint_session = int(joint_session)
-            if joint_session > num_sessions_with_data:
-                self.add_error(
-                    'joint_for_session',
-                    forms.ValidationError(
-                        f'Session {joint_session} can not be the joint session, the session has not been requested.'
-                    )
-                )
-
-        return data
-
-    @property
-    def media(self):
-        # get media for our formset
-        return super().media + self.session_forms.media + forms.Media(js=('secr/js/session_form.js',))
-
-
-# Used for totally virtual meetings during COVID-19 to omit the expected
-# number of attendees since there were no room size limitations
-#
-# class VirtualSessionForm(SessionForm):
-#     '''A SessionForm customized for special virtual meeting requirements'''
-#     attendees = forms.IntegerField(required=False)
-
-
-class ToolStatusForm(forms.Form):
-    message = forms.CharField(widget=forms.Textarea(attrs={'rows':'3','cols':'80'}), strip=False)
-
diff --git a/ietf/secr/sreq/templatetags/__init__.py b/ietf/secr/sreq/templatetags/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/ietf/secr/sreq/urls.py b/ietf/secr/sreq/urls.py
deleted file mode 100644
index 7e0db8117a..0000000000
--- a/ietf/secr/sreq/urls.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Copyright The IETF Trust 2007-2019, All Rights Reserved
-
-from django.conf import settings
-
-from ietf.secr.sreq import views
-from ietf.utils.urls import url
-
-urlpatterns = [
-    url(r'^$', views.main),
-    url(r'^status/$', views.tool_status),
-    url(r'^%(acronym)s/$' % settings.URL_REGEXPS, views.view),
-    url(r'^(?P<num>[A-Za-z0-9_\-\+]+)/%(acronym)s/view/$' % settings.URL_REGEXPS, views.view),
-    url(r'^%(acronym)s/approve/$' % settings.URL_REGEXPS, views.approve),
-    url(r'^%(acronym)s/cancel/$' % settings.URL_REGEXPS, views.cancel),
-    url(r'^%(acronym)s/confirm/$' % settings.URL_REGEXPS, views.confirm),
-    url(r'^%(acronym)s/edit/$' % settings.URL_REGEXPS, views.edit),
-    url(r'^%(acronym)s/new/$' % settings.URL_REGEXPS, views.new),
-    url(r'^%(acronym)s/no_session/$' % settings.URL_REGEXPS, views.no_session),
-    url(r'^(?P<num>[A-Za-z0-9_\-\+]+)/%(acronym)s/edit/$' % settings.URL_REGEXPS, views.edit),
-]
diff --git a/ietf/secr/telechat/tests.py b/ietf/secr/telechat/tests.py
index 39949b83a2..fa26d33a5c 100644
--- a/ietf/secr/telechat/tests.py
+++ b/ietf/secr/telechat/tests.py
@@ -13,6 +13,7 @@
     IndividualDraftFactory, ConflictReviewFactory)
 from ietf.doc.models import BallotDocEvent, BallotType, BallotPositionDocEvent, State, Document
 from ietf.doc.utils import update_telechat, create_ballot_if_not_open
+from ietf.meeting.factories import MeetingFactory
 from ietf.utils.test_utils import TestCase
 from ietf.utils.timezone import date_today, datetime_today
 from ietf.iesg.models import TelechatDate
@@ -25,6 +26,26 @@
 def augment_data():
     TelechatDate.objects.create(date=date_today())
 
+class SecrUrlTests(TestCase):
+    def test_urls(self):
+        MeetingFactory(type_id='ietf', date=date_today())
+
+        # check public options
+        response = self.client.get("/secr/")
+        self.assertEqual(response.status_code, 200)
+        q = PyQuery(response.content)
+        links = q('div.secr-menu a')
+        self.assertEqual(len(links), 1)
+        self.assertEqual(PyQuery(links[0]).text(), 'Announcements')
+
+        # check secretariat only options
+        self.client.login(username="secretary", password="secretary+password")
+        response = self.client.get("/secr/")
+        self.assertEqual(response.status_code, 200)
+        q = PyQuery(response.content)
+        links = q('div.secr-menu a')
+        self.assertEqual(len(links), 4)
+
 class SecrTelechatTestCase(TestCase):
     def test_main(self):
         "Main Test"
diff --git a/ietf/secr/templates/includes/activities.html b/ietf/secr/templates/includes/activities.html
deleted file mode 100644
index 1304b7c48d..0000000000
--- a/ietf/secr/templates/includes/activities.html
+++ /dev/null
@@ -1,23 +0,0 @@
-<h2>Activities Log</h2>
-<div class="inline-related last-related">
-    <table class="full-width">
-        <thead>
-            <tr>
-                <th scope="col">Date</th>
-                <th scope="col">Time</th>
-                <th scope="col">Action</th>
-                <th scope="col">Name</th>
-            </tr>
-        </thead>
-        <tbody>
-            {% for entry in activities %}
-                <tr class="{% cycle 'row1' 'row2' %}">
-                    <td>{{ entry.act_date }}</td>
-                    <td>{{ entry.act_time }}</td>
-                    <td>{{ entry.activity }}</td>
-                    <td>{{ entry.act_by }}</td>
-                </tr>
-            {% endfor %}
-        </tbody>
-    </table>
-</div> <!-- inline-related -->
diff --git a/ietf/secr/templates/includes/buttons_next_cancel.html b/ietf/secr/templates/includes/buttons_next_cancel.html
deleted file mode 100644
index 95d25f55bc..0000000000
--- a/ietf/secr/templates/includes/buttons_next_cancel.html
+++ /dev/null
@@ -1,6 +0,0 @@
-<div class="button-group">
-    <ul>
-        <li><button type="submit" name="submit" value="Next">Next</button></li>
-        <li><button type="submit" name="submit" value="Cancel">Cancel</button></li>
-    </ul>
-</div> <!-- button-group -->
diff --git a/ietf/secr/templates/includes/buttons_submit_cancel.html b/ietf/secr/templates/includes/buttons_submit_cancel.html
deleted file mode 100644
index df40c98255..0000000000
--- a/ietf/secr/templates/includes/buttons_submit_cancel.html
+++ /dev/null
@@ -1,6 +0,0 @@
-<div class="button-group">
-    <ul>
-        <li><button type="submit" name="submit" value="Submit">Submit</button></li>
-        <li><button type="submit" name="submit" value="Cancel">Cancel</button></li>
-    </ul>
-</div> <!-- button-group -->
diff --git a/ietf/secr/templates/includes/sessions_footer.html b/ietf/secr/templates/includes/sessions_footer.html
deleted file mode 100755
index 2a26440047..0000000000
--- a/ietf/secr/templates/includes/sessions_footer.html
+++ /dev/null
@@ -1,5 +0,0 @@
-<ul>
-<li><a href="https://www.ietf.org/chairs/session-request-tool-instructions/" target="_blank">Instructions</a>.</li>
-<li>If you require assistance in using this tool, or wish to report a bug, then please send a message to <a href="mailto:ietf-action@ietf.org">ietf-action@ietf.org</a>.</li>
-<li>To submit your request via email, please send your request to <a href="mailto:agenda@ietf.org">agenda@ietf.org</a>.</li>
-</ul>
\ No newline at end of file
diff --git a/ietf/secr/templates/includes/sessions_request_form.html b/ietf/secr/templates/includes/sessions_request_form.html
deleted file mode 100755
index 61b1673357..0000000000
--- a/ietf/secr/templates/includes/sessions_request_form.html
+++ /dev/null
@@ -1,130 +0,0 @@
-<span class="required">*</span> Required Field
-<form id="session-request-form" action="." method="post" name="form_post">{% csrf_token %}
-    {{ form.session_forms.management_form }}
-    {% if form.non_field_errors %}
-        {{ form.non_field_errors }}
-    {% endif %}
-    <table id="sessions-new-table">
-        <tbody>
-        <tr class="bg1"><td>Working Group Name:</td><td>{{ group.name }} ({{ group.acronym }})</td></tr>
-        <tr class="bg2"><td>Area Name:</td><td>{% if group.parent %}{{ group.parent.name }} ({{ group.parent.acronym }}){% endif %}</td></tr>
-        <tr class="bg1"><td>Number of Sessions:<span class="required">*</span></td><td>{{ form.num_session.errors }}{{ form.num_session }}</td></tr>
-        {% if group.features.acts_like_wg %}<tr class="bg2" id="session_row_0"><td>Session 1:<span class="required">*</span></td><td>{% include 'meeting/session_details_form.html' with form=form.session_forms.0 hide_onsite_tool_prompt=True only %}</td></tr>
-            <tr class="bg2" id="session_row_1"><td>Session 2:<span class="required">*</span></td><td>{% include 'meeting/session_details_form.html' with form=form.session_forms.1 hide_onsite_tool_prompt=True only %}</td></tr>
-            {% if not is_virtual %}
-                <tr class="bg2"><td>Time between two sessions:</td><td>{{ form.session_time_relation.errors }}{{ form.session_time_relation }}</td></tr>
-            {% endif %}
-            <tr class="bg2" id="third_session_row"><td>Additional Session Request:</td><td>{{ form.third_session }} Check this box to request an additional session.<br>
-                Additional slot may be available after agenda scheduling has closed and with the approval of an Area Director.<br>
-                <div id="session_row_2">
-                    Third Session:
-                    {% include 'meeting/session_details_form.html' with form=form.session_forms.2 hide_onsite_tool_prompt=True only %}
-                </div>
-            </td></tr>
-        {% else %}{# else not group.features.acts_like_wg #}
-            {% for session_form in form.session_forms %}
-                <tr class="bg2" id="session_row_{{ forloop.counter0 }}"><td>Session {{ forloop.counter }}:<span class="required">*</span></td><td>{% include 'meeting/session_details_form.html' with form=session_form only %}</td></tr>
-            {% endfor %}
-        {% endif %}
-        <tr class="bg1"><td>Number of Attendees:{% if not is_virtual %}<span class="required">*</span>{% endif %}</td><td>{{ form.attendees.errors }}{{ form.attendees }}</td></tr>
-        <tr class="bg2">
-            <td>Participants who must be present:</td>
-            <td>
-                {{ form.bethere.errors }}
-                {{ form.bethere }}
-                <p class="fw-bold text-danger">
-                    Do not include Area Directors and WG Chairs; the system already tracks their availability.
-                </p>
-            </td>
-        </tr>
-        <tr class="bg1"><td>Conflicts to Avoid:</td>
-            <td>
-                <table>
-                    <tbody>
-                    <tr>
-                        <td colspan="2">Other WGs that included {{ group.name }} in their conflict lists:</td>
-                        <td>{{ session_conflicts.inbound|default:"None" }}</td>
-                    </tr>
-                    {% for cname, cfield, cselector  in form.wg_constraint_fields %}
-                        <tr class="bg1">
-                            {% if forloop.first %}<td rowspan="{{ form.wg_constraint_count }}" >WG Sessions:<br>You may select multiple WGs within each category</td>{% endif %}
-                            <td>{{ cname|title }}</td>
-                            <td>{{ cselector }}
-                                <button type="button" id="wg_delete_{{ cname.slug }}" onclick="ietf_sessions.delete_wg_constraint_clicked('{{ cname.slug }}')">Delete the last entry</button><br>
-                                {{ cfield.errors }}{{ cfield }}
-                            </td>
-                        </tr>
-                    {% empty %}{# shown if there are no constraint fields #}
-                        <tr class="bg1"><td ></td><td colspan="2">No constraints are enabled for this meeting.</td></tr>
-                    {% endfor %}
-                    {% if form.inactive_wg_constraints %}
-                        {% for cname, value, field in form.inactive_wg_constraints %}
-                            <tr class="bg1">
-                                {% if forloop.first %}
-                                    <td rowspan="{{ form.inactive_wg_constraint_count }}" >
-                                        Disabled for this meeting
-                                    </td>
-                                {% endif %}
-                                <td>{{ cname|title }}</td>
-                                <td><input type="text" value="{{ value }}" maxlength="255" class="wg_constraint" disabled><br>{{ field }} {{ field.label }}</td>
-                        </tr>
-                        {% endfor %}
-                    {% endif %}
-                    <tr>
-                        <td colspan="2">BOF Sessions:</td>
-                        <td>If the sessions can not be found in the fields above, please enter free form requests in the Special Requests field below.</td>
-                    </tr>
-                    </tbody>
-                </table>
-            </td>
-        </tr>
-
-        {% if not is_virtual %}
-
-            <tr class="bg2"><td>Resources requested:</td>
-                <td>
-                    {{ form.resources.errors }} {{ form.resources }}
-                </td>
-            </tr>
-            <tr class="bg1">
-                <td>Times during which this WG can <strong>not</strong> meet:<br>Please explain any selections in Special Requests below.</td>
-                 <!-- [html-validate-disable-block no-redundant-for -- FIXME: form renders with redundant for attributes] -->
-                <td>{{ form.timeranges.errors }}{{ form.timeranges }}</td>
-            </tr>
-            <tr class="bg2">
-                <td>
-                    Plan session adjacent with another WG:<br>
-                    (Immediately before or after another WG, no break in between, in the same room.)
-                </td>
-                <td>{{ form.adjacent_with_wg.errors }}{{ form.adjacent_with_wg }}</td>
-            </tr>
-            <tr class="bg1">
-                <td>
-                    Joint session with:<br>
-                    (To request one session for multiple WGs together.)
-                </td>
-                <td>To request a joint session with another group, please contact the secretariat.</td>
-            </tr>
-
-        {% endif %}
-
-        <tr class="bg2">
-            <td>Special Requests:<br>&nbsp;<br>i.e. restrictions on meeting times / days, etc.<br>  (limit 200 characters)</td>
-            <td>{{ form.comments.errors }}{{ form.comments }}</td>
-        </tr>
-        {% if form.notifications_optional %}
-            <tr class="bg1">
-                <td>{{ form.send_notifications.label }}</td>
-                <td>{{ form.send_notifications.errors }}{{ form.send_notifications }}</td>
-            </tr>
-        {% endif %}
-        </tbody>
-    </table>
-
-    <div class="button-group">
-        <ul>
-            <li><button type="submit" name="submit" value="Save"{% if is_locked %} disabled{% endif %}>Save</button></li>
-            <li><button type="button" onclick="history.back()">Back</button></li>
-        </ul>
-    </div>
-</form>
\ No newline at end of file
diff --git a/ietf/secr/templates/includes/sessions_request_view.html b/ietf/secr/templates/includes/sessions_request_view.html
deleted file mode 100644
index bc6aef0611..0000000000
--- a/ietf/secr/templates/includes/sessions_request_view.html
+++ /dev/null
@@ -1,73 +0,0 @@
-{% load ams_filters %}
-<table class="full-width">
-    <tbody>
-    <tr class="row1"><td>Working Group Name:</td><td>{{ group.name }} ({{ group.acronym }})</td></tr>
-    <tr class="row2"><td>Area Name:</td><td>{{ group.parent }}</td></tr>
-    <tr class="row1"><td>Number of Sessions Requested:</td><td>{% if session.third_session %}3{% else %}{{ session.num_session }}{% endif %}</td></tr>
-    {% if form %}
-        {% include 'includes/sessions_request_view_formset.html' with formset=form.session_forms group=group session=session only %}
-    {% else %}
-        {% include 'includes/sessions_request_view_session_set.html' with session_set=sessions group=group session=session only %}
-    {% endif %}
-    <tr class="row1"><td>Number of Attendees:</td><td>{{ session.attendees }}</td></tr>
-    <tr class="row2">
-        <td>Conflicts to Avoid:</td>
-        <td>
-            {% if session_conflicts.outbound %}
-                <table>
-                    <tbody>
-                    {% for conflict in session_conflicts.outbound %}
-                        <tr><td>{{ conflict.name|title }}: </td><td>{{ conflict.groups }}</td></tr>
-                    {% endfor %}
-                    </tbody>
-                </table>
-            {% else %}<i>None</i>{% endif %}
-        </td>
-    </tr>
-    <tr class="row1">
-        <td>Other WGs that included {{ group }} in their conflict list:</td>
-        <td>{% if session_conflicts.inbound %}{{ session_conflicts.inbound }}{% else %}<i>None so far</i>{% endif %}</td>
-    </tr>
-    {% if not is_virtual %}
-        <tr class="row2">
-            <td>Resources requested:</td>
-            <td>{% if session.resources %}<ul>{% for resource in session.resources %}<li>{{ resource.desc }}</li>{% endfor %}</ul>{% else %}<i>None so far</i>{% endif %}</td>
-        </tr>
-    {% endif %}
-    <tr class="row1">
-        <td>Participants who must be present:</td>
-        <td>{% if session.bethere %}<ul>{% for person in session.bethere %}<li>{{ person }}</li>{% endfor %}</ul>{% else %}<i>None</i>{% endif %}</td>
-    </tr>
-    <tr class="row2">
-        <td>Can not meet on:</td>
-        <td>{% if session.timeranges_display %}{{ session.timeranges_display|join:', ' }}{% else %}No constraints{% endif %}</td>
-    </tr>
-    {% if not is_virtual %}
-        <tr class="row1">
-            <td>Adjacent with WG:</td>
-            <td>{{ session.adjacent_with_wg|default:'No preference' }}</td>
-        </tr>
-        <tr class="row2">
-            <td>Joint session:</td>
-            <td>
-                {% if session.joint_with_groups %}
-                    {{ session.joint_for_session_display }} with: {{ session.joint_with_groups }}
-                {% else %}
-                    Not a joint session
-                {% endif %}
-            </td>
-        </tr>
-    {% endif %}
-    <tr class="row1"><td>Special Requests:</td><td>{{ session.comments }}</td></tr>
-    {% if form and form.notifications_optional %}
-        <tr class="row2">
-            <td>
-                {{ form.send_notifications.label}}
-            </td>
-            <td>
-                {% if form.cleaned_data.send_notifications %}Yes{% else %}No{% endif %}
-            </td>
-        </tr>
-    {% endif %}
-    </tbody>
-</table>
\ No newline at end of file
diff --git a/ietf/secr/templates/includes/sessions_request_view_formset.html b/ietf/secr/templates/includes/sessions_request_view_formset.html
deleted file mode 100644
index 80cad8d829..0000000000
--- a/ietf/secr/templates/includes/sessions_request_view_formset.html
+++ /dev/null
@@ -1,32 +0,0 @@
-{% load ams_filters %}{# keep this in sync with sessions_request_view_session_set.html #}
-{% for sess_form in formset %}{% if sess_form.cleaned_data and not sess_form.cleaned_data.DELETE %}
-  <tr class="row2">
-    <td>Session {{ forloop.counter }}:</td>
-    <td>
-      <dl>
-        <dt>Length</dt>
-        <dd>{{ sess_form.cleaned_data.requested_duration.total_seconds|display_duration }}</dd>
-        {% if sess_form.cleaned_data.name %}
-          <dt>Name</dt>
-          <dd>{{ sess_form.cleaned_data.name }}</dd>{% endif %}
-        {% if sess_form.cleaned_data.purpose.slug != 'regular' %}
-          <dt>Purpose</dt>
-          <dd>
-            {{ sess_form.cleaned_data.purpose }}
-            {% if sess_form.cleaned_data.purpose.timeslot_types|length > 1 %}({{ sess_form.cleaned_data.type }}
-              ){% endif %}
-          </dd>
-          <dt>Onsite tool?</dt>
-          <dd>{{ sess_form.cleaned_data.has_onsite_tool|yesno }}</dd>
-        {% endif %}
-      </dl>
-    </td>
-  </tr>
-  {% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %}
-    <tr class="row2">
-      <td>Time between sessions:</td>
-      <td>{% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No
-        preference{% endif %}</td>
-    </tr>
-  {% endif %}
-{% endif %}{% endfor %}
\ No newline at end of file
diff --git a/ietf/secr/templates/includes/sessions_request_view_session_set.html b/ietf/secr/templates/includes/sessions_request_view_session_set.html
deleted file mode 100644
index a434b9d22b..0000000000
--- a/ietf/secr/templates/includes/sessions_request_view_session_set.html
+++ /dev/null
@@ -1,32 +0,0 @@
-{% load ams_filters %}{# keep this in sync with sessions_request_view_formset.html #}
-{% for sess in session_set %}
-  <tr class="row2">
-    <td>Session {{ forloop.counter }}:</td>
-    <td>
-      <dl>
-        <dt>Length</dt>
-        <dd>{{ sess.requested_duration.total_seconds|display_duration }}</dd>
-        {% if sess.name %}
-          <dt>Name</dt>
-          <dd>{{ sess.name }}</dd>{% endif %}
-        {% if sess.purpose.slug != 'regular' %}
-          <dt>Purpose</dt>
-          <dd>
-            {{ sess.purpose }}
-            {% if sess.purpose.timeslot_types|length > 1 %}({{ sess.type }}
-              ){% endif %}
-          </dd>
-          <dt>Onsite tool?</dt>
-          <dd>{{ sess.has_onsite_tool|yesno }}</dd>
-        {% endif %}
-      </dl>
-    </td>
-  </tr>
-  {% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %}
-    <tr class="row2">
-      <td>Time between sessions:</td>
-      <td>{% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No
-        preference{% endif %}</td>
-    </tr>
-  {% endif %}
-{% endfor %}
\ No newline at end of file
diff --git a/ietf/secr/templates/index.html b/ietf/secr/templates/index.html
index 05fa3db41f..9ea7021279 100644
--- a/ietf/secr/templates/index.html
+++ b/ietf/secr/templates/index.html
@@ -1,11 +1,11 @@
-{# Copyright The IETF Trust 2007, All Rights Reserved #}
+{# Copyright The IETF Trust 2007-2025, All Rights Reserved #}
 {% extends "base.html" %}
 {% load static %}
 {% load ietf_filters %}
 {% block title %}Secretariat Dashboard{% endblock %}
 {% block content %}
     <h1>Secretariat Dashboard</h1>
-    <div class="">
+    <div class="secr-menu">
         {% if user|has_role:"Secretariat" %}
             <h2>IESG</h2>
             <ul>
@@ -20,12 +20,10 @@ <h2>IDs and WGs Process</h2>
 
             <h2>Meetings and Proceedings</h2>
             <ul>
-                <li> <a href="{% url 'ietf.secr.sreq.views.main' %} "><b>Session Requests</b></a></li>
                 <li> <a href="{% url 'ietf.secr.meetings.views.main' %}"><b>Meeting Manager</b></a></li>
             </ul>
         {% else %}
             <ul>
-                <li> <a href="{% url 'ietf.secr.sreq.views.main' %}"><b>Session Requests</b></a></li>
                 <li> <a href="{% url 'ietf.secr.announcement.views.main' %}"><b>Announcements</b></a></li>
             </ul>
         {% endif %}
diff --git a/ietf/secr/templates/sreq/confirm.html b/ietf/secr/templates/sreq/confirm.html
deleted file mode 100755
index 025375af32..0000000000
--- a/ietf/secr/templates/sreq/confirm.html
+++ /dev/null
@@ -1,57 +0,0 @@
-{% extends "base_site.html" %}
-{% load static %}
-
-{% block title %}Sessions - Confirm{% endblock %}
-
-{% block extrastyle %}
-    <style>
-        dl {width: 100%;}
-        dt {float: left; width: 30%; margin: 0.1em 0 0.1em 0; }
-        dt::after {content: ":";}
-        dd {float: left; width: 70%; margin: 0.1em 0 0.1em 0;}
-    </style>
-{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-    <script src="{% static 'secr/js/utils.js' %}"></script>
-    {{ form.media }}
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }}
-    &raquo; <a href="{% url "ietf.secr.sreq.views.main" %}">Sessions</a>
-    &raquo; <a href="../">New</a>
-    &raquo; Session Request Confirmation
-{% endblock %}
-
-{% block content %}
-
-    <div class="module interim-container">
-        <h2>Sessions - Confirm</h2>
-
-        {% include "includes/sessions_request_view.html" %}
-
-        {% if group.features.acts_like_wg and form.session_forms.forms_to_keep|length > 2 %}
-            <br>
-            <p class="alert">
-                <b>
-                    Note: Your request for a third session must be approved by an area director before
-                    being submitted to agenda@ietf.org.  Click "Submit" below to email an approval
-                    request to the area directors.
-                </b>
-            </p>
-            <br>
-        {% endif %}
-
-        <form action="{% url "ietf.secr.sreq.views.confirm" acronym=group.acronym %}" method="post">
-            {% csrf_token %}
-            {{ form }}
-            {{ form.session_forms.management_form }}
-            {% for sf in form.session_forms %}
-                {% include 'meeting/session_details_form.html' with form=sf hidden=True only %}
-            {% endfor %}
-            {% include "includes/buttons_submit_cancel.html" %}
-        </form>
-
-    </div>
-
-{% endblock %}
\ No newline at end of file
diff --git a/ietf/secr/templates/sreq/edit.html b/ietf/secr/templates/sreq/edit.html
deleted file mode 100755
index f6e62104b0..0000000000
--- a/ietf/secr/templates/sreq/edit.html
+++ /dev/null
@@ -1,39 +0,0 @@
-{% extends "base_site.html" %}
-{% load static %}
-{% block title %}Sessions - Edit{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-    <script src="{% static 'secr/js/utils.js' %}"></script>
-    <script src="{% static 'secr/js/sessions.js' %}"></script>
-    {{ form.media }}
-  <style>
-      .hidden {display: none !important;}
-      div.form-group {display: inline;}
-  </style>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }}
-    &raquo; <a href="../../">Sessions</a>
-    &raquo; <a href="../">{{ group.acronym }}</a>
-    &raquo; Edit
-{% endblock %}
-
-{% block instructions %}
-    <a href="https://www.ietf.org/chairs/session-request-tool-instructions/" target="_blank">Instructions</a>
-{% endblock %}
-
-{% block content %}
-    <div class="module interim-container">
-        <h2>IETF {{ meeting.number }}: Edit Session Request</h2>
-        <div class="inline-related">
-            <br>
-
-            {% include "includes/sessions_request_form.html" %}
-
-        </div> <!-- module -->
-    </div>
-{% endblock %}
-
-{% block footer-extras %}
-    {% include "includes/sessions_footer.html" %}
-{% endblock %}
\ No newline at end of file
diff --git a/ietf/secr/templates/sreq/locked.html b/ietf/secr/templates/sreq/locked.html
deleted file mode 100755
index c27cf578ed..0000000000
--- a/ietf/secr/templates/sreq/locked.html
+++ /dev/null
@@ -1,30 +0,0 @@
-{% extends "base_site.html" %}
-{% load static %}
-
-{% block title %}Sessions{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-    <script src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }}
-    &raquo; Sessions (Locked)
-{% endblock %}
-
-{% block content %}
-    <p>&raquo; <a href="{% url 'ietf.meeting.views.meeting_requests' %}">View list of timeslot requests</a></p>
-    <div class="module interim-container">
-        <h2>Sessions - Status</h2>
-
-        <p>{{ message }}</p>
-
-        <div class="button-group">
-            <ul>
-                <li><button type="button" onclick="window.location='../'">Back</button></li>
-            </ul>
-        </div> <!-- button-group -->
-
-
-    </div> <!-- module -->
-
-{% endblock %}
\ No newline at end of file
diff --git a/ietf/secr/templates/sreq/main.html b/ietf/secr/templates/sreq/main.html
deleted file mode 100755
index a6695cd4f3..0000000000
--- a/ietf/secr/templates/sreq/main.html
+++ /dev/null
@@ -1,65 +0,0 @@
-{% extends "base_site.html" %}
-{% load ietf_filters %}
-{% load static %}
-
-{% block title %}Sessions{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-    <script src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }}
-    &raquo; Sessions
-{% endblock %}
-{% block instructions %}
-    <a href="https://www.ietf.org/chairs/session-request-tool-instructions/" target="_blank">Instructions</a>
-{% endblock %}
-
-{% block content %}
-    <p>&raquo; <a href="{% url 'ietf.meeting.views.meeting_requests' %}">View list of timeslot requests</a></p>
-    <div class="module interim-container">
-        <h2>
-            Sessions Request Tool: IETF {{ meeting.number }}
-            {% if user|has_role:"Secretariat" %}
-                {% if is_locked %}
-                    <span class="locked"><a href="{% url "ietf.secr.sreq.views.tool_status" %}">Tool Status: Locked</a></span>
-                {% else %}
-                    <span class="unlocked"><a href="{% url "ietf.secr.sreq.views.tool_status" %}">Tool Status: Unlocked</a></span>
-                {% endif %}
-            {% endif %}
-        </h2>
-        <div class="inline-related">
-            <h3><b>Request New Session</b></h3>
-            <p>The list below includes those working groups that you currently chair which do not already have a session scheduled. You can click on an acronym to initiate a request for a new session at the upcoming IETF meeting.  Click "Group will not meet" to send a notification that the group does not plan to meet.</p>
-            <ul>
-                {% for group in unscheduled_groups %}
-                    <li>
-                        <a href="{% url "ietf.secr.sreq.views.new" acronym=group.acronym %}">{{ group.acronym }}</a>
-                        {% if group.not_meeting %}
-                            <span class="required"> (Currently, this group does not plan to hold a session at IETF {{ meeting.number }})</span>
-                        {% endif %}
-                    </li>
-                {% empty %}
-                    <li><i>NONE</i></li>
-                {% endfor %}
-            </ul>
-        </div> <!-- inline-related -->
-        <br>
-        <div class="inline-related">
-            <h3>Edit / Cancel Previously Requested Sessions</h3>
-            <p>The list below includes those working groups for which you or your co-chair has requested sessions at the upcoming IETF meeting. You can click on an acronym to initiate changes to a session, or cancel a session.</p>
-            <ul>
-                {% for group in scheduled_groups %}
-                    <li><a href="{% url "ietf.secr.sreq.views.view" acronym=group.acronym %}">{{ group.acronym }} - {{ group.status_message }}</a></li>
-                {% empty %}
-                    <li><i>NONE</i></li>
-                {% endfor %}
-            </ul>
-        </div> <!-- inline-related -->
-    </div> <!-- module -->
-
-{% endblock %}
-
-{% block footer-extras %}
-    {% include "includes/sessions_footer.html" %}
-{% endblock %}
\ No newline at end of file
diff --git a/ietf/secr/templates/sreq/new.html b/ietf/secr/templates/sreq/new.html
deleted file mode 100755
index 3f46e6f897..0000000000
--- a/ietf/secr/templates/sreq/new.html
+++ /dev/null
@@ -1,43 +0,0 @@
-{% extends "base_site.html" %}
-{% load static %}
-
-{% block title %}Sessions- New{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-    <script src="{% static 'secr/js/utils.js' %}"></script>
-    <script src="{% static 'secr/js/sessions.js' %}"></script>
-    {{ form.media }}
-  <style>
-      .hidden {display: none !important;}
-      div.form-group {display: inline;}
-  </style>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }}
-    &raquo; <a href="../../">Sessions</a>
-    &raquo; New Session Request
-{% endblock %}
-
-{% block instructions %}
-    <a href="https://www.ietf.org/chairs/session-request-tool-instructions/" target="_blank">Instructions</a>
-{% endblock %}
-
-{% block content %}
-    <div class="module interim-container">
-        <h2>IETF {{ meeting.number }}: New Session Request</h2>
-        <div class="inline-related">
-            <br>
-            <ul class="session-buttons">
-                <li><button type="button" onclick="if (window.confirm('A message will be sent to agenda@ietf.com as well as to the area director(s).\n\nContinue?')) { window.location='{% url "ietf.secr.sreq.views.no_session" acronym=group.acronym %}'};">Send a notification that the group does not plan to hold a session at IETF {{ meeting.number }}</button></li>
-                <li><button type="button" onclick="window.location='{% url "ietf.secr.sreq.views.new" acronym=group.acronym %}?previous'">Retrieve all information from previous meeting</button></li>
-            </ul>
-        </div>
-            {% include "includes/sessions_request_form.html" %}
-
-        </div> <!-- module -->
-
-{% endblock %}
-
-{% block footer-extras %}
-    {% include "includes/sessions_footer.html" %}
-{% endblock %}
\ No newline at end of file
diff --git a/ietf/secr/templates/sreq/tool_status.html b/ietf/secr/templates/sreq/tool_status.html
deleted file mode 100755
index b91e73a129..0000000000
--- a/ietf/secr/templates/sreq/tool_status.html
+++ /dev/null
@@ -1,42 +0,0 @@
-{% extends "base_site.html" %}
-{% load static %}
-
-{% block title %}Sessions{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-    <script src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }}
-    &raquo; <a href="../">Sessions</a>
-    &raquo; Session Status
-{% endblock %}
-
-{% block content %}
-
-    <div class="module interim-container">
-        <h2>Sessions - Status</h2>
-        <p>Enter the message that you would like displayed to the WG Chair when this tool is locked.</p>
-        <form method="post">{% csrf_token %}
-            <table>
-                <tbody>
-                 <!-- [html-validate-disable-block wcag/h63 -- FIXME: as_table renders without scope] -->
-                 {{ form.as_table }}
-                 </tbody>
-            </table>
-            <div class="button-group">
-                <ul>
-                    {% if is_locked %}
-                        <li><button type="submit" name="submit" value="Unlock">Unlock</button></li>
-                    {% else %}
-                        <li><button type="submit" name="submit" value="Lock">Lock</button></li>
-                    {% endif %}
-                    <li><button type="submit" name="submit" value="Back">Back</button></li>
-                </ul>
-            </div> <!-- button-group -->
-
-        </form>
-
-    </div> <!-- module -->
-
-{% endblock %}
diff --git a/ietf/secr/templates/sreq/view.html b/ietf/secr/templates/sreq/view.html
deleted file mode 100644
index 9a0a3b01c1..0000000000
--- a/ietf/secr/templates/sreq/view.html
+++ /dev/null
@@ -1,55 +0,0 @@
-{% extends "base_site.html" %}
-{% load static %}
-
-{% block title %}Sessions - View{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-    <script src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block extrastyle %}
-    <style>
-        dl {width: 100%;}
-        dt {float: left; width: 30%; margin: 0.1em 0 0.1em 0; }
-        dt::after {content: ":";}
-        dd {float: left; width: 70%; margin: 0.1em 0 0.1em 0;}
-    </style>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }}
-    &raquo; <a href="../">Sessions</a>
-    &raquo; {{ group.acronym }}
-{% endblock %}
-
-{% block instructions %}
-    <a href="https://www.ietf.org/chairs/session-request-tool-instructions/" target="_blank">Instructions</a>
-{% endblock %}
-
-{% block content %}
-
-    <div class="module interim-container">
-        <h2>Sessions - View (meeting: {{ meeting.number }})</h2>
-
-        {% include "includes/sessions_request_view.html" %}
-
-        <br>
-
-        {% include "includes/activities.html" %}
-
-        <div class="button-group">
-            <ul>
-                <li><button type="button" name="edit" onclick="window.location='{% url "ietf.secr.sreq.views.edit" acronym=group.acronym num=meeting.number %}'"{% if not can_edit %} disabled{% endif %}>Edit</button></li>
-                {% if show_approve_button %}
-                    <li><button type="button" onclick="window.location='approve/'">Approve Third Session</button></li>
-                {% endif %}
-                <li><button type="button" onclick="if (window.confirm('Do you really want to cancel this session?')) { window.location='cancel/' };" {% if not can_cancel %} disabled{% endif %}>Cancel this Request</button></li>
-                <li><button type="button" onclick="window.location='{% url "ietf.secr.sreq.views.main" %}'">Back</button></li>
-            </ul>
-        </div> <!-- button-group -->
-    </div> <!-- module -->
-
-{% endblock %}
-
-{% block footer-extras %}
-    {% include "includes/sessions_footer.html" %}
-{% endblock %}
diff --git a/ietf/secr/urls.py b/ietf/secr/urls.py
index 4a3e5b0363..ab21046654 100644
--- a/ietf/secr/urls.py
+++ b/ietf/secr/urls.py
@@ -1,11 +1,22 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.conf import settings
 from django.urls import re_path, include
 from django.views.generic import TemplateView
+from django.views.generic.base import RedirectView
 
 urlpatterns = [
     re_path(r'^$', TemplateView.as_view(template_name='index.html'), name='ietf.secr'),
     re_path(r'^announcement/', include('ietf.secr.announcement.urls')),
     re_path(r'^meetings/', include('ietf.secr.meetings.urls')),
     re_path(r'^rolodex/', include('ietf.secr.rolodex.urls')),
-    re_path(r'^sreq/', include('ietf.secr.sreq.urls')),
+    # remove these redirects after 125
+    re_path(r'^sreq/$', RedirectView.as_view(url='/meeting/session/request/', permanent=True)),
+    re_path(r'^sreq/%(acronym)s/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/session/request/%(acronym)s/view/', permanent=True)),
+    re_path(r'^sreq/%(acronym)s/edit/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/session/request/%(acronym)s/edit/', permanent=True)),
+    re_path(r'^sreq/%(acronym)s/new/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/session/request/%(acronym)s/new/', permanent=True)),
+    re_path(r'^sreq/(?P<num>[A-Za-z0-9_\-\+]+)/%(acronym)s/view/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/%(num)s/session/request/%(acronym)s/view/', permanent=True)),
+    re_path(r'^sreq/(?P<num>[A-Za-z0-9_\-\+]+)/%(acronym)s/edit/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/%(num)s/session/request/%(acronym)s/edit/', permanent=True)),
+    # ---------------------------------
     re_path(r'^telechat/', include('ietf.secr.telechat.urls')),
 ]
diff --git a/ietf/secr/utils/group.py b/ietf/secr/utils/group.py
deleted file mode 100644
index 40a9065ace..0000000000
--- a/ietf/secr/utils/group.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Copyright The IETF Trust 2013-2020, All Rights Reserved
-# -*- coding: utf-8 -*-
-
-
-# Python imports
-
-# Django imports
-from django.core.exceptions import ObjectDoesNotExist
-
-# Datatracker imports
-from ietf.group.models import Group
-from ietf.ietfauth.utils import has_role
-
-
-def get_my_groups(user,conclude=False):
-    '''
-    Takes a Django user object (from request)
-    Returns a list of groups the user has access to.  Rules are as follows
-    secretariat - has access to all groups
-    area director - has access to all groups in their area
-    wg chair or secretary - has access to their own group
-    chair of irtf has access to all irtf groups
-
-    If user=None than all groups are returned.
-    concluded=True means include concluded groups.  Need this to upload materials for groups
-    after they've been concluded.  it happens.
-    '''
-    my_groups = set()
-    states = ['bof','proposed','active']
-    if conclude:
-        states.extend(['conclude','bof-conc'])
-
-    all_groups = Group.objects.filter(type__features__has_meetings=True, state__in=states).order_by('acronym')
-    if user == None or has_role(user,'Secretariat'):
-        return all_groups
-    
-    try:
-        person = user.person
-    except ObjectDoesNotExist:
-        return list()
-
-    for group in all_groups:
-        if group.role_set.filter(person=person,name__in=('chair','secr','ad')):
-            my_groups.add(group)
-            continue
-        if group.parent and group.parent.role_set.filter(person=person,name__in=('ad','chair')):
-            my_groups.add(group)
-            continue
-
-    return list(my_groups)
diff --git a/ietf/settings.py b/ietf/settings.py
index d6be1d1e0f..9a213c1a73 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -537,7 +537,6 @@ def skip_unreadable_post(record):
     'ietf.secr.announcement',
     'ietf.secr.meetings',
     'ietf.secr.rolodex',
-    'ietf.secr.sreq',
     'ietf.secr.telechat',
 ]
 
diff --git a/ietf/static/js/custom_striped.js b/ietf/static/js/custom_striped.js
new file mode 100644
index 0000000000..480ad7cf82
--- /dev/null
+++ b/ietf/static/js/custom_striped.js
@@ -0,0 +1,16 @@
+// Copyright The IETF Trust 2025, All Rights Reserved
+
+document.addEventListener('DOMContentLoaded', () => {
+    // add stripes
+    const firstRow = document.querySelector('.custom-stripe .row')
+    if (firstRow) {
+        const parent = firstRow.parentElement;
+        const allRows = Array.from(parent.children).filter(child => child.classList.contains('row'))
+        allRows.forEach((row, index) => {
+            row.classList.remove('bg-light')
+            if (index % 2 === 1) {
+                row.classList.add('bg-light')
+            }
+        })
+    }
+})
diff --git a/ietf/secr/static/js/session_form.js b/ietf/static/js/session_form.js
similarity index 92%
rename from ietf/secr/static/js/session_form.js
rename to ietf/static/js/session_form.js
index 6f28f16db4..bd61293d7c 100644
--- a/ietf/secr/static/js/session_form.js
+++ b/ietf/static/js/session_form.js
@@ -1,4 +1,4 @@
-/* Copyright The IETF Trust 2021, All Rights Reserved
+/* Copyright The IETF Trust 2021-2025, All Rights Reserved
  *
  * JS support for the SessionForm
  * */
diff --git a/ietf/secr/static/js/sessions.js b/ietf/static/js/session_request.js
similarity index 90%
rename from ietf/secr/static/js/sessions.js
rename to ietf/static/js/session_request.js
index a2770e6262..dfb169f675 100644
--- a/ietf/secr/static/js/sessions.js
+++ b/ietf/static/js/session_request.js
@@ -1,4 +1,4 @@
-// Copyright The IETF Trust 2015-2021, All Rights Reserved
+// Copyright The IETF Trust 2015-2025, All Rights Reserved
 /* global alert */
 var ietf_sessions; // public interface
 
@@ -38,7 +38,7 @@ var ietf_sessions; // public interface
     const only_one_session = (val === 1);
     if (document.form_post.session_time_relation) {
       document.form_post.session_time_relation.disabled = only_one_session;
-      document.form_post.session_time_relation.closest('tr').hidden = only_one_session;
+      document.form_post.session_time_relation.closest('div.row').hidden = only_one_session;
     }
     if (document.form_post.joint_for_session) {
       document.form_post.joint_for_session.disabled = only_one_session;
@@ -129,6 +129,11 @@ var ietf_sessions; // public interface
     }
   }
 
+  function wg_constraint_delete_clicked(event) {
+    const constraint_name = event.currentTarget.dataset.constraint_name;
+    delete_last_wg_constraint(constraint_name);
+  }
+
   /* Initialization */
   function on_load() {
     // Attach event handler to session count select
@@ -146,6 +151,9 @@ var ietf_sessions; // public interface
       selectors[index].addEventListener('change', wg_constraint_selector_changed, false)
     }
 
+    // Attach event handler to constraint delete buttons
+    document.querySelectorAll('.wg_constraint_delete')
+      .forEach(btn => btn.addEventListener('click', wg_constraint_delete_clicked));
   }
 
   // initialize after page loads
diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html
index bd8c0bf3cd..1e7c1688ff 100644
--- a/ietf/templates/base/menu.html
+++ b/ietf/templates/base/menu.html
@@ -1,4 +1,4 @@
-{# Copyright The IETF Trust 2015-2022, All Rights Reserved #}
+{# Copyright The IETF Trust 2015-2025, All Rights Reserved #}
 {% load origin %}
 {% origin %}
 {% load ietf_filters managed_groups wg_menu active_groups_menu group_filters cache meetings_filters %}
@@ -304,7 +304,7 @@
     </li>
     <li>
         <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
-           href="{% url 'ietf.secr.sreq.views.main' %}">
+           href="{% url 'ietf.meeting.views_session_request.list_view' %}">
             Request a session
         </a>
     </li>
diff --git a/ietf/templates/group/meetings-row.html b/ietf/templates/group/meetings-row.html
index 25605ec0f1..8927eb61a2 100644
--- a/ietf/templates/group/meetings-row.html
+++ b/ietf/templates/group/meetings-row.html
@@ -1,3 +1,4 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
 {% load origin tz %}
 {% origin %}
 {% for s in sessions %}
@@ -25,7 +26,7 @@
             {% if show_request and s.meeting.type_id == 'ietf' %}
                 {% if can_edit %}
                     <a class="btn btn-primary btn-sm"
-                       href="{% url 'ietf.secr.sreq.views.view' num=s.meeting.number acronym=s.group.acronym %}">
+                       href="{% url 'ietf.meeting.views_session_request.view_request' num=s.meeting.number acronym=s.group.acronym %}">
                         Edit Session Request
                     </a>
                 {% endif %}
diff --git a/ietf/templates/group/meetings.html b/ietf/templates/group/meetings.html
index bee8111025..30f478da13 100644
--- a/ietf/templates/group/meetings.html
+++ b/ietf/templates/group/meetings.html
@@ -1,3 +1,4 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
 {% extends "group/group_base.html" %}
 {% load origin static %}
 {% block title %}
@@ -9,7 +10,7 @@
     <a class="btn btn-primary"
        href="{% url 'ietf.meeting.views.meeting_requests' %}">Session requests</a>
     {% if can_edit or can_always_edit %}
-        <a class="btn btn-primary" href="{% url 'ietf.secr.sreq.views.main' %}">Request a session</a>
+        <a class="btn btn-primary" href="{% url 'ietf.meeting.views_session_request.list_view' %}">Request a session</a>
         <a class="btn btn-primary"
            href="{% url 'ietf.meeting.views.interim_request' %}?group={{ group.id }}">
             Request an interim meeting
diff --git a/ietf/templates/meeting/important_dates_for_meeting.ics b/ietf/templates/meeting/important_dates_for_meeting.ics
index df5fe46818..e6d403da93 100644
--- a/ietf/templates/meeting/important_dates_for_meeting.ics
+++ b/ietf/templates/meeting/important_dates_for_meeting.ics
@@ -1,3 +1,4 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
 {% load tz ietf_filters %}{% for d in meeting.important_dates %}BEGIN:VEVENT
 UID:ietf-{{ meeting.number }}-{{ d.name_id }}-{{ d.date.isoformat }}
 SUMMARY:IETF {{ meeting.number }}: {{ d.name.name }}
@@ -8,11 +9,11 @@ TRANSP:TRANSPARENT
 DESCRIPTION:{{ d.name.desc }}{% if first and d.name.slug == 'openreg' or first and d.name.slug == 'earlybird' %}\n
  Register here: https://www.ietf.org/how/meetings/register/{% endif %}{% if d.name.slug == 'opensched' %}\n
  To request a Working Group session, use the IETF Meeting Session Request Tool:\n
- {{ request.scheme }}://{{ request.get_host}}{% url 'ietf.secr.sreq.views.main' %}\n
+ {{ request.scheme }}://{{ request.get_host}}{% url 'ietf.meeting.views_session_request.list_view' %}\n
  If you are working on a BOF request, it is highly recommended to tell the IESG\n
  now by sending an email to iesg@ietf.org to get advance help with the request.{% endif %}{% if d.name.slug == 'cutoffwgreq' %}\n
  To request a Working Group session, use the IETF Meeting Session Request Tool:\n
- {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.secr.sreq.views.main' %}{% endif %}{% if d.name.slug == 'cutoffbofreq' %}\n
+ {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.meeting.views_session_request.list_view' %}{% endif %}{% if d.name.slug == 'cutoffbofreq' %}\n
  To request a BOF, please see instructions on Requesting a BOF:\n
  https://www.ietf.org/how/bofs/bof-procedures/{% endif %}{% if d.name.slug == 'idcutoff' %}\n
  Upload using the I-D Submission Tool:\n
diff --git a/ietf/templates/meeting/requests.html b/ietf/templates/meeting/requests.html
index 3008ceb662..0abee95887 100644
--- a/ietf/templates/meeting/requests.html
+++ b/ietf/templates/meeting/requests.html
@@ -1,5 +1,5 @@
 {% extends "base.html" %}
-{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{# Copyright The IETF Trust 2015-2025, All Rights Reserved #}
 {% load origin %}
 {% load ietf_filters static person_filters textfilters %}
 {% block pagehead %}
@@ -151,7 +151,7 @@ <h2 class="mt-5" id="{% firstof area.grouper.acronym "other-groups" %}">
                 {% endifchanged %}
                     <tr>
                         <td>
-                            <a href="{% url "ietf.secr.sreq.views.edit" num=meeting.number acronym=session.group.acronym %}">
+                            <a href="{% url 'ietf.meeting.views_session_request.edit_request' num=meeting.number acronym=session.group.acronym %}">
                                 {{ session.group.acronym }}
                             </a>
                             {% if session.purpose_id != "regular" and session.purpose_id != "none" %}
diff --git a/ietf/secr/templates/sreq/session_approval_notification.txt b/ietf/templates/meeting/session_approval_notification.txt
similarity index 56%
rename from ietf/secr/templates/sreq/session_approval_notification.txt
rename to ietf/templates/meeting/session_approval_notification.txt
index 7bb63aa3fa..74eca09bd8 100644
--- a/ietf/secr/templates/sreq/session_approval_notification.txt
+++ b/ietf/templates/meeting/session_approval_notification.txt
@@ -1,3 +1,4 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
 Dear {{ group.parent }} Director(s):
 
 {{ header }} meeting session request has just been
@@ -5,11 +6,11 @@ submitted by {{ requester }}.
 The third session requires your approval.
 
 To approve the session go to the session request view here:
-{{ settings.IDTRACKER_BASE_URL }}{% url "ietf.secr.sreq.views.view" acronym=group.acronym %}
+{{ settings.IDTRACKER_BASE_URL }}{% url "ietf.meeting.views_session_request.view_request" acronym=group.acronym %}
 and click "Approve Third Session".
 
 Regards,
 
 The IETF Secretariat.
 
-{% include "includes/session_info.txt" %}
+{% include "meeting/session_request_info.txt" %}
diff --git a/ietf/secr/templates/sreq/session_cancel_notification.txt b/ietf/templates/meeting/session_cancel_notification.txt
similarity index 71%
rename from ietf/secr/templates/sreq/session_cancel_notification.txt
rename to ietf/templates/meeting/session_cancel_notification.txt
index 8aee6c89db..3de67fc8f4 100644
--- a/ietf/secr/templates/sreq/session_cancel_notification.txt
+++ b/ietf/templates/meeting/session_cancel_notification.txt
@@ -1,3 +1,4 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
 {% autoescape off %}{% load ams_filters %}
 
 A request to cancel a meeting session has just been submitted by {{ requester }}.{% endautoescape %}
diff --git a/ietf/templates/meeting/session_details_form.html b/ietf/templates/meeting/session_details_form.html
index 6b59e7dacd..9cd1b6e85c 100644
--- a/ietf/templates/meeting/session_details_form.html
+++ b/ietf/templates/meeting/session_details_form.html
@@ -1,42 +1,48 @@
-{# Copyright The IETF Trust 2007-2020, All Rights Reserved #}
+{# Copyright The IETF Trust 2007-2025, All Rights Reserved #}
+{% load django_bootstrap5 %}
+
 <div class="session-details-form my-3" data-prefix="{{ form.prefix }}">
     {% if hidden %}
         {{ form.name.as_hidden }}{{ form.purpose.as_hidden }}{{ form.type.as_hidden }}{{ form.requested_duration.as_hidden }}
         {{ form.has_onsite_tool.as_hidden }}
     {% else %}
-    <table>
-        <tbody>
         {% comment %}
         The form-group class is used by session_details_form.js to identify the correct element
         to hide the name / purpose / type fields when not needed. This is a bootstrap class - the
         secr app does not use it, so this (and the hidden class, also needed by session_details_form.js)
         are defined in edit.html and new.html as a kludge to make this work.
         {% endcomment %}
-      <tr class="form-group">
-                <th scope="row">{{ form.name.label_tag }}</th>
-                <td>{{ form.name }}{{ form.purpose.errors }}</td>
-            </tr>
-      <tr class="form-group">
-                <th scope="row">{{ form.purpose.label_tag }}</th>
-                <td>
-            {{ form.purpose }} <div class="form-group">{{ form.type }}</div>
-                    {{ form.purpose.errors }}{{ form.type.errors }}
-                </td>
-            </tr>
-            <tr>
-                <th scope="row">{{ form.requested_duration.label_tag }}</th>
-                <td>{{ form.requested_duration }}{{ form.requested_duration.errors }}</td>
-            </tr>
-            {% if not hide_onsite_tool_prompt %}
-            <tr>
-                <th scope="row">{{ form.has_onsite_tool.label_tag }}</th>
-                <td>{{ form.has_onsite_tool }}{{ form.has_onsite_tool.errors }}</td>
-            </tr>
-            {% endif %}
-            </tbody>
-        </table>
-        {% if hide_onsite_tool_prompt %}{{ form.has_onsite_tool.as_hidden }}{% endif %}
+
+        <div class="form-group">
+            {% bootstrap_field form.name layout="horizontal" %}
+        </div>
+
+        <div class="form-group">
+            <div class="row mb-3 g-3">
+                <label class="col-md-2 fw-bold col-form-label" for="{{ form.purpose.id_for_label }}">Purpose</label>
+                <div class="col-md-5">{{ form.purpose }}</div>
+                <div class="col-md-5 form-group">{{ form.type }}</div>
+                {{ form.purpose.errors }}{{ form.type.errors }}
+            </div>
+        </div>
+
+        {% bootstrap_field form.requested_duration layout="horizontal" %}
+        {% if not hide_onsite_tool_prompt %}
+            {% bootstrap_field form.has_onsite_tool layout="horizontal" %}
+        {% endif %}
+        
+        {% if hide_onsite_tool_prompt %}
+            {{ form.has_onsite_tool.as_hidden }}
+        {% endif %}
     {% endif %}
+
     {# hidden fields included whether or not the whole form is hidden #}
-    {{ form.attendees.as_hidden }}{{ form.comments.as_hidden }}{{ form.id.as_hidden }}{{ form.on_agenda.as_hidden }}{{ form.DELETE.as_hidden }}{{ form.remote_instructions.as_hidden }}{{ form.short.as_hidden }}{{ form.agenda_note.as_hidden }}
-</div>
\ No newline at end of file
+    {{ form.attendees.as_hidden }}
+    {{ form.comments.as_hidden }}
+    {{ form.id.as_hidden }}
+    {{ form.on_agenda.as_hidden }}
+    {{ form.DELETE.as_hidden }}
+    {{ form.remote_instructions.as_hidden }}
+    {{ form.short.as_hidden }}
+    {{ form.agenda_note.as_hidden }}
+</div>
diff --git a/ietf/secr/templates/sreq/not_meeting_notification.txt b/ietf/templates/meeting/session_not_meeting_notification.txt
similarity index 83%
rename from ietf/secr/templates/sreq/not_meeting_notification.txt
rename to ietf/templates/meeting/session_not_meeting_notification.txt
index 1120f8480c..0e5c940708 100644
--- a/ietf/secr/templates/sreq/not_meeting_notification.txt
+++ b/ietf/templates/meeting/session_not_meeting_notification.txt
@@ -1,3 +1,4 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
 {% load ams_filters %}
 
 {{ login|smart_login }} {{ group.acronym }} working group, indicated that the {{ group.acronym }} working group does not plan to hold a session at IETF {{ meeting.number }}.
diff --git a/ietf/templates/meeting/session_request_confirm.html b/ietf/templates/meeting/session_request_confirm.html
new file mode 100644
index 0000000000..09043d3d0c
--- /dev/null
+++ b/ietf/templates/meeting/session_request_confirm.html
@@ -0,0 +1,38 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% extends "base.html" %}
+{% load static ietf_filters django_bootstrap5 %}
+{% block title %}Confirm Session Request{% endblock %}
+    
+{% block content %}
+    <h1 class="">Confirm Session Request - IETF {{ meeting.number }}</h1>
+
+    <div class="d-flex mb-3">
+      <a href="https://www.ietf.org/chairs/session-request-tool-instructions/" class="me-3 btn btn-primary" target="_blank">Instructions</a>
+    </div>
+
+    <hr>
+
+    <div class="custom-stripe">
+
+        {% include "meeting/session_request_view_table.html" %}
+
+        <form action="{% url 'ietf.meeting.views_session_request.confirm' acronym=group.acronym %}" method="post">
+            {% csrf_token %}
+            {{ form }}
+            {{ form.session_forms.management_form }}
+            {% for sf in form.session_forms %}
+                {% include 'meeting/session_details_form.html' with form=sf hidden=True only %}
+            {% endfor %}
+
+            <button class="btn btn-primary" type="submit" name="submit" value="Submit">Submit</button>
+            <button class="btn btn-secondary float-end" type="submit" name="submit" value="Cancel">Cancel</button>
+
+        </form>
+
+    </div> <!-- view-container -->
+
+{% endblock %}
+
+{% block js %}
+    <script src="{% static 'ietf/js/custom_striped.js' %}"></script>
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/meeting/session_request_form.html b/ietf/templates/meeting/session_request_form.html
new file mode 100644
index 0000000000..ecf5cb7268
--- /dev/null
+++ b/ietf/templates/meeting/session_request_form.html
@@ -0,0 +1,206 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% extends "base.html" %}
+{% load static ietf_filters django_bootstrap5 %}
+{% block title %}{% if is_create %}New {% else %}Edit {% endif %}Session Request{% endblock %}
+{% block morecss %}{{ block.super }}
+  .hidden {display: none !important;}
+  div.form-group {display: inline;}
+{% endblock %}
+{% block content %}
+    <h1>{% if is_create %}New {% else %}Edit {% endif %}Session Request</h1>
+
+    {% if is_create %}
+    <div class="buttonlist mb-3">
+        <a class="btn btn-primary me-3"
+           onclick="if (window.confirm('A message will be sent to agenda@ietf.com as well as to the area director(s).\n\nContinue?')) { window.location='{% url "ietf.meeting.views_session_request.no_session" acronym=group.acronym %}'};"
+           href="{% url 'ietf.meeting.views_session_request.no_session' acronym=group.acronym %}">Send a notification that the group does not plan to hold a session at IETF {{ meeting.number }}</a>
+        <a class="btn btn-primary"
+           href="{% url 'ietf.meeting.views_session_request.new_request' acronym=group.acronym %}?previous">Retrieve all information from previous meeting</a>
+    </div>
+    {% endif %}
+
+    <hr>
+
+    <form id="session-request-form" action="." method="post" name="form_post">
+        {% csrf_token %}
+        {{ form.session_forms.management_form }}
+        {% if form.non_field_errors %}
+            <div class="my-3 alert alert-danger">{{ form.non_field_errors }}</div>
+        {% endif %}
+
+        <div class="row mb-3">
+            <label class="col-md-2 fw-bold col-form-label" for="id_group_name">Working Group Name</label>
+            <div class="col-md-10">
+                <input type="text" class="form-control" value="{{ group.name }} ({{ group.acronym }})" id="id_group_name" disabled>
+            </div>
+        </div>
+
+        <div class="row mb-3">
+            <label class="col-md-2 fw-bold col-form-label" for="id_area_name">Area Name</label>
+            <div class="col-md-10">
+                <input type="text" class="form-control" value="{% if group.parent %}{{ group.parent.name }} ({{ group.parent.acronym }}){% endif %}" id="id_area_name" disabled>
+            </div>
+        </div>
+
+        {% bootstrap_field form.num_session layout="horizontal" %}
+
+        {% if group.features.acts_like_wg %}
+
+            <div class="row mb-3" id="session_row_0">
+                <div class="col-md-2 fw-bold col-form-label">Session 1</div>
+                <div class="col-md-10">
+                    {% include 'meeting/session_details_form.html' with form=form.session_forms.0 hide_onsite_tool_prompt=True only %}
+                </div>
+            </div>
+
+            <div class="row mb-3" id="session_row_1">
+                <div class="col-md-2 fw-bold col-form-label">Session 2</div>
+                <div class="col-md-10">
+                    {% include 'meeting/session_details_form.html' with form=form.session_forms.1 hide_onsite_tool_prompt=True only %}
+                </div>
+            </div>
+
+            {% if not is_virtual %}
+                {% bootstrap_field form.session_time_relation layout="horizontal" %}
+            {% endif %}
+
+            <div class="row mb-3" id="third_session_row">
+                <div class="col-md-2 fw-bold col-form-label">Additional Session Request</div>
+                <div class="col-md-10">
+                    <div class="form-check">
+                        {{ form.third_session }}
+                        <label class="form-check-label" for="id_third_session">Check this box to request an additional session</label>
+                        <div id="id_third_session_helptext" class="form-text">Additional slot may be available after agenda scheduling has closed and with the approval of an Area Director.</div>
+                    </div>
+                    
+                </div>
+            </div>
+
+            <div class="row mb-3" id="session_row_2">
+                <div class="col-md-2 fw-bold col-form-label">Third session request</div>
+                <div class="col-md-10">
+                    {% include 'meeting/session_details_form.html' with form=form.session_forms.2 hide_onsite_tool_prompt=True only %}
+                </div>
+            </div>
+
+        {% else %}{# else not group.features.acts_like_wg #}
+            {% for session_form in form.session_forms %}
+            <div class="row mb-3" id="session_row_{{ forloop.counter0 }}">
+                <div class="col-md-2 fw-bold col-form-label">Session {{ forloop.counter }}</div>
+                <div class="col-md-10">
+                    {% include 'meeting/session_details_form.html' with form=session_form only %}
+                </div>
+            </div>
+            {% endfor %}
+        {% endif %}
+
+        {% bootstrap_field form.attendees layout="horizontal" %}
+
+        {% bootstrap_field form.bethere layout="horizontal" %}
+
+        <div class="row mb-3">
+            <div class="col-md-2 fw-bold col-form-label">Conflicts to avoid</div>
+            <div class="col-md-10">
+                <div class="row mb-3">
+                    <div class="col-md-2">Other WGs that included <strong>{{ group.acronym }}</strong> in their conflict lists</div>
+                    <div class="col-md-8">{{ session_conflicts.inbound|default:"None" }}</div>
+                </div>
+                <div class="row mb-3">
+                    <div class="col-md-2">WG Sessions<br>You may select multiple WGs within each category</div>
+                    <div class="col-md">
+                        {% for cname, cfield, cselector  in form.wg_constraint_fields %}
+                        <div class="row mb-2">
+                            <div class="col-md-2"><label for="id_wg_selector_{{ cname.slug }}">{{ cname|title }}</label></div>
+                            <div class="col">
+                                <div class="row g-3 mb-1">
+                                    <div class="col">
+                                        {{ cselector }}
+                                    </div>
+                                    <div class="col">
+                                        <button type="button" class="btn btn-primary wg_constraint_delete" id="wg_delete_{{ cname.slug }}" data-constraint_name="{{ cname.slug }}">Delete the last entry</button>
+                                    </div>
+                                </div>
+                                <div class="row g-3">
+                                    <div class="col">
+                                        {{ cfield.errors }}{{ cfield }}
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
+                        {% empty %}{# shown if there are no constraint fields #}
+                            <div class="col-md-1"></div>
+                            <div class="col-md">No constraints are enabled for this meeting.</div>
+                        {% endfor %}
+                    </div>
+                </div>
+                
+                {% if form.inactive_wg_constraint_count %}
+                <div class="row mb-3">
+                    <div class="col-md-2">Disabled for this meeting</div>
+                    <div class="col">
+                        {% for cname, value, field in form.inactive_wg_constraints %}
+                        <div class="row mb2">
+                            <div class="col-md-2">{{ cname|title }}</div>
+                            <div class="col">
+                                <div class="row">
+                                    <div class="col">
+                                        <input type="text" value="{{ value }}" maxlength="255" class="wg_constraint form-control" aria-label="constraint target" disabled>
+                                    </div>
+                                    <div class="col form-check">
+                                        <input type="checkbox" name="{{ field.name }}" id="{{ field.id_for_label }}" class="form-check-input">
+                                        <label class="form-check-label" for="{{ field.id_for_label }}">Delete this conflict</label>
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
+                        {% endfor %}
+                    </div>
+                </div>
+                {% endif %}
+
+                <div class="row mb-3">
+                    <div class="col-md-2">BOF Sessions</div>
+                    <div class="col">If the sessions can not be found in the fields above, please enter free form requests in the Special Requests field below.</div>
+                </div>
+            </div>
+        </div>
+
+        {% if not is_virtual %}
+
+            {% bootstrap_field form.resources layout="horizontal" %}
+
+            {% bootstrap_field form.timeranges layout="horizontal" %}
+
+            {% bootstrap_field form.adjacent_with_wg layout="horizontal" %}
+
+            <div class="row mb-3">
+                <div class="col-md-2 fw-bold col-form-label">Joint session with: (To request one session for multiple WGs together)</div>
+                <div class="col-md-10">To request a joint session with another group, please contact the secretariat.</div>
+            </div>
+
+        {% endif %}
+
+        {% bootstrap_field form.comments layout="horizontal" %}
+
+        {% if form.notifications_optional %}
+        <div class="row mb-3">
+            <label class="col-md-2 fw-bold col-form-label" for="id_send_notifications">Notifications</label>
+            <div class="col">
+                <div class="form-check">
+                    <input type="checkbox" name="send_notifications" class="form-check-input" id="id_send_notifications">
+                    <label class="form-check-label" for="id_send_notifications">Send notification emails?</label>
+                </div>
+            </div>
+        </div>
+        {% endif %}
+        
+        <button type="submit" class="btn btn-primary" name="submit" value="Save"{% if is_locked %} disabled{% endif %}>Save</button>
+        <a class="btn btn-secondary float-end"
+               href="{% url 'ietf.meeting.views_session_request.list_view' %}">Cancel</a>
+    </form>
+
+{% endblock %}
+{% block js %}
+    <script src="{% static 'ietf/js/session_request.js' %}"></script>
+    {{ form.media }}
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/meeting/session_request_info.txt b/ietf/templates/meeting/session_request_info.txt
new file mode 100644
index 0000000000..2e96efb31f
--- /dev/null
+++ b/ietf/templates/meeting/session_request_info.txt
@@ -0,0 +1,26 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% load ams_filters %}
+---------------------------------------------------------
+Working Group Name: {{ group.name }}
+Area Name: {{ group.parent }}
+Session Requester: {{ login }}
+{% if session.joint_with_groups %}{{ session.joint_for_session_display }} joint with: {{ session.joint_with_groups }}{% endif %}
+
+Number of Sessions: {{ session.num_session }}
+Length of Session(s): {% for session_length in session_lengths %}{{ session_length.total_seconds|display_duration }}{% if not forloop.last %}, {% endif %}{% endfor %}
+Number of Attendees: {{ session.attendees }}
+Conflicts to Avoid: 
+{% for line in session.outbound_conflicts %} {{line}}
+{% endfor %}{% if session.session_time_relation_display %} {{ session.session_time_relation_display }}{% endif %}
+{% if session.adjacent_with_wg %} Adjacent with WG: {{ session.adjacent_with_wg }}{% endif %}       
+{% if session.timeranges_display %} Can't meet: {{ session.timeranges_display|join:", " }}{% endif %}
+
+Participants who must be present:
+{% for person in session.bethere %}  {{ person.ascii_name }}
+{% endfor %}
+Resources Requested:
+{% for resource in session.resources %}  {{ resource.desc }}
+{% endfor %}
+Special Requests:
+  {{ session.comments }}
+---------------------------------------------------------
diff --git a/ietf/templates/meeting/session_request_list.html b/ietf/templates/meeting/session_request_list.html
new file mode 100644
index 0000000000..789b7006e5
--- /dev/null
+++ b/ietf/templates/meeting/session_request_list.html
@@ -0,0 +1,65 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% extends "base.html" %}
+{% load static %}
+{% load ietf_filters %}
+{% load django_bootstrap5 %}
+{% block title %}Session Requests{% endblock %}
+{% block content %}
+    <h1>Session Requests IETF {{ meeting.number }}</h1>
+
+    <div class="d-flex mb-3">
+      <a href="https://www.ietf.org/chairs/session-request-tool-instructions/" class="btn btn-primary me-3" target="_blank">Instructions</a>
+
+      <a href="{% url 'ietf.meeting.views.meeting_requests' %}" class="btn btn-primary me-3">View list of timeslot requests</a>
+      {% if user|has_role:"Secretariat" %}
+        {% if is_locked %}
+          <a href="{% url 'ietf.meeting.views_session_request.status' %}" class="btn btn-danger ms-auto">Unlock Tool</a>
+        {% else %}
+          <a href="{% url 'ietf.meeting.views_session_request.status' %}" class="btn btn-primary ms-auto">Lock Tool</a>
+        {% endif %}
+      {% endif %}
+    </div>
+    
+    <div class="card mb-3">
+      <div class="card-header">
+        Request New Session
+      </div>
+      <div class="card-body">
+        <p class="card-text">The list below includes those working groups that you currently chair which do not already have a session scheduled. You can click on an acronym to complete a request for a new session at the upcoming IETF meeting.  Click "Group will not meet" to send a notification that the group does not plan to meet.</p>
+        <ul>
+            {% for group in unscheduled_groups %}
+                <li>
+                    <a href="{% url 'ietf.meeting.views_session_request.new_request' acronym=group.acronym %}">{{ group.acronym }}</a>
+                    {% if group.not_meeting %}
+                        <span class="required"> (Currently, this group does not plan to hold a session at IETF {{ meeting.number }})</span>
+                    {% endif %}
+                </li>
+            {% empty %}
+                <li><i>NONE</i></li>
+            {% endfor %}
+        </ul>
+      </div>
+    </div> <!-- card -->
+
+
+    <div class="card">
+      <div class="card-header">
+        Edit / Cancel Previously Requested Sessions
+      </div>
+      <div class="card-body">
+        <p class="card-text">The list below includes those working groups for which you or your co-chair has requested sessions at the upcoming IETF meeting. You can click on an acronym to initiate changes to a session, or cancel a session.</p>
+        <ul>
+            {% for group in scheduled_groups %}
+                <li><a href="{% url 'ietf.meeting.views_session_request.view_request' acronym=group.acronym %}">{{ group.acronym }} - {{ group.status_message }}</a></li>
+            {% empty %}
+                <li><i>NONE</i></li>
+            {% endfor %}
+        </ul>
+      </div>
+    </div> <!-- card -->
+
+{% endblock %}
+
+{% block footer-extras %}
+    {% include "includes/sessions_footer.html" %}
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/meeting/session_request_locked.html b/ietf/templates/meeting/session_request_locked.html
new file mode 100644
index 0000000000..15c023ce33
--- /dev/null
+++ b/ietf/templates/meeting/session_request_locked.html
@@ -0,0 +1,21 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% extends "base.html" %}
+{% load static ietf_filters django_bootstrap5 %}
+{% block title %}Session Request{% endblock %}
+    
+{% block content %}
+    <h1 class="">Session Request - IETF {{ meeting.number }}</h1>
+
+    <a href="{% url 'ietf.meeting.views.meeting_requests' %}" class="btn btn-primary">View list of timeslot requests</a>
+
+    <hr>
+
+    <div class="">
+        <p>{{ message }}</p>
+
+        <div>
+            <button class="btn btn-primary" type="button" onclick="history.back()">Back</button>
+        </div>
+    </div> 
+
+{% endblock %}
diff --git a/ietf/secr/templates/sreq/session_request_notification.txt b/ietf/templates/meeting/session_request_notification.txt
similarity index 56%
rename from ietf/secr/templates/sreq/session_request_notification.txt
rename to ietf/templates/meeting/session_request_notification.txt
index 75f2cbbae4..49dbbfc42c 100644
--- a/ietf/secr/templates/sreq/session_request_notification.txt
+++ b/ietf/templates/meeting/session_request_notification.txt
@@ -1,5 +1,6 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
 {% autoescape off %}{% load ams_filters %}
 
 {% filter wordwrap:78 %}{{ header }} meeting session request has just been submitted by {{ requester }}.{% endfilter %}
 
-{% include "includes/session_info.txt" %}{% endautoescape %}
+{% include "meeting/session_request_info.txt" %}{% endautoescape %}
diff --git a/ietf/templates/meeting/session_request_status.html b/ietf/templates/meeting/session_request_status.html
new file mode 100644
index 0000000000..65e98d6d23
--- /dev/null
+++ b/ietf/templates/meeting/session_request_status.html
@@ -0,0 +1,28 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% extends "base.html" %}
+{% load static %}
+{% load ietf_filters %}
+{% load django_bootstrap5 %}
+{% block title %}Session Request Status{% endblock %}
+{% block content %}
+    <h1>Session Request Status</h1>
+
+    <div class="card">
+      <div class="card-header">
+        Session Request Status
+      </div>
+      <div class="card-body">
+        <p class="card-text">Enter the message that you would like displayed to the WG Chair when this tool is locked.</p>
+        <form method="post">{% csrf_token %}
+            {% bootstrap_form form %}
+            {% if is_locked %}
+                <button class="btn btn-primary" type="submit" name="submit" value="Unlock">Unlock</button>
+            {% else %}
+                <button class="btn btn-primary" type="submit" name="submit" value="Lock">Lock</button>
+            {% endif %}
+            <button class="btn btn-primary ms-auto" type="submit" name="submit" value="Back">Back</button>
+        </form>
+      </div>
+    </div>
+
+{% endblock %}
diff --git a/ietf/templates/meeting/session_request_view.html b/ietf/templates/meeting/session_request_view.html
new file mode 100644
index 0000000000..3db16f56cb
--- /dev/null
+++ b/ietf/templates/meeting/session_request_view.html
@@ -0,0 +1,59 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% extends "base.html" %}
+{% load static ietf_filters django_bootstrap5 %}
+{% block title %}Session Request{% endblock %}
+    
+{% block content %}
+    <h1 class="">Session Request - IETF {{ meeting.number }}</h1>
+
+    <div class="d-flex mb-3">
+      <a href="https://www.ietf.org/chairs/session-request-tool-instructions/" class="me-3 btn btn-primary" target="_blank">Instructions</a>
+    </div>
+
+    <hr>
+
+    <div class="custom-stripe">
+
+        {% include "meeting/session_request_view_table.html" %}
+
+        <hr>
+
+        <h2>Activities Log</h2>
+        <div class="">
+            <table class=" table table-sm full-width">
+                <thead>
+                    <tr>
+                        <th scope="col">Date</th>
+                        <th scope="col">Time</th>
+                        <th scope="col">Action</th>
+                        <th scope="col">Name</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for entry in activities %}
+                        <tr class="{% cycle 'row1' 'row2' %}">
+                            <td>{{ entry.act_date }}</td>
+                            <td>{{ entry.act_time }}</td>
+                            <td>{{ entry.activity }}</td>
+                            <td>{{ entry.act_by }}</td>
+                        </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+        </div>
+
+
+      <button type="button" name="edit" class="btn btn-primary me-2" onclick="window.location='{% url 'ietf.meeting.views_session_request.edit_request' acronym=group.acronym num=meeting.number %}'"{% if not can_edit %} disabled{% endif %}>Edit</button>
+      {% if show_approve_button %}
+        <a href="{% url 'ietf.meeting.views_session_request.approve_request' acronym=group.acronym %}" class="btn btn-primary me-2">Approve Third Session</a>
+      {% endif %}
+      <button type="button" class="btn btn-primary me-2" onclick="if (window.confirm('Do you really want to cancel this session?')) { window.location='{% url 'ietf.meeting.views_session_request.cancel_request' acronym=group.acronym %}' };" {% if not can_cancel %} disabled{% endif %}>Cancel this Request</button>
+      <a href="{% url 'ietf.meeting.views_session_request.list_view' %}" class="btn btn-secondary float-end">Back</a>
+  
+    </div> <!-- view-container -->
+
+{% endblock %}
+
+{% block js %}
+    <script src="{% static 'ietf/js/custom_striped.js' %}"></script>
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/meeting/session_request_view_formset.html b/ietf/templates/meeting/session_request_view_formset.html
new file mode 100644
index 0000000000..72811b8c2c
--- /dev/null
+++ b/ietf/templates/meeting/session_request_view_formset.html
@@ -0,0 +1,49 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% load ams_filters %}{# keep this in sync with sessions_request_view_session_set.html #}
+{% for sess_form in formset %}
+    {% if sess_form.cleaned_data and not sess_form.cleaned_data.DELETE %}
+        <div class="row mb-3">
+            <div class="col-md-2 fw-bold">
+                Session {{ forloop.counter }}
+            </div>
+            <div class="col">
+                <div class="row mb-2">
+                    <div class="col-sm-2 fw-bold">Length</div>
+                    <div class="col">{{ sess_form.cleaned_data.requested_duration.total_seconds|display_duration }}</div>
+                </div>
+                {% if sess_form.cleaned_data.name %}
+                    <div class="row mb-2">
+                        <div class="col-sm-2 fw-bold">Name</div>
+                        <div class="col">{{ sess_form.cleaned_data.name }}</div>
+                    </div>
+                {% endif %}
+                {% if sess_form.cleaned_data.purpose.slug != 'regular' %}
+                    <div class="row mb-2">
+                        <div class="col-sm-2 fw-bold">Purpose</div>
+                        <div class="col">
+                            {{ sess_form.cleaned_data.purpose }}
+                            {% if sess_form.cleaned_data.purpose.timeslot_types|length > 1 %}({{ sess_form.cleaned_data.type }}
+                      ){% endif %}
+                        </div>
+                    </div>
+                    <div class="row mb-2">
+                        <div class="col-sm-2 fw-bold">Onsite tool?</div>
+                        <div class="col">{{ sess_form.cleaned_data.has_onsite_tool|yesno }}</div>
+                    </div>
+                {% endif %}
+            </div>
+        </div>
+
+        {% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %}
+        <div class="row mb-3">
+            <div class="col-md-2 fw-bold">
+                Time between sessions
+            </div>
+            <div class="col">
+                {% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No
+            preference{% endif %}
+            </div>
+        </div>
+        {% endif %}
+    {% endif %}
+{% endfor %}
\ No newline at end of file
diff --git a/ietf/templates/meeting/session_request_view_session_set.html b/ietf/templates/meeting/session_request_view_session_set.html
new file mode 100644
index 0000000000..0b8412b04f
--- /dev/null
+++ b/ietf/templates/meeting/session_request_view_session_set.html
@@ -0,0 +1,47 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% load ams_filters %}{# keep this in sync with sessions_request_view_formset.html #}
+{% for sess in session_set %}
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Session {{ forloop.counter }}
+    </div>
+    <div class="col">
+        <div class="row mb-2">
+            <div class="col-sm-2 fw-bold">Length</div>
+            <div class="col">{{ sess.requested_duration.total_seconds|display_duration }}</div>
+        </div>
+        {% if sess.name %}
+            <div class="row mb-2">
+                <div class="col-sm-2 fw-bold">Name</div>
+                <div class="col">{{ sess.name }}</div>
+            </div>
+        {% endif %}
+        {% if sess.purpose.slug != 'regular' %}
+            <div class="row mb-2">
+                <div class="col-sm-2 fw-bold">Purpose</div>
+                <div class="col">
+                    {{ sess.purpose }}
+                    {% if sess.purpose.timeslot_types|length > 1 %}({{ sess.type }}){% endif %}
+                </div>
+            </div>
+            <div class="row mb-2">
+                <div class="col-sm-2 fw-bold">Onsite tool?</div>
+                <div class="col">{{ sess.has_onsite_tool|yesno }}</div>
+            </div>
+        {% endif %}
+    </div>
+</div>
+
+{% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %}
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Time between sessions
+    </div>
+    <div class="col">
+        {% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No
+    preference{% endif %}
+    </div>
+</div>
+{% endif %}
+
+{% endfor %}
\ No newline at end of file
diff --git a/ietf/templates/meeting/session_request_view_table.html b/ietf/templates/meeting/session_request_view_table.html
new file mode 100644
index 0000000000..a5cb85c252
--- /dev/null
+++ b/ietf/templates/meeting/session_request_view_table.html
@@ -0,0 +1,146 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% load ams_filters %}
+
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Working Group Name
+    </div>
+    <div class="col">
+        {{ group.name }} ({{ group.acronym }})
+    </div>
+</div>
+
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Area Name
+    </div>
+    <div class="col">
+        {{ group.parent }}
+    </div>
+</div>
+
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Number of Sessions Requested
+    </div>
+    <div class="col">
+        {% if session.third_session %}3{% else %}{{ session.num_session }}{% endif %}
+    </div>
+</div>
+
+{% if form %}
+    {% include 'meeting/session_request_view_formset.html' with formset=form.session_forms group=group session=session only %}
+{% else %}
+    {% include 'meeting/session_request_view_session_set.html' with session_set=sessions group=group session=session only %}
+{% endif %}
+
+
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Number of Attendees
+    </div>
+    <div class="col">
+        {{ session.attendees }}
+    </div>
+</div>
+
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Conflicts to Avoid
+    </div>
+    <div class="col">
+        {% if session_conflicts.outbound %}
+            {% for conflict in session_conflicts.outbound %}
+                <div class="row mb-1">
+                    <div class="col-sm-2 fw-bold">
+                        {{ conflict.name|title }}
+                    </div>
+                    <div class="col">
+                        {{ conflict.groups }}
+                    </div>
+                </div>
+            {% endfor %}
+        {% else %}<i>None</i>{% endif %}
+    </div>
+</div>
+
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Other WGs that included {{ group }} in their conflict list
+    </div>
+    <div class="col">
+        {% if session_conflicts.inbound %}{{ session_conflicts.inbound }}{% else %}<i>None so far</i>{% endif %}
+    </div>
+</div>
+
+{% if not is_virtual %}
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Resources requested
+    </div>
+    <div class="col">
+        {% if session.resources %}<ul>{% for resource in session.resources %}<li>{{ resource.desc }}</li>{% endfor %}</ul>{% else %}<i>None so far</i>{% endif %}
+    </div>
+</div>
+{% endif %}
+
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Participants who must be present
+    </div>
+    <div class="col">
+        {% if session.bethere %}<ul>{% for person in session.bethere %}<li>{{ person }}</li>{% endfor %}</ul>{% else %}<i>None</i>{% endif %}
+    </div>
+</div>
+
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Can not meet on
+    </div>
+    <div class="col">
+        {% if session.timeranges_display %}{{ session.timeranges_display|join:', ' }}{% else %}No constraints{% endif %}
+    </div>
+</div>
+
+{% if not is_virtual %}
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Adjacent with WG
+    </div>
+    <div class="col">
+        {{ session.adjacent_with_wg|default:'No preference' }}
+    </div>
+</div>
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Joint session
+    </div>
+    <div class="col">
+        {% if session.joint_with_groups %}
+            {{ session.joint_for_session_display }} with: {{ session.joint_with_groups }}
+        {% else %}
+            Not a joint session
+        {% endif %}
+    </div>
+</div>
+{% endif %}
+
+<div class="row mb-3">
+    <div class="col-md-2 fw-bold">
+        Special Requests
+    </div>
+    <div class="col">
+        {{ session.comments }}    
+    </div>
+</div>
+
+{% if form and form.notifications_optional %}
+    <div class="row mb-3">
+        <div class="col-md-2 fw-bold">
+            {{ form.send_notifications.label}}
+        </div>
+        <div class="col">
+            {% if form.cleaned_data.send_notifications %}Yes{% else %}No{% endif %}    
+        </div>
+    </div>
+{% endif %}
diff --git a/package.json b/package.json
index e3e89288e7..e2e6fd7dab 100644
--- a/package.json
+++ b/package.json
@@ -118,6 +118,7 @@
         "ietf/static/js/complete-review.js",
         "ietf/static/js/create_timeslot.js",
         "ietf/static/js/create_timeslot.js",
+        "ietf/static/js/custom_striped.js",
         "ietf/static/js/d3.js",
         "ietf/static/js/datepicker.js",
         "ietf/static/js/doc-search.js",
@@ -148,6 +149,8 @@
         "ietf/static/js/password_strength.js",
         "ietf/static/js/select2.js",
         "ietf/static/js/session_details_form.js",
+        "ietf/static/js/session_form.js",
+        "ietf/static/js/session_request.js",
         "ietf/static/js/sortable.js",
         "ietf/static/js/stats.js",
         "ietf/static/js/status-change-edit-relations.js",
@@ -208,8 +211,6 @@
         "ietf/secr/static/images/tooltag-arrowright.webp",
         "ietf/secr/static/images/tooltag-arrowright_over.webp",
         "ietf/secr/static/js/dynamic_inlines.js",
-        "ietf/secr/static/js/session_form.js",
-        "ietf/secr/static/js/sessions.js",
         "ietf/secr/static/js/utils.js"
       ]
     }

From 4961f376756de40ca1fe1d2db6a4ec7ff32b92a9 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 14 Oct 2025 14:51:43 -0500
Subject: [PATCH 470/656] feat: preview ballot email before save (#9646)
 (#9721)

* feat: skeleton for modal email preview

* fix: rudimentary transfer of the comment and discuss text

* fix: put. the buttons. back.

* fix: transfer of the data on the current form to the modal

* fix: pull cc_select and additional_cc foward in the workflow UI

* fix: refactor where ballot email is sent

* fix: refactor build_position_email

* chore: remove abandoned imports

* chore: remove abandoned template
---
 ietf/doc/tests_ballot.py                      | 103 +++-----
 ietf/doc/tests_draft.py                       |  77 +++++-
 ietf/doc/tests_irsg_ballot.py                 |  63 +++--
 ietf/doc/tests_rsab_ballot.py                 |  51 +---
 ietf/doc/urls.py                              |   3 +-
 ietf/doc/views_ballot.py                      | 247 ++++++++++--------
 ietf/mailtrigger/forms.py                     |   1 +
 ietf/templates/doc/ballot/edit_position.html  |  89 ++++++-
 .../doc/ballot/send_ballot_comment.html       |  44 ----
 9 files changed, 393 insertions(+), 285 deletions(-)
 delete mode 100644 ietf/templates/doc/ballot/send_ballot_comment.html

diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py
index 810ee598f6..8420e411e2 100644
--- a/ietf/doc/tests_ballot.py
+++ b/ietf/doc/tests_ballot.py
@@ -25,7 +25,6 @@
 from ietf.group.models import Group, Role
 from ietf.group.factories import GroupFactory, RoleFactory, ReviewTeamFactory
 from ietf.ipr.factories import HolderIprDisclosureFactory
-from ietf.name.models import BallotPositionName
 from ietf.iesg.models import TelechatDate
 from ietf.person.models import Person
 from ietf.person.factories import PersonFactory, PersonalApiKeyFactory
@@ -37,9 +36,18 @@
 
 
 class EditPositionTests(TestCase):
+
+    # N.B. This test needs to be rewritten to exercise all types of ballots (iesg, irsg, rsab)
+    # and test against the output of the mailtriggers instead of looking for hardcoded values
+    # in the To and CC results. See #7864
     def test_edit_position(self):
         ad = Person.objects.get(user__username="ad")
-        draft = IndividualDraftFactory(ad=ad,stream_id='ietf')
+        draft = WgDraftFactory(
+            ad=ad,
+            stream_id="ietf",
+            notify="somebody@example.com",
+            group__acronym="mars",
+        )
         ballot = create_ballot_if_not_open(None, draft, ad, 'approve')
         url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name,
                                                           ballot_id=ballot.pk))
@@ -55,11 +63,20 @@ def test_edit_position(self):
         self.assertEqual(len(q('form textarea[name=comment]')), 1)
 
         # vote
+        empty_outbox()
         events_before = draft.docevent_set.count()
-        
-        r = self.client.post(url, dict(position="discuss",
-                                       discuss=" This is a discussion test. \n ",
-                                       comment=" This is a test. \n "))
+
+        r = self.client.post(
+            url,
+            dict(
+                position="discuss",
+                discuss=" This is a discussion test. \n ",
+                comment=" This is a test. \n ",
+                additional_cc="test298347@example.com",
+                cc_choices=["doc_notify", "doc_group_chairs"],
+                send_mail=1,
+            ),
+        )
         self.assertEqual(r.status_code, 302)
 
         pos = draft.latest_event(BallotPositionDocEvent, balloter=ad)
@@ -70,6 +87,22 @@ def test_edit_position(self):
         self.assertTrue(pos.comment_time != None)
         self.assertTrue("New position" in pos.desc)
         self.assertEqual(draft.docevent_set.count(), events_before + 3)
+        self.assertEqual(len(outbox),1)
+        m = outbox[0]
+        self.assertTrue("COMMENT" in m['Subject'])
+        self.assertTrue("DISCUSS" in m['Subject'])
+        self.assertTrue(draft.name in m['Subject'])
+        self.assertTrue("This is a discussion test." in str(m))
+        self.assertTrue("This is a test" in str(m))
+        self.assertTrue("iesg@" in m['To'])
+        # cc_choice doc_group_chairs
+        self.assertTrue("mars-chairs@" in m['Cc'])
+        # cc_choice doc_notify
+        self.assertTrue("somebody@example.com" in m['Cc'])
+        # cc_choice doc_group_email_list was not selected
+        self.assertFalse(draft.group.list_email in m['Cc'])
+        # extra-cc    
+        self.assertTrue("test298347@example.com" in m['Cc'])        
 
         # recast vote
         events_before = draft.docevent_set.count()
@@ -230,64 +263,6 @@ def test_cannot_edit_position_as_pre_ad(self):
         r = self.client.post(url, dict(position="discuss", discuss="Test discuss text"))
         self.assertEqual(r.status_code, 403)
         
-    # N.B. This test needs to be rewritten to exercise all types of ballots (iesg, irsg, rsab)
-    # and test against the output of the mailtriggers instead of looking for hardcoded values
-    # in the To and CC results. See #7864
-    def test_send_ballot_comment(self):
-        ad = Person.objects.get(user__username="ad")
-        draft = WgDraftFactory(ad=ad,group__acronym='mars')
-        draft.notify = "somebody@example.com"
-        draft.save_with_history([DocEvent.objects.create(doc=draft, rev=draft.rev, type="changed_document", by=Person.objects.get(user__username="secretary"), desc="Test")])
-
-        ballot = create_ballot_if_not_open(None, draft, ad, 'approve')
-
-        BallotPositionDocEvent.objects.create(
-            doc=draft, rev=draft.rev, type="changed_ballot_position",
-            by=ad, balloter=ad, ballot=ballot, pos=BallotPositionName.objects.get(slug="discuss"),
-            discuss="This draft seems to be lacking a clearer title?",
-            discuss_time=timezone.now(),
-            comment="Test!",
-            comment_time=timezone.now())
-        
-        url = urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=draft.name,
-                                                                ballot_id=ballot.pk))
-        login_testing_unauthorized(self, "ad", url)
-
-        # normal get
-        r = self.client.get(url)
-        self.assertEqual(r.status_code, 200)
-        q = PyQuery(r.content)
-        self.assertTrue(len(q('form input[name="extra_cc"]')) > 0)
-
-        # send
-        mailbox_before = len(outbox)
-
-        r = self.client.post(url, dict(extra_cc="test298347@example.com", cc_choices=['doc_notify','doc_group_chairs']))
-        self.assertEqual(r.status_code, 302)
-
-        self.assertEqual(len(outbox), mailbox_before + 1)
-        m = outbox[-1]
-        self.assertTrue("COMMENT" in m['Subject'])
-        self.assertTrue("DISCUSS" in m['Subject'])
-        self.assertTrue(draft.name in m['Subject'])
-        self.assertTrue("clearer title" in str(m))
-        self.assertTrue("Test!" in str(m))
-        self.assertTrue("iesg@" in m['To'])
-        # cc_choice doc_group_chairs
-        self.assertTrue("mars-chairs@" in m['Cc'])
-        # cc_choice doc_notify
-        self.assertTrue("somebody@example.com" in m['Cc'])
-        # cc_choice doc_group_email_list was not selected
-        self.assertFalse(draft.group.list_email in m['Cc'])
-        # extra-cc    
-        self.assertTrue("test298347@example.com" in m['Cc'])
-
-        r = self.client.post(url, dict(cc=""))
-        self.assertEqual(r.status_code, 302)
-        self.assertEqual(len(outbox), mailbox_before + 2)
-        m = outbox[-1]
-        self.assertTrue("iesg@" in m['To'])
-        self.assertFalse(m['Cc'] and draft.group.list_email in m['Cc'])
 
 
 class BallotWriteupsTests(TestCase):
diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py
index ab33acebe6..4d262c5a2f 100644
--- a/ietf/doc/tests_draft.py
+++ b/ietf/doc/tests_draft.py
@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 
+import json
 import os
 import datetime
 import io
@@ -11,7 +12,7 @@
 from pathlib import Path
 from pyquery import PyQuery
 
-from django.db.models import Q
+from django.db.models import Max, Q
 from django.urls import reverse as urlreverse
 from django.conf import settings
 from django.utils import timezone
@@ -2391,3 +2392,77 @@ def test_editorial_metadata(self):
         top_level_metadata_headings = q("tbody>tr>th:first-child").text()
         self.assertNotIn("IESG", top_level_metadata_headings)
         self.assertNotIn("IANA", top_level_metadata_headings)
+
+class BallotEmailAjaxTests(TestCase):
+    def test_ajax_build_position_email(self):
+        def _post_json(self, url, json_to_post):
+            r = self.client.post(
+                url, json.dumps(json_to_post), content_type="application/json"
+            )
+            self.assertEqual(r.status_code, 200)
+            return json.loads(r.content)
+
+        doc = WgDraftFactory()
+        ad = RoleFactory(
+            name_id="ad", group=doc.group, person__name="Some Areadirector"
+        ).person
+        url = urlreverse("ietf.doc.views_ballot.ajax_build_position_email")
+        login_testing_unauthorized(self, "secretary", url)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 405)
+        response = _post_json(self, url, {})
+        self.assertFalse(response["success"])
+        self.assertEqual(response["errors"], ["post_data not provided"])
+        response = _post_json(self, url, {"dictis": "not empty"})
+        self.assertFalse(response["success"])
+        self.assertEqual(response["errors"], ["post_data not provided"])
+        response = _post_json(self, url, {"post_data": {}})
+        self.assertFalse(response["success"])
+        self.assertEqual(len(response["errors"]), 7)
+        response = _post_json(
+            self,
+            url,
+            {
+                "post_data": {
+                    "discuss": "aaaaaa",
+                    "comment": "bbbbbb",
+                    "position": "discuss",
+                    "balloter": Person.objects.aggregate(maxpk=Max("pk") + 1)["maxpk"],
+                    "docname": "this-draft-does-not-exist",
+                    "cc_choices": ["doc_group_mail_list"],
+                    "additional_cc": "foo@example.com",
+                }
+            },
+        )
+        self.assertFalse(response["success"])
+        self.assertEqual(
+            response["errors"],
+            ["No person found matching balloter", "No document found matching docname"],
+        )
+        response = _post_json(
+            self,
+            url,
+            {
+                "post_data": {
+                    "discuss": "aaaaaa",
+                    "comment": "bbbbbb",
+                    "position": "discuss",
+                    "balloter": ad.pk,
+                    "docname": doc.name,
+                    "cc_choices": ["doc_group_mail_list"],
+                    "additional_cc": "foo@example.com",
+                }
+            },
+        )
+        self.assertTrue(response["success"])
+        for snippet in [
+            "aaaaaa",
+            "bbbbbb",
+            "DISCUSS",
+            ad.plain_name(),
+            doc.name,
+            doc.group.list_email,
+            "foo@example.com",
+        ]:
+            self.assertIn(snippet, response["text"])
+
diff --git a/ietf/doc/tests_irsg_ballot.py b/ietf/doc/tests_irsg_ballot.py
index aa62d8aaf9..d96cf9dbef 100644
--- a/ietf/doc/tests_irsg_ballot.py
+++ b/ietf/doc/tests_irsg_ballot.py
@@ -355,28 +355,35 @@ def test_issue_ballot(self):
     def test_take_and_email_position(self):
         draft = RgDraftFactory()
         ballot = IRSGBallotDocEventFactory(doc=draft)
-        url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) + self.balloter
+        url = (
+            urlreverse(
+                "ietf.doc.views_ballot.edit_position",
+                kwargs=dict(name=draft.name, ballot_id=ballot.pk),
+            )
+            + self.balloter
+        )
         empty_outbox()
 
         login_testing_unauthorized(self, self.username, url)
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
 
-        r = self.client.post(url, dict(position='yes', comment='oib239sb', send_mail='Save and send email'))
+        empty_outbox()
+        r = self.client.post(
+            url,
+            dict(
+                position="yes",
+                comment="oib239sb",
+                send_mail="Save and send email",
+                cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"],
+            ),
+        )
         self.assertEqual(r.status_code, 302)
         e = draft.latest_event(BallotPositionDocEvent)
-        self.assertEqual(e.pos.slug,'yes')
-        self.assertEqual(e.comment, 'oib239sb')
-
-        url = urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) + self.balloter
-
-        r = self.client.get(url)
-        self.assertEqual(r.status_code, 200)
-
-        r = self.client.post(url, dict(cc_choices=['doc_authors','doc_group_chairs','doc_group_mail_list'], body="Stuff"))
-        self.assertEqual(r.status_code, 302)
-        self.assertEqual(len(outbox),1)
-        self.assertNotIn('discuss-criteria', get_payload_text(outbox[0]))
+        self.assertEqual(e.pos.slug, "yes")
+        self.assertEqual(e.comment, "oib239sb")
+        self.assertEqual(len(outbox), 1)
+        self.assertNotIn("discuss-criteria", get_payload_text(outbox[0]))
 
     def test_close_ballot(self):
         draft = RgDraftFactory()
@@ -482,27 +489,31 @@ def test_cant_take_position_on_iesg_ballot(self):
     def test_take_and_email_position(self):
         draft = RgDraftFactory()
         ballot = IRSGBallotDocEventFactory(doc=draft)
-        url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name, ballot_id=ballot.pk))
+        url = urlreverse(
+            "ietf.doc.views_ballot.edit_position",
+            kwargs=dict(name=draft.name, ballot_id=ballot.pk),
+        )
         empty_outbox()
 
         login_testing_unauthorized(self, self.username, url)
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
 
-        r = self.client.post(url, dict(position='yes', comment='oib239sb', send_mail='Save and send email'))
+        r = self.client.post(
+            url,
+            dict(
+                position="yes",
+                comment="oib239sb",
+                send_mail="Save and send email",
+                cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"],
+            ),
+        )
         self.assertEqual(r.status_code, 302)
         e = draft.latest_event(BallotPositionDocEvent)
-        self.assertEqual(e.pos.slug,'yes')
-        self.assertEqual(e.comment, 'oib239sb')
-
-        url = urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=draft.name, ballot_id=ballot.pk))
-
-        r = self.client.get(url)
-        self.assertEqual(r.status_code, 200)
-
-        r = self.client.post(url, dict(cc_choices=['doc_authors','doc_group_chairs','doc_group_mail_list'], body="Stuff"))
+        self.assertEqual(e.pos.slug, "yes")
+        self.assertEqual(e.comment, "oib239sb")
         self.assertEqual(r.status_code, 302)
-        self.assertEqual(len(outbox),1)
+        self.assertEqual(len(outbox), 1)
 
 class IESGMemberTests(TestCase):
 
diff --git a/ietf/doc/tests_rsab_ballot.py b/ietf/doc/tests_rsab_ballot.py
index 028f548232..9086106ba9 100644
--- a/ietf/doc/tests_rsab_ballot.py
+++ b/ietf/doc/tests_rsab_ballot.py
@@ -333,34 +333,19 @@ def test_take_and_email_position(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
 
-        r = self.client.post(
-            url,
-            dict(position="yes", comment="oib239sb", send_mail="Save and send email"),
-        )
-        self.assertEqual(r.status_code, 302)
-        e = draft.latest_event(BallotPositionDocEvent)
-        self.assertEqual(e.pos.slug, "yes")
-        self.assertEqual(e.comment, "oib239sb")
-
-        url = (
-            urlreverse(
-                "ietf.doc.views_ballot.send_ballot_comment",
-                kwargs=dict(name=draft.name, ballot_id=ballot.pk),
-            )
-            + self.balloter
-        )
-
-        r = self.client.get(url)
-        self.assertEqual(r.status_code, 200)
-
         r = self.client.post(
             url,
             dict(
+                position="yes",
+                comment="oib239sb",
+                send_mail="Save and send email",
                 cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"],
-                body="Stuff",
             ),
         )
         self.assertEqual(r.status_code, 302)
+        e = draft.latest_event(BallotPositionDocEvent)
+        self.assertEqual(e.pos.slug, "yes")
+        self.assertEqual(e.comment, "oib239sb")
         self.assertEqual(len(outbox), 1)
         self.assertNotIn("discuss-criteria", get_payload_text(outbox[0]))
 
@@ -532,31 +517,19 @@ def test_take_and_email_position(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
 
-        r = self.client.post(
-            url,
-            dict(position="yes", comment="oib239sb", send_mail="Save and send email"),
-        )
-        self.assertEqual(r.status_code, 302)
-        e = draft.latest_event(BallotPositionDocEvent)
-        self.assertEqual(e.pos.slug, "yes")
-        self.assertEqual(e.comment, "oib239sb")
-
-        url = urlreverse(
-            "ietf.doc.views_ballot.send_ballot_comment",
-            kwargs=dict(name=draft.name, ballot_id=ballot.pk),
-        )
-
-        r = self.client.get(url)
-        self.assertEqual(r.status_code, 200)
-
         r = self.client.post(
             url,
             dict(
+                position="yes",
+                comment="oib239sb",
+                send_mail="Save and send email",
                 cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"],
-                body="Stuff",
             ),
         )
         self.assertEqual(r.status_code, 302)
+        e = draft.latest_event(BallotPositionDocEvent)
+        self.assertEqual(e.pos.slug, "yes")
+        self.assertEqual(e.comment, "oib239sb")
         self.assertEqual(len(outbox), 1)
 
 
diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py
index 7b444782d7..8e9c0569e2 100644
--- a/ietf/doc/urls.py
+++ b/ietf/doc/urls.py
@@ -93,6 +93,8 @@
     url(r'^ballots/irsg/$', views_ballot.irsg_ballot_status),
     url(r'^ballots/rsab/$', views_ballot.rsab_ballot_status),
 
+    url(r'^build-position-email/$', views_ballot.ajax_build_position_email),
+
     url(r'^(?P<type_id>(bcp|std|fyi))/?$', views_search.index_subseries),
 
     url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main),
@@ -111,7 +113,6 @@
     url(r'^%(name)s/ballot/rsab/$' % settings.URL_REGEXPS, views_doc.document_rsab_ballot),
     url(r'^%(name)s/ballot/(?P<ballot_id>[0-9]+)/$' % settings.URL_REGEXPS, views_doc.document_ballot),
     url(r'^%(name)s/ballot/(?P<ballot_id>[0-9]+)/position/$' % settings.URL_REGEXPS, views_ballot.edit_position),
-    url(r'^%(name)s/ballot/(?P<ballot_id>[0-9]+)/emailposition/$' % settings.URL_REGEXPS, views_ballot.send_ballot_comment),
     url(r'^%(name)s/(?:%(rev)s/)?doc.json$' % settings.URL_REGEXPS, views_doc.document_json),
     url(r'^%(name)s/ballotpopup/(?P<ballot_id>[0-9]+)/$' % settings.URL_REGEXPS, views_doc.ballot_popup),
     url(r'^(?P<name>[A-Za-z0-9._+-]+)/reviewrequest/', include("ietf.doc.urls_review")),
diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py
index 0ba340890d..03cf01a4a1 100644
--- a/ietf/doc/views_ballot.py
+++ b/ietf/doc/views_ballot.py
@@ -4,18 +4,18 @@
 # Directors and Secretariat
 
 
-import datetime, json
+import datetime
+import json
 
 from django import forms
 from django.conf import settings
-from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseBadRequest
+from django.http import HttpResponse, HttpResponseNotAllowed, HttpResponseRedirect, Http404, HttpResponseBadRequest
 from django.shortcuts import render, get_object_or_404, redirect
 from django.template.defaultfilters import striptags
 from django.template.loader import render_to_string
 from django.urls import reverse as urlreverse
 from django.views.decorators.csrf import csrf_exempt
 from django.utils.html import escape
-from urllib.parse import urlencode as urllib_urlencode
 
 import debug                            # pyflakes:ignore
 
@@ -34,14 +34,15 @@
 from ietf.doc.templatetags.ietf_filters import can_ballot
 from ietf.iesg.models import TelechatDate
 from ietf.ietfauth.utils import has_role, role_required, is_authorized_in_doc_stream
+from ietf.mailtrigger.models import Recipient
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.mailtrigger.forms import CcSelectForm
 from ietf.message.utils import infer_message
 from ietf.name.models import BallotPositionName, DocTypeName
 from ietf.person.models import Person
-from ietf.utils.fields import ModelMultipleChoiceField
+from ietf.utils.fields import ModelMultipleChoiceField, MultiEmailField
 from ietf.utils.http import validate_return_to_path
-from ietf.utils.mail import send_mail_text, send_mail_preformatted
+from ietf.utils.mail import decode_header_value, send_mail_text, send_mail_preformatted
 from ietf.utils.decorators import require_api_key
 from ietf.utils.response import permission_denied
 from ietf.utils.timezone import date_today, datetime_from_date, DEADLINE_TZINFO
@@ -179,6 +180,9 @@ def save_position(form, doc, ballot, balloter, login=None, send_email=False):
 
     return pos
 
+class AdditionalCCForm(forms.Form):
+    additional_cc = MultiEmailField(required=False)
+
 @role_required("Area Director", "Secretariat", "IRSG Member", "RSAB Member")
 def edit_position(request, name, ballot_id):
     """Vote and edit discuss and comment on document"""
@@ -199,50 +203,67 @@ def edit_position(request, name, ballot_id):
             raise Http404
         balloter = get_object_or_404(Person, pk=balloter_id)
 
+    if doc.stream_id == 'irtf':
+        mailtrigger_slug='irsg_ballot_saved'
+    elif doc.stream_id == 'editorial':
+        mailtrigger_slug='rsab_ballot_saved'
+    else:
+        mailtrigger_slug='iesg_ballot_saved'
+
     if request.method == 'POST':
         old_pos = None
         if not has_role(request.user, "Secretariat") and not can_ballot(request.user, doc):
             # prevent pre-ADs from taking a position
             permission_denied(request, "Must be an active member (not a pre-AD for example) of the balloting body to take a position")
         
+        if request.POST.get("Defer") and doc.stream.slug != "irtf":
+            return redirect('ietf.doc.views_ballot.defer_ballot', name=doc)
+        elif request.POST.get("Undefer") and doc.stream.slug != "irtf":
+            return redirect('ietf.doc.views_ballot.undefer_ballot', name=doc)
+        
         form = EditPositionForm(request.POST, ballot_type=ballot.ballot_type)
-        if form.is_valid():
+        cc_select_form = CcSelectForm(data=request.POST,mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc})
+        additional_cc_form = AdditionalCCForm(request.POST)
+        if form.is_valid() and cc_select_form.is_valid() and additional_cc_form.is_valid():
             send_mail = True if request.POST.get("send_mail") else False
-            save_position(form, doc, ballot, balloter, login, send_mail)
-
+            pos = save_position(form, doc, ballot, balloter, login, send_mail)
             if send_mail:
-                query = {}
-                if request.GET.get('balloter'):
-                    query['balloter'] = request.GET.get('balloter')
-                if request.GET.get('ballot_edit_return_point'):
-                    query['ballot_edit_return_point'] = request.GET.get('ballot_edit_return_point')
-                qstr = ""
-                if len(query) > 0:
-                    qstr = "?" + urllib_urlencode(query, safe='/')
-                return HttpResponseRedirect(urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=doc.name, ballot_id=ballot_id)) + qstr)
-            elif request.POST.get("Defer") and doc.stream.slug != "irtf":
-                return redirect('ietf.doc.views_ballot.defer_ballot', name=doc)
-            elif request.POST.get("Undefer") and doc.stream.slug != "irtf":
-                return redirect('ietf.doc.views_ballot.undefer_ballot', name=doc)
-            else:
-                return HttpResponseRedirect(return_to_url)
+                addrs, frm, subject, body = build_position_email(balloter, doc, pos)
+                if doc.stream_id == 'irtf':
+                    mailtrigger_slug='irsg_ballot_saved'
+                elif doc.stream_id == 'editorial':
+                    mailtrigger_slug='rsab_ballot_saved'
+                else:
+                    mailtrigger_slug='iesg_ballot_saved'
+                cc = []
+                cc.extend(cc_select_form.get_selected_addresses())
+                extra_cc = additional_cc_form.cleaned_data["additional_cc"]
+                if extra_cc:
+                    cc.extend(extra_cc)
+                cc_set = set(cc)
+                cc_set.discard("")
+                cc = sorted(list(cc_set))
+                send_mail_text(request, addrs.to, frm, subject, body, cc=", ".join(cc))
+            return redirect(return_to_url)
     else:
         initial = {}
         old_pos = doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position", balloter=balloter, ballot=ballot)
         if old_pos:
             initial['position'] = old_pos.pos.slug
             initial['discuss'] = old_pos.discuss
-            initial['comment'] = old_pos.comment
-            
+            initial['comment'] = old_pos.comment      
         form = EditPositionForm(initial=initial, ballot_type=ballot.ballot_type)
+        cc_select_form = CcSelectForm(mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc})
+        additional_cc_form = AdditionalCCForm()
 
     blocking_positions = dict((p.pk, p.name) for p in form.fields["position"].queryset.all() if p.blocking)
-
     ballot_deferred = doc.active_defer_event()
 
     return render(request, 'doc/ballot/edit_position.html',
                               dict(doc=doc,
                                    form=form,
+                                   cc_select_form=cc_select_form,
+                                   additional_cc_form=additional_cc_form,
                                    balloter=balloter,
                                    return_to_url=return_to_url,
                                    old_pos=old_pos,
@@ -301,21 +322,98 @@ def err(code, text):
     )
 
 
-def build_position_email(balloter, doc, pos):
+@role_required("Area Director", "Secretariat")
+@csrf_exempt
+def ajax_build_position_email(request):
+    if request.method != "POST":
+        return HttpResponseNotAllowed(["POST"])
+    errors = list()
+    try:
+        json_body = json.loads(request.body)
+    except json.decoder.JSONDecodeError:
+        errors.append("Post body is not valid json")
+    if len(errors) == 0:
+        post_data = json_body.get("post_data")
+        if post_data is None:
+            errors.append("post_data not provided")
+        else:
+            for key in [
+                "discuss",
+                "comment",
+                "position",
+                "balloter",
+                "docname",
+                "cc_choices",
+                "additional_cc",
+            ]:
+                if key not in post_data:
+                    errors.append(f"{key} not found in post_data")
+    if len(errors) == 0:
+        person = Person.objects.filter(pk=post_data.get("balloter")).first()
+        if person is None:
+            errors.append("No person found matching balloter")
+        doc = Document.objects.filter(name=post_data.get("docname")).first()
+        if doc is None:
+            errors.append("No document found matching docname")
+    if len(errors) > 0:
+        response = {
+            "success": False,
+            "errors": errors,
+        }
+    else:
+        wanted = dict()  # consider named tuple instead
+        wanted["discuss"] = post_data.get("discuss")
+        wanted["comment"] = post_data.get("comment")
+        wanted["position_name"] = post_data.get("position")
+        wanted["balloter"] = person
+        wanted["doc"] = doc
+        addrs, frm, subject, body = build_position_email_from_dict(wanted)
+
+        recipient_slugs = post_data.get("cc_choices")
+        # Consider refactoring gather_address_lists so this isn't duplicated from there
+        cc_addrs = set()
+        for r in Recipient.objects.filter(slug__in=recipient_slugs):
+            cc_addrs.update(r.gather(doc=doc))
+        additional_cc = post_data.get("additional_cc")
+        for addr in additional_cc.split(","):
+            cc_addrs.add(addr.strip())
+        cc_addrs.discard("")
+        cc_addrs = sorted(list(cc_addrs))
+
+        response_text = "\n".join(
+            [
+                f"From: {decode_header_value(frm)}",
+                f"To: {', '.join([decode_header_value(addr) for addr in addrs.to])}",
+                f"Cc: {', '.join([decode_header_value(addr) for addr in cc_addrs])}",
+                f"Subject: {subject}",
+                "",
+                body,
+            ]
+        )
+
+        response = {
+            "success": True,
+            "text": response_text,
+        }
+    return HttpResponse(json.dumps(response), content_type="application/json")
+
+def build_position_email_from_dict(pos_dict):
+    doc = pos_dict["doc"]
     subj = []
     d = ""
     blocking_name = "DISCUSS"
-    if pos.pos.blocking and pos.discuss:
-        d = pos.discuss
-        blocking_name = pos.pos.name.upper()
+    pos_name = BallotPositionName.objects.filter(slug=pos_dict["position_name"]).first()
+    if pos_name.blocking and pos_dict.get("discuss"):
+        d = pos_dict.get("discuss")
+        blocking_name = pos_name.name.upper()
         subj.append(blocking_name)
     c = ""
-    if pos.comment:
-        c = pos.comment
+    if pos_dict.get("comment"):
+        c = pos_dict.get("comment")
         subj.append("COMMENT")
-
+    balloter = pos_dict.get("balloter")
     balloter_name_genitive = balloter.plain_name() + "'" if balloter.plain_name().endswith('s') else balloter.plain_name() + "'s"
-    subject = "%s %s on %s" % (balloter_name_genitive, pos.pos.name if pos.pos else "No Position", doc.name + "-" + doc.rev)
+    subject = "%s %s on %s" % (balloter_name_genitive, pos_name.name if pos_name else "No Position", doc.name + "-" + doc.rev)
     if subj:
         subject += ": (with %s)" % " and ".join(subj)
 
@@ -324,7 +422,7 @@ def build_position_email(balloter, doc, pos):
                                  comment=c,
                                  balloter=balloter.plain_name(),
                                  doc=doc,
-                                 pos=pos.pos,
+                                 pos=pos_name,
                                  blocking_name=blocking_name,
                                  settings=settings))
     frm = balloter.role_email("ad").formatted_email()
@@ -338,79 +436,16 @@ def build_position_email(balloter, doc, pos):
 
     return addrs, frm, subject, body
 
-@role_required('Area Director','Secretariat','IRSG Member', 'RSAB Member')
-def send_ballot_comment(request, name, ballot_id):
-    """Email document ballot position discuss/comment for Area Director."""
-    doc = get_object_or_404(Document, name=name)
-    ballot = get_object_or_404(BallotDocEvent, type="created_ballot", pk=ballot_id, doc=doc)
 
-    if not has_role(request.user, 'Secretariat'):
-        if any([
-            doc.stream_id == 'ietf' and not has_role(request.user, 'Area Director'),
-            doc.stream_id == 'irtf' and not has_role(request.user, 'IRSG Member'),
-            doc.stream_id == 'editorial' and not has_role(request.user, 'RSAB Member'),
-        ]):
-            raise Http404
-
-    balloter = request.user.person
-
-    try:
-        return_to_url = parse_ballot_edit_return_point(request.GET.get('ballot_edit_return_point'), doc.name, ballot_id)
-    except ValueError:
-        return HttpResponseBadRequest('ballot_edit_return_point is invalid')
-    
-    if 'HTTP_REFERER' in request.META:
-        back_url = request.META['HTTP_REFERER']
-    else:
-        back_url = urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name, ballot_id=ballot_id))
-
-    # if we're in the Secretariat, we can select a balloter (such as an AD) to act as stand-in for
-    if has_role(request.user, "Secretariat"):
-        balloter_id = request.GET.get('balloter')
-        if not balloter_id:
-            raise Http404
-        balloter = get_object_or_404(Person, pk=balloter_id)
-
-    pos = doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position", balloter=balloter, ballot=ballot)
-    if not pos:
-        raise Http404
-
-    addrs, frm, subject, body = build_position_email(balloter, doc, pos)
-
-    if doc.stream_id == 'irtf':
-        mailtrigger_slug='irsg_ballot_saved'
-    elif doc.stream_id == 'editorial':
-        mailtrigger_slug='rsab_ballot_saved'
-    else:
-        mailtrigger_slug='iesg_ballot_saved'
-        
-    if request.method == 'POST':
-        cc = []
-        cc_select_form = CcSelectForm(data=request.POST,mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc})
-        if cc_select_form.is_valid():
-            cc.extend(cc_select_form.get_selected_addresses())
-        extra_cc = [x.strip() for x in request.POST.get("extra_cc","").split(',') if x.strip()]
-        if extra_cc:
-            cc.extend(extra_cc)
-
-        send_mail_text(request, addrs.to, frm, subject, body, cc=", ".join(cc))
-            
-        return HttpResponseRedirect(return_to_url)
-
-    else: 
+def build_position_email(balloter, doc, pos):
 
-        cc_select_form = CcSelectForm(mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc})
-  
-        return render(request, 'doc/ballot/send_ballot_comment.html',
-                      dict(doc=doc,
-                          subject=subject,
-                          body=body,
-                          frm=frm,
-                          to=addrs.as_strings().to,
-                          balloter=balloter,
-                          back_url=back_url,
-                          cc_select_form = cc_select_form,
-                      ))
+    pos_dict=dict()
+    pos_dict["doc"]=doc
+    pos_dict["position_name"]=pos.pos.slug
+    pos_dict["discuss"]=pos.discuss
+    pos_dict["comment"]=pos.comment
+    pos_dict["balloter"]=balloter
+    return build_position_email_from_dict(pos_dict)
 
 @role_required('Area Director','Secretariat')
 def clear_ballot(request, name, ballot_type_slug):
diff --git a/ietf/mailtrigger/forms.py b/ietf/mailtrigger/forms.py
index 366c429d8c..8d13c5edf3 100644
--- a/ietf/mailtrigger/forms.py
+++ b/ietf/mailtrigger/forms.py
@@ -11,6 +11,7 @@
 class CcSelectForm(forms.Form):
     expansions = dict()                 # type: Dict[str, List[str]]
     cc_choices = forms.MultipleChoiceField(
+                   required=False,
                    label='Cc',
                    choices=[],
                    widget=forms.CheckboxSelectMultiple(),
diff --git a/ietf/templates/doc/ballot/edit_position.html b/ietf/templates/doc/ballot/edit_position.html
index 293c186112..b57e9a3652 100644
--- a/ietf/templates/doc/ballot/edit_position.html
+++ b/ietf/templates/doc/ballot/edit_position.html
@@ -20,24 +20,48 @@ <h1>
             Ballot deferred by {{ ballot_deferred.by }} on {{ ballot_deferred.time|date:"Y-m-d" }}.
         </p>
     {% endif %}
+    <div class="d-none" id="id_docname" data-value="{{doc.name}}"></div>
+    <div class="d-none" id="id_balloter" data-value="{{balloter.pk}}"></div>
+    {% if form.errors or cc_select_form.errors or additional_cc_form.errors %}
+        <div class="alert alert-danger my-3">
+                There were errors in the submitted form -- see below. Please correct these and resubmit.
+        </div>
+        {% if form.errors %}
+            <div>Position entry</div>
+            {% bootstrap_form_errors form %}
+        {% endif %}
+        {% if cc_select_form.errors %}
+            <div>CC selection</div>
+            {% bootstrap_form_errors cc_select_form %}
+        {% endif %}
+        {% if additional_cc_form.errors %}
+            <div>Additional Cc Addresses</div>
+            {% bootstrap_form_errors additional_cc_form %}
+        {% endif %}
+    {% endif %}
     <form method="post" id="ballot-form" class="my-3">
         {% csrf_token %}
         {% for field in form %}
             {% if field.name == "discuss" %}<div id="div_id_discuss">{% endif %}
             {% bootstrap_field field %}
             {% if field.name == "discuss" and old_pos and old_pos.discuss_time %}
-                <div class="form-text mt-n3 mb-3">Last edited {{ old_pos.discuss_time }}</div>
+                <div class="form-text mt-n3 mb-3">Last saved {{ old_pos.discuss_time }}</div>
             {% elif field.name == "comment" and old_pos and old_pos.comment_time %}
-                <div class="form-text mt-n3 mb-3">Last edited {{ old_pos.comment_time }}</div>
+                <div class="form-text mt-n3 mb-3">Last saved {{ old_pos.comment_time }}</div>
             {% endif %}
             {% if field.name == "discuss" %}</div>{% endif %}
         {% endfor %}
+        {% bootstrap_form cc_select_form %}
+        {% bootstrap_form additional_cc_form %}
         <div class="mt-3">
+            <button type="button" class="btn btn-primary" data-bs-toggle="modal" data-bs-target="#emailPreview">
+                Preview email
+            </button>
             <button type="submit"
                     class="btn btn-primary"
                     name="send_mail"
                     value="Save and send email">Save &amp; send email</button>
-            <button type="submit" class="btn btn-primary">Save</button>
+            <button type="submit" class="btn btn-warning">Save without sending email</button>
             {% if doc.type_id == "draft" or doc.type_id == "conflrev" %}
                 {% if doc.stream.slug != "irtf" %}
                     {% if ballot_deferred %}
@@ -58,7 +82,24 @@ <h1>
                 Back
             </a>
         </div>
-    </form>
+    </form>  
+    <!-- Modal -->
+    <div class="modal fade" id="emailPreview" tabindex="-1" aria-labelledby="emailPreviewLabel" aria-hidden="true">
+    <div class="modal-dialog modal-xl">
+        <div class="modal-content">
+        <div class="modal-header">
+            <h1 class="modal-title fs-5" id="emailPreviewLabel">This email has not yet been sent</h1>
+            <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+        </div>
+        <pre class="modal-body">
+            Loading
+        </pre>
+        <div class="modal-footer">
+            <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+        </div>
+        </div>
+    </div>
+    </div>
 {% endblock %}
 {% block js %}
     <script>
@@ -78,4 +119,44 @@ <h1>
 
         discussToggle($("input[name=position]:checked").val());
     </script>
+
+    <script>
+        function getCheckedCcValues() {
+            const checkedInputs = document.querySelectorAll('#id_cc_choices input:checked')
+            const values = Array.from(checkedInputs).map(input => input.value)
+            return values
+        }
+        const emailPreview = document.getElementById("emailPreview")
+        if (emailPreview) {
+            emailPreview.addEventListener("show.bs.modal", async (event) => {
+                const discuss = document.getElementById("id_discuss").value
+                const comment = document.getElementById("id_comment").value
+                const position = document.getElementById("id_position").querySelector('input[name="position"]:checked').value
+                const docname = document.getElementById("id_docname").dataset.value
+                const balloter = document.getElementById("id_balloter").dataset.value
+                const cc_choices = getCheckedCcValues()
+                const additional_cc = document.getElementById("id_additional_cc").value
+                const postData = {
+                    discuss: discuss,
+                    comment: comment,
+                    position: position,
+                    docname: docname,
+                    balloter: balloter,
+                    cc_choices: cc_choices,
+                    additional_cc: additional_cc
+                }
+                response = await fetch("/doc/build-position-email/", {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json"
+                    },
+                    body: JSON.stringify({"post_data": postData})
+                })
+                // Update the modal's content.
+                const modalBody = emailPreview.querySelector('.modal-body')
+                responseJson = await response.json()
+                modalBody.textContent = responseJson.text
+            })
+        }
+    </script>
 {% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/doc/ballot/send_ballot_comment.html b/ietf/templates/doc/ballot/send_ballot_comment.html
deleted file mode 100644
index 1c5f521859..0000000000
--- a/ietf/templates/doc/ballot/send_ballot_comment.html
+++ /dev/null
@@ -1,44 +0,0 @@
-{% extends "base.html" %}
-{# Copyright The IETF Trust 2015, All Rights Reserved #}
-{% load origin %}
-{% load django_bootstrap5 %}
-{% load ietf_filters %}
-{% block title %}Send ballot position for {{ balloter }} on {{ doc }}{% endblock %}
-{% block content %}
-    {% origin %}
-    <h1>
-        Send ballot position for {{ balloter }}
-        <br>
-        <small class="text-body-secondary">{{ doc }}</small>
-    </h1>
-    <form class="mt-3" method="post">
-        {% csrf_token %}
-        <div class="mb-3">
-            <label class="form-label" for="from">From</label>
-            <input id="from" class="form-control" type="text" placeholder="{{ frm }}" disabled>
-        </div>
-        <div class="mb-3">
-            <label class="form-label" for="to">To</label>
-            <input id="to" class="form-control" type="text" placeholder="{{ to }}" disabled>
-        </div>
-        {% bootstrap_form cc_select_form %}
-        <div class="mb-3">
-            <label class="form-label" for="additional">Additional Cc Addresses</label>
-            <input id="additional" class="form-control" type="text" name="extra_cc">
-            <div class="form-text">Separate email addresses with commas.</div>
-        </div>
-        <div class="mb-3">
-            <label class="form-label" for="subject">Subject</label>
-            <input id="subject" class="form-control" type="text" placeholder="{{ subject }}" disabled>
-        </div>
-        <div class="mb-3">
-            <p class="form-label">Body</p>
-            <pre class="border p-3">{{ body|maybewordwrap }}</pre>
-        </div>
-        <button type="submit" class="btn btn-danger">Send</button>
-        <a class="btn btn-secondary float-end"
-           href="{% url "ietf.doc.views_doc.document_main" name=doc.name %}">
-            Back
-        </a>
-    </form>
-{% endblock %}

From 8f2feef631acbd8b181a845140647c2c83a9299f Mon Sep 17 00:00:00 2001
From: NGPixel <github@ngpixel.com>
Date: Tue, 14 Oct 2025 18:57:50 -0400
Subject: [PATCH 471/656] ci: update build workflow to deploy to dev

---
 .github/workflows/build.yml               |  71 ++---
 dev/k8s-get-deploy-name/.editorconfig     |   7 +
 dev/k8s-get-deploy-name/.gitignore        |   1 +
 dev/k8s-get-deploy-name/.npmrc            |   3 +
 dev/k8s-get-deploy-name/README.md         |  16 ++
 dev/k8s-get-deploy-name/cli.js            |  22 ++
 dev/k8s-get-deploy-name/package-lock.json | 303 ++++++++++++++++++++++
 dev/k8s-get-deploy-name/package.json      |   8 +
 8 files changed, 396 insertions(+), 35 deletions(-)
 create mode 100644 dev/k8s-get-deploy-name/.editorconfig
 create mode 100644 dev/k8s-get-deploy-name/.gitignore
 create mode 100644 dev/k8s-get-deploy-name/.npmrc
 create mode 100644 dev/k8s-get-deploy-name/README.md
 create mode 100644 dev/k8s-get-deploy-name/cli.js
 create mode 100644 dev/k8s-get-deploy-name/package-lock.json
 create mode 100644 dev/k8s-get-deploy-name/package.json

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8567446cae..15eaba48d1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -16,13 +16,13 @@ on:
           - Skip
           - Staging Only
           - Staging + Prod
-      sandbox:
-        description: 'Deploy to Sandbox'
+      dev:
+        description: 'Deploy to Dev'
         default: true
         required: true
         type: boolean
-      sandboxNoDbRefresh:
-        description: 'Sandbox Disable Daily DB Refresh'
+      devNoDbRefresh:
+        description: 'Dev Disable Daily DB Refresh'
         default: false
         required: true
         type: boolean
@@ -392,44 +392,45 @@ jobs:
                   value: "Failed"
         
   # -----------------------------------------------------------------
-  # SANDBOX
+  # DEV
   # -----------------------------------------------------------------
-  sandbox:
-    name: Deploy to Sandbox
-    if: ${{ !failure() && !cancelled() && github.event.inputs.sandbox == 'true' }}
+  dev:
+    name: Deploy to Dev
+    if: ${{ !failure() && !cancelled() && github.event.inputs.dev == 'true' }}
     needs: [prepare, release]
-    runs-on: [self-hosted, dev-server]
+    runs-on: ubuntu-latest
     environment:
-      name: sandbox
+      name: dev
     env:
       PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
       
     steps:
-    - uses: actions/checkout@v4
-        
-    - name: Download a Release Artifact
-      uses: actions/download-artifact@v4.3.0
-      with:
-        name: release-${{ env.PKG_VERSION }}
-        
-    - name: Deploy to containers
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      run: |
-        echo "Reset production flags in settings.py..."
-        sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = True/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'development'/" ietf/settings.py
-        echo "Install Deploy to Container CLI dependencies..."
-        cd dev/deploy-to-container
-        npm ci
-        cd ../..
-        echo "Start Deploy..."
-        node ./dev/deploy-to-container/cli.js --branch ${{ github.ref_name }} --domain dev.ietf.org --appversion ${{ env.PKG_VERSION }} --commit ${{ github.sha }} --ghrunid ${{ github.run_id }} --nodbrefresh ${{ github.event.inputs.sandboxNoDbRefresh }}
-        
-    - name: Cleanup old docker resources
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      run: |
-        docker image prune -a -f
+      - uses: actions/checkout@v4
+        with:
+          ref: main
+          
+      - name: Get Deploy Name
+        env:
+          DEBIAN_FRONTEND: noninteractive
+        run: |
+          echo "Install Get Deploy Name CLI dependencies..."
+          cd dev/k8s-get-deploy-name
+          npm ci
+          echo "Get Deploy Name..."
+          echo "DEPLOY_NAMESPACE=$(node cli.js --branch ${{ github.ref_name }})" >> "$GITHUB_ENV"
+
+      - name: Deploy to dev
+        uses: the-actions-org/workflow-dispatch@v4
+        with:
+          workflow: deploy-dev.yml
+          repo: ietf-tools/infra-k8s
+          ref: main
+          token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
+          inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}" }'
+          wait-for-completion: true
+          wait-for-completion-timeout: 30m
+          wait-for-completion-interval: 30s
+          display-workflow-run-url: false
 
   # -----------------------------------------------------------------
   # STAGING
diff --git a/dev/k8s-get-deploy-name/.editorconfig b/dev/k8s-get-deploy-name/.editorconfig
new file mode 100644
index 0000000000..fec5c66519
--- /dev/null
+++ b/dev/k8s-get-deploy-name/.editorconfig
@@ -0,0 +1,7 @@
+[*]
+indent_size = 2
+indent_style = space
+charset = utf-8
+trim_trailing_whitespace = false
+end_of_line = lf
+insert_final_newline = true
diff --git a/dev/k8s-get-deploy-name/.gitignore b/dev/k8s-get-deploy-name/.gitignore
new file mode 100644
index 0000000000..07e6e472cc
--- /dev/null
+++ b/dev/k8s-get-deploy-name/.gitignore
@@ -0,0 +1 @@
+/node_modules
diff --git a/dev/k8s-get-deploy-name/.npmrc b/dev/k8s-get-deploy-name/.npmrc
new file mode 100644
index 0000000000..580a68c499
--- /dev/null
+++ b/dev/k8s-get-deploy-name/.npmrc
@@ -0,0 +1,3 @@
+audit = false
+fund = false
+save-exact = true
diff --git a/dev/k8s-get-deploy-name/README.md b/dev/k8s-get-deploy-name/README.md
new file mode 100644
index 0000000000..a6605e4dd2
--- /dev/null
+++ b/dev/k8s-get-deploy-name/README.md
@@ -0,0 +1,16 @@
+# Datatracker Get Deploy Name
+
+This tool process and slugify a git branch into an appropriate subdomain name.
+
+## Usage
+
+1. From the `dev/k8s-get-deploy-name` directory, install the dependencies:
+```sh
+npm install
+```
+2. Run the command: (replacing the `branch` argument)
+```sh
+node /cli.js --branch feat/fooBar-123
+```
+
+The subdomain name will be output. It can then be used in a workflow as a namespace name and subdomain value.
diff --git a/dev/k8s-get-deploy-name/cli.js b/dev/k8s-get-deploy-name/cli.js
new file mode 100644
index 0000000000..b6c3b5119e
--- /dev/null
+++ b/dev/k8s-get-deploy-name/cli.js
@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+
+import yargs from 'yargs/yargs'
+import { hideBin } from 'yargs/helpers'
+import slugify from 'slugify'
+
+const argv = yargs(hideBin(process.argv)).argv
+
+let branch = argv.branch
+if (!branch) {
+  throw new Error('Missing --branch argument!')
+}
+if (branch.indexOf('/') >= 0) {
+  branch = branch.split('/').slice(1).join('-')
+}
+branch = slugify(branch, { lower: true, strict: true })
+if (branch.length < 1) {
+  throw new Error('Branch name is empty!')
+}
+process.stdout.write(`dt-${branch}`)
+
+process.exit(0)
diff --git a/dev/k8s-get-deploy-name/package-lock.json b/dev/k8s-get-deploy-name/package-lock.json
new file mode 100644
index 0000000000..e492a4cd38
--- /dev/null
+++ b/dev/k8s-get-deploy-name/package-lock.json
@@ -0,0 +1,303 @@
+{
+  "name": "k8s-get-deploy-name",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "k8s-get-deploy-name",
+      "dependencies": {
+        "slugify": "1.6.6",
+        "yargs": "17.7.2"
+      }
+    },
+    "node_modules/ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/cliui": {
+      "version": "8.0.1",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
+      "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
+      "dependencies": {
+        "string-width": "^4.2.0",
+        "strip-ansi": "^6.0.1",
+        "wrap-ansi": "^7.0.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
+    },
+    "node_modules/emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+    },
+    "node_modules/escalade": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
+      "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/get-caller-file": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
+      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
+      "engines": {
+        "node": "6.* || 8.* || >= 10.*"
+      }
+    },
+    "node_modules/is-fullwidth-code-point": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/require-directory": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
+      "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/slugify": {
+      "version": "1.6.6",
+      "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz",
+      "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==",
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
+    "node_modules/string-width": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "dependencies": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "dependencies": {
+        "ansi-regex": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/wrap-ansi": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
+      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
+      "dependencies": {
+        "ansi-styles": "^4.0.0",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
+    "node_modules/y18n": {
+      "version": "5.0.8",
+      "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
+      "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/yargs": {
+      "version": "17.7.2",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",
+      "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==",
+      "dependencies": {
+        "cliui": "^8.0.1",
+        "escalade": "^3.1.1",
+        "get-caller-file": "^2.0.5",
+        "require-directory": "^2.1.1",
+        "string-width": "^4.2.3",
+        "y18n": "^5.0.5",
+        "yargs-parser": "^21.1.1"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/yargs-parser": {
+      "version": "21.1.1",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
+      "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
+      "engines": {
+        "node": ">=12"
+      }
+    }
+  },
+  "dependencies": {
+    "ansi-regex": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+      "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="
+    },
+    "ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "requires": {
+        "color-convert": "^2.0.1"
+      }
+    },
+    "cliui": {
+      "version": "8.0.1",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
+      "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
+      "requires": {
+        "string-width": "^4.2.0",
+        "strip-ansi": "^6.0.1",
+        "wrap-ansi": "^7.0.0"
+      }
+    },
+    "color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "requires": {
+        "color-name": "~1.1.4"
+      }
+    },
+    "color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
+    },
+    "emoji-regex": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+      "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
+    },
+    "escalade": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
+      "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw=="
+    },
+    "get-caller-file": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
+      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
+    },
+    "is-fullwidth-code-point": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+      "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
+    },
+    "require-directory": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
+      "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="
+    },
+    "slugify": {
+      "version": "1.6.6",
+      "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz",
+      "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw=="
+    },
+    "string-width": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+      "requires": {
+        "emoji-regex": "^8.0.0",
+        "is-fullwidth-code-point": "^3.0.0",
+        "strip-ansi": "^6.0.1"
+      }
+    },
+    "strip-ansi": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+      "requires": {
+        "ansi-regex": "^5.0.1"
+      }
+    },
+    "wrap-ansi": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
+      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
+      "requires": {
+        "ansi-styles": "^4.0.0",
+        "string-width": "^4.1.0",
+        "strip-ansi": "^6.0.0"
+      }
+    },
+    "y18n": {
+      "version": "5.0.8",
+      "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
+      "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="
+    },
+    "yargs": {
+      "version": "17.7.2",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",
+      "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==",
+      "requires": {
+        "cliui": "^8.0.1",
+        "escalade": "^3.1.1",
+        "get-caller-file": "^2.0.5",
+        "require-directory": "^2.1.1",
+        "string-width": "^4.2.3",
+        "y18n": "^5.0.5",
+        "yargs-parser": "^21.1.1"
+      }
+    },
+    "yargs-parser": {
+      "version": "21.1.1",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
+      "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="
+    }
+  }
+}
diff --git a/dev/k8s-get-deploy-name/package.json b/dev/k8s-get-deploy-name/package.json
new file mode 100644
index 0000000000..849f5d9b8d
--- /dev/null
+++ b/dev/k8s-get-deploy-name/package.json
@@ -0,0 +1,8 @@
+{
+  "name": "k8s-get-deploy-name",
+  "type": "module",
+  "dependencies": {
+    "slugify": "1.6.6",
+    "yargs": "17.7.2"
+  }
+}

From 5a7be260dd6dfd9c484bc7c50ef991642fa8ad8e Mon Sep 17 00:00:00 2001
From: NGPixel <github@ngpixel.com>
Date: Wed, 15 Oct 2025 03:07:07 -0400
Subject: [PATCH 472/656] chore: add disableDailyDbRefresh flag to build
 workflow

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 15eaba48d1..4c70456a73 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -426,7 +426,7 @@ jobs:
           repo: ietf-tools/infra-k8s
           ref: main
           token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
-          inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}" }'
+          inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }'
           wait-for-completion: true
           wait-for-completion-timeout: 30m
           wait-for-completion-interval: 30s

From 93c1124c21267556625df760c68f35f6d4ae8139 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 15 Oct 2025 13:06:45 -0500
Subject: [PATCH 473/656] ci: add ruff to devcontainer (#9731)

---
 .devcontainer/devcontainer.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 6b0fd79bb3..bf28550084 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -23,7 +23,6 @@
                 "dbaeumer.vscode-eslint",
                 "eamodio.gitlens",
                 "editorconfig.editorconfig",
-                // Newer volar >=3.0.0 causes crashes in devcontainers
                 "vue.volar@2.2.10",
                 "mrmlnc.vscode-duplicate",
                 "ms-azuretools.vscode-docker",
@@ -35,7 +34,8 @@
                 "redhat.vscode-yaml",
                 "spmeesseman.vscode-taskexplorer",
                 "visualstudioexptteam.vscodeintellicode",
-                "ms-python.pylint"
+                "ms-python.pylint",
+                "charliermarsh.ruff"
             ],
             "settings": {
                 "terminal.integrated.defaultProfile.linux": "zsh",

From d5660ab8e953fec25dbb20025aba73b2e58f0609 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 15 Oct 2025 18:30:18 -0300
Subject: [PATCH 474/656] fix: unbreak EmptyAwareJSONField (#9732)

* fix: specify default form_class correctly

* style: ruff ruff
---
 ietf/utils/db.py | 63 ++++++++++++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 23 deletions(-)

diff --git a/ietf/utils/db.py b/ietf/utils/db.py
index 865c9b999f..49c89da13a 100644
--- a/ietf/utils/db.py
+++ b/ietf/utils/db.py
@@ -1,33 +1,44 @@
-# Copyright The IETF Trust 2021, All Rights Reserved
-# -*- coding: utf-8 -*-
-
-# Taken from/inspired by
-# https://stackoverflow.com/questions/55147169/django-admin-jsonfield-default-empty-dict-wont-save-in-admin
-# 
-# JSONField should recognize {}, (), and [] as valid, non-empty JSON
-# values.  However, the base Field class excludes them
+# Copyright The IETF Trust 2021-2025, All Rights Reserved
 
 import jsonfield
 from django.db import models
 
-from ietf.utils.fields import IETFJSONField as FormIETFJSONField, EmptyAwareJSONField as FormEmptyAwareJSONField
+from ietf.utils.fields import (
+    IETFJSONField as FormIETFJSONField,
+    EmptyAwareJSONField as FormEmptyAwareJSONField,
+)
 
 
 class EmptyAwareJSONField(models.JSONField):
-    form_class = FormEmptyAwareJSONField
+    """JSONField that allows empty JSON values when model specifies empty=False
+
+    Taken from/inspired by
+    https://stackoverflow.com/questions/55147169/django-admin-jsonfield-default-empty-dict-wont-save-in-admin
+
+    JSONField should recognize {}, (), and [] as valid, non-empty JSON values.
 
-    def __init__(self, *args, empty_values=FormEmptyAwareJSONField.empty_values, accepted_empty_values=None, **kwargs):
+    If customizing the formfield, the field must accept the `empty_values` argument.
+    """
+
+    def __init__(
+        self,
+        *args,
+        empty_values=FormEmptyAwareJSONField.empty_values,
+        accepted_empty_values=None,
+        **kwargs,
+    ):
         if accepted_empty_values is None:
             accepted_empty_values = []
-        self.empty_values = [x
-                             for x in empty_values
-                             if x not in accepted_empty_values]
+        self.empty_values = [x for x in empty_values if x not in accepted_empty_values]
         super().__init__(*args, **kwargs)
 
     def formfield(self, **kwargs):
-        if 'form_class' not in kwargs or issubclass(kwargs['form_class'], FormEmptyAwareJSONField):
-            kwargs.setdefault('empty_values', self.empty_values)
-        return super().formfield(**{**kwargs})
+        defaults = {
+            "form_class": FormEmptyAwareJSONField,
+            "empty_values": self.empty_values,
+        }
+        defaults.update(kwargs)
+        return super().formfield(**defaults)
 
 
 class IETFJSONField(jsonfield.JSONField):  # pragma: no cover
@@ -36,15 +47,21 @@ class IETFJSONField(jsonfield.JSONField):  # pragma: no cover
     # Remove this class when migrations are squashed and it is no longer referenced
     form_class = FormIETFJSONField
 
-    def __init__(self, *args, empty_values=FormIETFJSONField.empty_values, accepted_empty_values=None, **kwargs):
+    def __init__(
+        self,
+        *args,
+        empty_values=FormIETFJSONField.empty_values,
+        accepted_empty_values=None,
+        **kwargs,
+    ):
         if accepted_empty_values is None:
             accepted_empty_values = []
-        self.empty_values = [x
-                             for x in empty_values
-                             if x not in accepted_empty_values]
+        self.empty_values = [x for x in empty_values if x not in accepted_empty_values]
         super().__init__(*args, **kwargs)
 
     def formfield(self, **kwargs):
-        if 'form_class' not in kwargs or issubclass(kwargs['form_class'], FormIETFJSONField):
-            kwargs.setdefault('empty_values', self.empty_values)
+        if "form_class" not in kwargs or issubclass(
+            kwargs["form_class"], FormIETFJSONField
+        ):
+            kwargs.setdefault("empty_values", self.empty_values)
         return super().formfield(**{**kwargs})

From 1d2d304fa5c99db6cd2a944328246ce900c73b3c Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 16 Oct 2025 12:39:04 -0300
Subject: [PATCH 475/656] fix: improve proceedings caching/performance (#9733)

* refactor: speed up get_attendance()

* fix: avoid cache invalidation by later draft rev

* fix: guard against empty value

* feat: freeze cache key for final proceedings
---
 ietf/meeting/models.py | 28 +++++++++++++++++++++-------
 ietf/meeting/utils.py  | 39 +++++++++++++++++++++++++++++++++++----
 2 files changed, 56 insertions(+), 11 deletions(-)

diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index f3df23e916..9e44df33b7 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -250,25 +250,39 @@ def get_attendance(self):
         # MeetingRegistration.attended started conflating badge-pickup and session attendance before IETF 114.
         # We've separated session attendance off to ietf.meeting.Attended, but need to report attendance at older
         # meetings correctly.
-
+        #
+        # Looking up by registration and attendance records separately and joining in
+        # python is far faster than combining the Q objects in the query (~100x).
+        # Further optimization may be possible, but the queries are tricky...
         attended_per_meeting_registration = (
             Q(registration__meeting=self) & (
                 Q(registration__attended=True) |
                 Q(registration__checkedin=True)
             )
         )
+        attendees_by_reg = set(
+            Person.objects.filter(attended_per_meeting_registration).values_list(
+                "pk", flat=True
+            )
+        )
+
         attended_per_meeting_attended = (
             Q(attended__session__meeting=self)
             # Note that we are not filtering to plenary, wg, or rg sessions
             # as we do for nomcom eligibility - if picking up a badge (see above)
             # is good enough, just attending e.g. a training session is also good enough
         )
-        attended = Person.objects.filter(
-            attended_per_meeting_registration | attended_per_meeting_attended
-        ).distinct()
-
-        onsite = set(attended.filter(registration__meeting=self, registration__tickets__attendance_type__slug='onsite'))
-        remote = set(attended.filter(registration__meeting=self, registration__tickets__attendance_type__slug='remote'))
+        attendees_by_att = set(
+            Person.objects.filter(attended_per_meeting_attended).values_list(
+                "pk", flat=True
+            )
+        )
+        
+        attendees = Person.objects.filter(
+            pk__in=attendees_by_att | attendees_by_reg
+        )
+        onsite = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='onsite'))
+        remote = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='remote'))
         remote.difference_update(onsite)
 
         return Attendance(
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index f6925269aa..feadb0c7fd 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -1027,10 +1027,41 @@ def generate_proceedings_content(meeting, force_refresh=False):
     :force_refresh: true to force regeneration and cache refresh
     """
     cache = caches["default"]
-    cache_version = Document.objects.filter(session__meeting__number=meeting.number).aggregate(Max('time'))["time__max"]
-    # Include proceedings_final in the bare_key so we'll always reflect that accurately, even at the cost of
-    # a recomputation in the view
-    bare_key = f"proceedings.{meeting.number}.{cache_version}.final={meeting.proceedings_final}"
+    key_components = [
+        "proceedings",
+        str(meeting.number),
+    ]
+    if meeting.proceedings_final:
+        # Freeze the cache key once proceedings are finalized. Further changes will
+        # not be picked up until the cache expires or is refreshed by the
+        # proceedings_content_refresh_task()
+        key_components.append("final")
+    else:
+        # Build a cache key that changes when materials are modified. For all but drafts,
+        # use the last modification time of the document. Exclude drafts from this because
+        # revisions long after the meeting ends will otherwise show up as changes and
+        # incorrectly invalidate the cache. Instead, include an ordered list of the
+        # drafts linked to the meeting so adding or removing drafts will trigger a
+        # recalculation. The list is long but that doesn't matter because we hash it into
+        # a fixed-length key.
+        meeting_docs = Document.objects.filter(session__meeting__number=meeting.number)
+        last_materials_update = (
+            meeting_docs.exclude(type_id="draft")
+            .filter(session__meeting__number=meeting.number)
+            .aggregate(Max("time"))["time__max"]
+        )
+        draft_names = (
+            meeting_docs
+            .filter(type_id="draft")
+            .order_by("name")
+            .values_list("name", flat=True)
+        )
+        key_components += [
+            last_materials_update.isoformat() if last_materials_update else "-",
+            ",".join(draft_names),
+        ]
+
+    bare_key = ".".join(key_components)
     cache_key = sha384(bare_key.encode("utf8")).hexdigest()
     if not force_refresh:
         cached_content = cache.get(cache_key, None)

From 2cfbaf90c3504a53135d61f9bf976bab3b388eb9 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 16 Oct 2025 14:28:13 -0300
Subject: [PATCH 476/656] ci: drop caching from build images step (#9738)

---
 .github/workflows/build.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4c70456a73..7eac7b1c64 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -278,8 +278,6 @@ jobs:
         tags: |
           ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }}
           ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }}
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
 
     - name: Update CHANGELOG
       id: changelog

From b0ec8c4b27d6225c6ffa6cac27ce554ec4a49a7c Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 17 Oct 2025 13:08:11 -0300
Subject: [PATCH 477/656] chore: remove unused variables (#9742)

---
 ietf/meeting/models.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py
index 9e44df33b7..7d9e318aab 100644
--- a/ietf/meeting/models.py
+++ b/ietf/meeting/models.py
@@ -956,8 +956,6 @@ class Meta:
     def __str__(self):
         return u"%s -> %s-%s" % (self.session, self.document.name, self.rev)
 
-constraint_cache_uses = 0
-constraint_cache_initials = 0
 
 class SessionQuerySet(models.QuerySet):
     def with_current_status(self):

From 62f720ceaf951fba91b5a818473d798663dfbf1d Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 21 Oct 2025 12:31:39 -0300
Subject: [PATCH 478/656] ci: imagePullPolicy for migration container (#9764)

---
 k8s/datatracker.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml
index 3d9e86a29d..50a2c69687 100644
--- a/k8s/datatracker.yaml
+++ b/k8s/datatracker.yaml
@@ -115,6 +115,7 @@ spec:
       initContainers:
         - name: migration
           image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
+          imagePullPolicy: Always
           env:
             - name: "CONTAINER_ROLE"
               value: "migrations"

From a3a3d215ca4067e722ead94e886175adb589e235 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 23 Oct 2025 12:14:48 -0500
Subject: [PATCH 479/656] fix: don't limit from_contact for incoming liaison
 statements (#9773)

---
 ietf/liaisons/forms.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py
index ef5b29535e..1747e55571 100644
--- a/ietf/liaisons/forms.py
+++ b/ietf/liaisons/forms.py
@@ -495,14 +495,18 @@ def set_from_fields(self):
             self.fields['from_groups'].initial = qs
 
         # Note that the IAB chair currently doesn't get to work with incoming liaison statements
-        if not (
-            has_role(self.user, "Secretariat")
-            or has_role(self.user, "Liaison Coordinator")
-        ):
-            self.fields["from_contact"].initial = (
-                self.person.role_set.filter(group=qs[0]).first().email.formatted_email()
-            )
-            self.fields["from_contact"].widget.attrs["disabled"] = True
+
+        # Removing this block at the request of the IAB - as a workaround until the new liaison tool is
+        # create, anyone with access to the form can set any from_contact value
+        #
+        # if not (
+        #     has_role(self.user, "Secretariat")
+        #     or has_role(self.user, "Liaison Coordinator")
+        # ):
+        #     self.fields["from_contact"].initial = (
+        #         self.person.role_set.filter(group=qs[0]).first().email.formatted_email()
+        #     )
+        #     self.fields["from_contact"].widget.attrs["disabled"] = True
 
     def set_to_fields(self):
         '''Set to_groups and to_contacts options and initial value based on user

From 1243957f06da485e5cf4c04a8479d551817d4d78 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 23 Oct 2025 14:15:22 -0300
Subject: [PATCH 480/656] feat: unversioned proceedings cache (#9779)

* feat: separate, unversioned proceedings cache

* refactor: don't double-hash the cache key
---
 ietf/meeting/utils.py |  8 ++++----
 ietf/settings.py      | 22 ++++++++++++++++++++++
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index feadb0c7fd..afcf7656f2 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -5,7 +5,6 @@
 import jsonschema
 import os
 import requests
-from hashlib import sha384
 
 import pytz
 import subprocess
@@ -1026,7 +1025,7 @@ def generate_proceedings_content(meeting, force_refresh=False):
     :meeting: meeting whose proceedings should be rendered
     :force_refresh: true to force regeneration and cache refresh
     """
-    cache = caches["default"]
+    cache = caches["proceedings"]
     key_components = [
         "proceedings",
         str(meeting.number),
@@ -1061,8 +1060,9 @@ def generate_proceedings_content(meeting, force_refresh=False):
             ",".join(draft_names),
         ]
 
-    bare_key = ".".join(key_components)
-    cache_key = sha384(bare_key.encode("utf8")).hexdigest()
+    # Key is potentially long, but the "proceedings" cache hashes it to a fixed
+    # length. If that changes, hash it separately here first.
+    cache_key = ".".join(key_components)
     if not force_refresh:
         cached_content = cache.get(cache_key, None)
         if cached_content is not None:
diff --git a/ietf/settings.py b/ietf/settings.py
index 9a213c1a73..5e576430ed 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -1374,6 +1374,17 @@ def skip_unreadable_post(record):
                 "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
                 "VERSION": __version__,
                 "KEY_PREFIX": "ietf:dt",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
+            },
+            "proceedings": {
+                "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt:proceedings",
+                # Key function is default except with sha384-encoded key
                 "KEY_FUNCTION": lambda key, key_prefix, version: (
                     f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
                 ),
@@ -1421,6 +1432,17 @@ def skip_unreadable_post(record):
                 "VERSION": __version__,
                 "KEY_PREFIX": "ietf:dt",
             },
+            "proceedings": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                # "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                # "LOCATION": "127.0.0.1:11211",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt:proceedings",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
+            },
             "sessions": {
                 "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
             },

From 6412d1e24a9c499c39245bba58c2c31ec8110c0e Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 23 Oct 2025 17:41:06 -0300
Subject: [PATCH 481/656] feat: blobstore-driven meeting materials (#9780)

* feat: meeting materials blob resolver API (#9700)

* refactor: exclude_deleted() for StoredObject queryset

* chore: comment

* feat: meeting materials blob resolver API

* feat: materials blob retrieval API (#9728)

* feat: materials blob retrieval API (WIP)

* refactor: alphabetize ARTIFACT_STORAGE_NAMES

* chore: limit buckets served

* refactor: any-meeting option in _get_materials_doc()

* feat: create missing blobs on retrieval

* feat: render HTML from markdown via API (#9729)

* chore: add comment

* fix: allow bluesheets to be retrieved

Normally not retrieved through /meeting/materials, but they're
close enough in purpose that we might as well make them available.

* fix: only update StoredObject.modified if changed

* fix: preserve mtime when creating blob

* refactor: better exception name

* feat: render .md.html from .md blob

* fix: explicit STATIC_IETF_ORG value in template

Django's context_processors are not applied to render_string calls as we use them here, so settings are not available.

* fix: typo

* fix: decode utf-8 properly

* feat: use filesystem to render .md.html

* fix: copy/paste error in api_resolve_materials_name

* refactor: get actual rev in _get_materials_doc (#9741)

* fix: return filename, not full path

* feat: precompute blob lookups for meeting materials (#9746)

* feat: ResolvedMaterial model + migration

* feat: method to populate ResolvedMaterial (WIP)

* refactor: don't delete ResolvedMaterials

Instead of deleting the ResolvedMaterials for a meeting, which might
lose updates made during processing, update existing rows with any
changes and warn if anything changed during the process.

* fix: fix _get_materials_doc()

Did not handle the possibility of multiple DocHistory objects with the
same rev.

* refactor: factor out material lookup helper

* feat: resolve blobs via blobdb/fs for cache

* chore: add resource

* feat: admin for ResolvedMaterial

* feat: cache-driven resolve materials API

* fix: add all ResolvedMaterials; var names

* fix: handle null case

* feat: resolve_meeting_materials_task

* feat: update resolver cache on material upload (#9759)

* feat: robustness + date range for resolve materials task (#9760)

* fix: limit types added to ResolvedMaterial

* feat: resolve meeting materials in order by date

* feat: add meetings_until param

* fix: log&continue if resolving fails on a meeting

* feat: log error message on parse errors

* refactor: move ResolvedMaterial to blobdb app (#9762)

* refactor: move ResolvedMaterial to blobdb app

* fix: undo accidental removal

* chore: fix lint (#9767)

* fix: don't use DocHistory to find materials (#9771)

* fix: don't use DocHistory to validate revs

The DocHistory records are incomplete and, in particular, -00 revs are
often missing.

* Revert "refactor: get actual rev in _get_materials_doc (#9741)"

This reverts commit 7fd15801

* chore: remove the on-demand resolver api

* chore: fix lint

* feat: populate materials buckets (#9777)

* refactor: drop .txt from filename_with_rev()

* feat: utilities to populate materials blobs

* feat: store materials for a full meeting as blobs

Plus a bunch of fixup from working with real data.
(Based on meetings 71, 83, and 118, picked arbitrarily)

* chore: update migration

* feat: task to store materials in blobdb

* refactor: reimplement api_retrieve_materials_blob

* fix: update resolving task, fix bugs

* Revert "refactor: drop .txt from filename_with_rev()"

This reverts commit a849d0f92d4df54296a7062b6c3a05fb0977be93.

* chore: fix lint

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
---
 ietf/api/urls.py                              |   3 +
 ietf/blobdb/admin.py                          |  11 +-
 .../migrations/0002_resolvedmaterial.py       |  48 +++
 ietf/blobdb/models.py                         |  20 +
 ietf/doc/models.py                            |   9 +
 ietf/doc/storage.py                           |  10 +-
 ietf/doc/storage_utils.py                     |  12 +-
 ietf/doc/views_material.py                    |   4 +
 ietf/meeting/resources.py                     |  14 +-
 ietf/meeting/tasks.py                         | 131 ++++++-
 ietf/meeting/utils.py                         | 355 +++++++++++++++++-
 ietf/meeting/views.py                         | 177 ++++++++-
 ietf/settings.py                              |  44 ++-
 ietf/templates/minimal.html                   |   4 +-
 14 files changed, 798 insertions(+), 44 deletions(-)
 create mode 100644 ietf/blobdb/migrations/0002_resolvedmaterial.py

diff --git a/ietf/api/urls.py b/ietf/api/urls.py
index 6f2efb3c1e..04575b34cb 100644
--- a/ietf/api/urls.py
+++ b/ietf/api/urls.py
@@ -49,6 +49,9 @@
     url(r'^group/role-holder-addresses/$', api_views.role_holder_addresses),
     # Let IESG members set positions programmatically
     url(r'^iesg/position', views_ballot.api_set_position),
+    # Find the blob to store for a given materials document path
+    url(r'^meeting/(?:(?P<num>(?:interim-)?[a-z0-9-]+)/)?materials/%(document)s(?P<ext>\.[A-Za-z0-9]+)?/resolve-cached/$' % settings.URL_REGEXPS, meeting_views.api_resolve_materials_name_cached),
+    url(r'^meeting/blob/(?P<bucket>[a-z0-9-]+)/(?P<name>[a-z][a-z0-9.-]+)$', meeting_views.api_retrieve_materials_blob),
     # Let Meetecho set session video URLs
     url(r'^meeting/session/video/url$', meeting_views.api_set_session_video_url),
     # Let Meetecho tell us the name of its recordings
diff --git a/ietf/blobdb/admin.py b/ietf/blobdb/admin.py
index f4cd002e07..3e1a2a311f 100644
--- a/ietf/blobdb/admin.py
+++ b/ietf/blobdb/admin.py
@@ -3,7 +3,7 @@
 from django.db.models.functions import Length
 from rangefilter.filters import DateRangeQuickSelectListFilterBuilder
 
-from .models import Blob
+from .models import Blob, ResolvedMaterial
 
 
 @admin.register(Blob)
@@ -29,3 +29,12 @@ def get_queryset(self, request):
     def object_size(self, instance):
         """Get the size of the object"""
         return instance.object_size  # annotation added in get_queryset()
+
+
+@admin.register(ResolvedMaterial)
+class ResolvedMaterialAdmin(admin.ModelAdmin):
+    model = ResolvedMaterial
+    list_display = ["name", "meeting_number", "bucket", "blob"]
+    list_filter = ["meeting_number", "bucket"]
+    search_fields = ["name", "blob"]
+    ordering = ["name"]
diff --git a/ietf/blobdb/migrations/0002_resolvedmaterial.py b/ietf/blobdb/migrations/0002_resolvedmaterial.py
new file mode 100644
index 0000000000..e0ab405b11
--- /dev/null
+++ b/ietf/blobdb/migrations/0002_resolvedmaterial.py
@@ -0,0 +1,48 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("blobdb", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ResolvedMaterial",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(help_text="Name to resolve", max_length=300)),
+                (
+                    "meeting_number",
+                    models.CharField(
+                        help_text="Meeting material is related to", max_length=64
+                    ),
+                ),
+                (
+                    "bucket",
+                    models.CharField(help_text="Resolved bucket name", max_length=255),
+                ),
+                (
+                    "blob",
+                    models.CharField(help_text="Resolved blob name", max_length=300),
+                ),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name="resolvedmaterial",
+            constraint=models.UniqueConstraint(
+                fields=("name", "meeting_number"), name="unique_name_per_meeting"
+            ),
+        ),
+    ]
diff --git a/ietf/blobdb/models.py b/ietf/blobdb/models.py
index 8f423d9f6c..fa7831f203 100644
--- a/ietf/blobdb/models.py
+++ b/ietf/blobdb/models.py
@@ -96,3 +96,23 @@ def _emit_blob_change_event(self, using=None):
             ),
             using=using,
         )
+
+
+class ResolvedMaterial(models.Model):
+    # A Document name can be 255 characters; allow this name to be a bit longer
+    name = models.CharField(max_length=300, help_text="Name to resolve")
+    meeting_number = models.CharField(
+        max_length=64, help_text="Meeting material is related to"
+    )
+    bucket = models.CharField(max_length=255, help_text="Resolved bucket name")
+    blob = models.CharField(max_length=300, help_text="Resolved blob name")
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["name", "meeting_number"], name="unique_name_per_meeting"
+            )
+        ]
+    
+    def __str__(self):
+        return f"{self.name}@{self.meeting_number} -> {self.bucket}:{self.blob}"
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index 25ee734cbe..8bb79b64ed 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -913,6 +913,7 @@ def role_for_doc(self):
             roles.append('Action Holder')
         return ', '.join(roles) 
 
+# N.B., at least a couple dozen documents exist that do not satisfy this validator
 validate_docname = RegexValidator(
     r'^[-a-z0-9]+$',
     "Provide a valid document name consisting of lowercase letters, numbers and hyphens.",
@@ -1588,9 +1589,17 @@ class BofreqResponsibleDocEvent(DocEvent):
     """ Capture the responsible leadership (IAB and IESG members) for a BOF Request """
     responsible = models.ManyToManyField('person.Person', blank=True)
 
+
+class StoredObjectQuerySet(models.QuerySet):
+    def exclude_deleted(self):
+        return self.filter(deleted__isnull=True)
+
+
 class StoredObject(models.Model):
     """Hold metadata about objects placed in object storage"""
 
+    objects = StoredObjectQuerySet.as_manager() 
+
     store = models.CharField(max_length=256)
     name = models.CharField(max_length=1024, null=False, blank=False) # N.B. the 1024 limit on name comes from S3
     sha384 = models.CharField(max_length=96)
diff --git a/ietf/doc/storage.py b/ietf/doc/storage.py
index a234ef2d4f..375620ccaf 100644
--- a/ietf/doc/storage.py
+++ b/ietf/doc/storage.py
@@ -32,7 +32,7 @@ def __init__(self, file, name, mtime=None, content_type="", store=None, doc_name
     @classmethod
     def from_storedobject(cls, file, name, store):
         """Alternate constructor for objects that already exist in the StoredObject table"""
-        stored_object = StoredObject.objects.filter(store=store, name=name, deleted__isnull=True).first()
+        stored_object = StoredObject.objects.exclude_deleted().filter(store=store, name=name).first()
         if stored_object is None:
             raise FileNotFoundError(f"StoredObject for {store}:{name} does not exist or was deleted")
         file = cls(file, name, store, doc_name=stored_object.doc_name, doc_rev=stored_object.doc_rev)
@@ -140,7 +140,11 @@ def _save_stored_object(self, name, content) -> StoredObject:
                 ),
             ),
         )
-        if not created:
+        if not created and (
+            record.sha384 != content.custom_metadata["sha384"]
+            or record.len != int(content.custom_metadata["len"])
+            or record.deleted is not None
+        ):
             record.sha384 = content.custom_metadata["sha384"]
             record.len = int(content.custom_metadata["len"])
             record.modified = now
@@ -160,7 +164,7 @@ def _delete_stored_object(self, name) -> Optional[StoredObject]:
         else:
             now = timezone.now()
             # Note that existing_record is a queryset that will have one matching object
-            existing_record.filter(deleted__isnull=True).update(deleted=now)
+            existing_record.exclude_deleted().update(deleted=now)
         return existing_record.first()
 
     def _save(self, name, content):
diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py
index 510c98c4f5..81588c83ec 100644
--- a/ietf/doc/storage_utils.py
+++ b/ietf/doc/storage_utils.py
@@ -12,6 +12,14 @@
 from ietf.utils.log import log
 
 
+class StorageUtilsError(Exception):
+    pass
+
+
+class AlreadyExistsError(StorageUtilsError):
+    pass
+
+
 def _get_storage(kind: str) -> Storage:
     if kind in settings.ARTIFACT_STORAGE_NAMES:
         return storages[kind]
@@ -70,7 +78,7 @@ def store_file(
             # debug.show('f"Asked to store {name} in {kind}: is_new={is_new}, allow_overwrite={allow_overwrite}"')
             if not allow_overwrite and not is_new:
                 debug.show('f"Failed to save {kind}:{name} - name already exists in store"')
-                raise RuntimeError(f"Failed to save {kind}:{name} - name already exists in store")
+                raise AlreadyExistsError(f"Failed to save {kind}:{name} - name already exists in store")
             new_name = _get_storage(kind).save(
                 name,
                 StoredObjectFile(
@@ -85,7 +93,7 @@ def store_file(
             if new_name != name:
                 complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead."
                 debug.show("complaint")
-                raise RuntimeError(complaint)
+                raise StorageUtilsError(complaint)
         except Exception as err:
             log(f"Blobstore Error: Failed to store file {kind}:{name}: {repr(err)}")
             if settings.SERVER_MODE == "development":
diff --git a/ietf/doc/views_material.py b/ietf/doc/views_material.py
index 6f8b8a8f12..eefac0ca61 100644
--- a/ietf/doc/views_material.py
+++ b/ietf/doc/views_material.py
@@ -22,6 +22,7 @@
 from ietf.doc.utils import add_state_change_event, check_common_doc_name_rules
 from ietf.group.models import Group
 from ietf.group.utils import can_manage_materials
+from ietf.meeting.utils import resolve_uploaded_material
 from ietf.utils import log
 from ietf.utils.decorators import ignore_view_kwargs
 from ietf.utils.meetecho import MeetechoAPIError, SlidesManager
@@ -179,6 +180,9 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None):
                             "There was an error creating a hardlink at %s pointing to %s: %s"
                             % (ftp_filepath, filepath, ex)
                         )
+                else:
+                    for meeting in set([s.meeting for s in doc.session_set.all()]):
+                        resolve_uploaded_material(meeting=meeting, doc=doc)
 
             if prev_rev != doc.rev:
                 e = NewRevisionDocEvent(type="new_revision", doc=doc, rev=doc.rev)
diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py
index ede2b5b993..88562a88fe 100644
--- a/ietf/meeting/resources.py
+++ b/ietf/meeting/resources.py
@@ -11,11 +11,15 @@
 
 from ietf import api
 
-from ietf.meeting.models import ( Meeting, ResourceAssociation, Constraint, Room, Schedule, Session,
-                                TimeSlot, SchedTimeSessAssignment, SessionPresentation, FloorPlan,
-                                UrlResource, ImportantDate, SlideSubmission, SchedulingEvent,
-                                BusinessConstraint, ProceedingsMaterial, MeetingHost, Attended,
-                                Registration, RegistrationTicket)
+from ietf.meeting.models import (Meeting, ResourceAssociation, Constraint, Room,
+                                 Schedule, Session,
+                                 TimeSlot, SchedTimeSessAssignment, SessionPresentation,
+                                 FloorPlan,
+                                 UrlResource, ImportantDate, SlideSubmission,
+                                 SchedulingEvent,
+                                 BusinessConstraint, ProceedingsMaterial, MeetingHost,
+                                 Attended,
+                                 Registration, RegistrationTicket)
 
 from ietf.name.resources import MeetingTypeNameResource
 class MeetingResource(ModelResource):
diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py
index 784eb00d87..c361325f9a 100644
--- a/ietf/meeting/tasks.py
+++ b/ietf/meeting/tasks.py
@@ -1,13 +1,20 @@
-# Copyright The IETF Trust 2024, All Rights Reserved
+# Copyright The IETF Trust 2024-2025, All Rights Reserved
 #
 # Celery task definitions
 #
+import datetime
+
 from celery import shared_task
+# from django.db.models import QuerySet
 from django.utils import timezone
 
 from ietf.utils import log
 from .models import Meeting
-from .utils import generate_proceedings_content
+from .utils import (
+    generate_proceedings_content,
+    resolve_materials_for_one_meeting,
+    store_blobs_for_one_meeting,
+)
 from .views import generate_agenda_data
 from .utils import fetch_attendance_from_meetings
 
@@ -61,3 +68,123 @@ def fetch_meeting_attendance_task():
                     meeting_stats['processed']
                 )
             )
+
+
+def _select_meetings(
+    meetings: list[str] | None = None,
+    meetings_since: str | None = None,
+    meetings_until: str | None = None
+):  # nyah
+    """Select meetings by number or date range"""
+    # IETF-1 = 1986-01-16
+    EARLIEST_MEETING_DATE = datetime.datetime(1986, 1, 1)
+    meetings_since_dt: datetime.datetime | None = None
+    meetings_until_dt: datetime.datetime | None = None
+
+    if meetings_since == "zero":
+        meetings_since_dt = EARLIEST_MEETING_DATE
+    elif meetings_since is not None:
+        try:
+            meetings_since_dt = datetime.datetime.fromisoformat(meetings_since)
+        except ValueError:
+            log.log(
+                "Failed to parse meetings_since='{meetings_since}' with fromisoformat"
+            )
+            raise
+
+    if meetings_until is not None:
+        try:
+            meetings_until_dt = datetime.datetime.fromisoformat(meetings_until)
+        except ValueError:
+            log.log(
+                "Failed to parse meetings_until='{meetings_until}' with fromisoformat"
+            )
+            raise
+        if meetings_since_dt is None:
+            # if we only got meetings_until, start from the first meeting
+            meetings_since_dt = EARLIEST_MEETING_DATE
+
+    if meetings is None:
+        if meetings_since_dt is None:
+            log.log("No meetings requested, doing nothing.")
+            return Meeting.objects.none()
+        meetings_qs = Meeting.objects.filter(date__gte=meetings_since_dt)
+        if meetings_until_dt is not None:
+            meetings_qs = meetings_qs.filter(date__lte=meetings_until_dt)
+            log.log(
+                "Selecting meetings between "
+                f"{meetings_since_dt} and {meetings_until_dt}"
+            )
+        else:
+            log.log(f"Selecting meetings since {meetings_since_dt}")
+    else:
+        if meetings_since_dt is not None:
+            log.log(
+                "Ignoring meetings_since and meetings_until "
+                "because specific meetings were requested."
+            )
+        meetings_qs = Meeting.objects.filter(number__in=meetings)
+    return meetings_qs
+
+
+@shared_task
+def resolve_meeting_materials_task(
+    *,  # only allow kw arguments
+    meetings: list[str] | None=None,
+    meetings_since: str | None=None,
+    meetings_until: str | None=None
+):
+    """Run materials resolver on meetings
+    
+    Can request a set of meetings by number by passing a list in the meetings arg, or
+    by range by passing an iso-format timestamps in meetings_since / meetings_until.
+    To select all meetings, set meetings_since="zero" and omit other parameters. 
+    """
+    meetings_qs = _select_meetings(meetings, meetings_since, meetings_until)
+    for meeting in meetings_qs.order_by("date"):
+        log.log(
+            f"Resolving materials for {meeting.type_id} "
+            f"meeting {meeting.number} ({meeting.date})..."
+        )
+        mark = timezone.now()
+        try:
+            resolve_materials_for_one_meeting(meeting)
+        except Exception as err:
+            log.log(
+                "Exception raised while resolving materials for "
+                f"meeting {meeting.number}: {err}"
+            )
+        else:
+            log.log(f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds.")
+
+
+@shared_task
+def store_meeting_materials_as_blobs_task(
+    *,  # only allow kw arguments
+    meetings: list[str] | None = None,
+    meetings_since: str | None = None,
+    meetings_until: str | None = None
+):
+    """Push meeting materials into the blob store
+
+    Can request a set of meetings by number by passing a list in the meetings arg, or
+    by range by passing an iso-format timestamps in meetings_since / meetings_until.
+    To select all meetings, set meetings_since="zero" and omit other parameters. 
+    """
+    meetings_qs = _select_meetings(meetings, meetings_since, meetings_until)
+    for meeting in meetings_qs.order_by("date"):
+        log.log(
+            f"Creating blobs for materials for {meeting.type_id} "
+            f"meeting {meeting.number} ({meeting.date})..."
+        )
+        mark = timezone.now()
+        try:
+            store_blobs_for_one_meeting(meeting)
+        except Exception as err:
+            log.log(
+                "Exception raised while creating blobs for "
+                f"meeting {meeting.number}: {err}"
+            )
+        else:
+            log.log(
+                f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds.")
diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index afcf7656f2..bdf3d3d3d3 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -2,6 +2,9 @@
 # -*- coding: utf-8 -*-
 import datetime
 import itertools
+from contextlib import suppress
+from dataclasses import dataclass
+
 import jsonschema
 import os
 import requests
@@ -26,16 +29,33 @@
 import debug                            # pyflakes:ignore
 
 from ietf.dbtemplate.models import DBTemplate
-from ietf.doc.storage_utils import store_bytes, store_str
-from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot,
-    Constraint, SchedTimeSessAssignment, SessionPresentation, Attended,
-    Registration, Meeting, RegistrationTicket)
-from ietf.doc.models import Document, State, NewRevisionDocEvent, StateDocEvent
+from ietf.doc.storage_utils import store_bytes, store_str, AlreadyExistsError
+from ietf.meeting.models import (
+    Session,
+    SchedulingEvent,
+    TimeSlot,
+    Constraint,
+    SchedTimeSessAssignment,
+    SessionPresentation,
+    Attended,
+    Registration,
+    Meeting,
+    RegistrationTicket,
+)
+from ietf.blobdb.models import ResolvedMaterial
+from ietf.doc.models import (
+    Document,
+    State,
+    NewRevisionDocEvent,
+    StateDocEvent,
+    StoredObject,
+)
 from ietf.doc.models import DocEvent
 from ietf.group.models import Group
 from ietf.group.utils import can_manage_materials
 from ietf.name.models import SessionStatusName, ConstraintName, DocTypeName
 from ietf.person.models import Person
+from ietf.utils import markdown
 from ietf.utils.html import clean_html
 from ietf.utils.log import log
 from ietf.utils.timezone import date_today
@@ -220,6 +240,7 @@ def save_bluesheet(request, session, file, encoding='utf-8'):
     save_error = handle_upload_file(file, filename, session.meeting, 'bluesheets', request=request, encoding=encoding)
     if not save_error:
         doc.save_with_history([e])
+        resolve_uploaded_material(meeting=session.meeting, doc=doc)
     return save_error
 
 
@@ -832,6 +853,330 @@ def write_doc_for_session(session, type_id, filename, contents):
     store_str(type_id, filename.name, contents)
     return None
 
+
+@dataclass
+class BlobSpec:
+    bucket: str
+    name: str
+
+
+def resolve_one_material(
+    doc: Document, rev: str | None, ext: str | None
+) -> BlobSpec | None:
+    if doc.type_id is None:
+        log(f"Cannot resolve a doc with no type: {doc.name}")
+        return None
+
+    # Get the Document's base name. It may or may not have an extension.
+    if rev is None:
+        basename = Path(doc.get_base_name())
+    else:
+        basename = Path(f"{doc.name}-{int(rev):02d}")
+
+    # If the document's file exists, the blob is _always_ named with this stem,
+    # even if it's different from the original.
+    blob_stem = Path(f"{doc.name}-{rev or doc.rev}")
+
+    # If we have an extension, either from the URL or the Document's base name, look up
+    # the blob or file or return 404. N.b. the suffix check needs adjustment to handle
+    # a bare "." extension when we reach py3.14.
+    if ext or basename.suffix != "": 
+        if ext:
+            blob_name = str(blob_stem.with_suffix(ext))
+        else:
+            blob_name = str(blob_stem.with_suffix(basename.suffix))
+
+        # See if we have a stored object under that name
+        preferred_blob = (
+            StoredObject.objects.exclude_deleted()
+            .filter(store=doc.type_id, name=blob_name)
+            .first()
+        )
+        if preferred_blob is not None:
+            return BlobSpec(
+                bucket=preferred_blob.store,
+                name=preferred_blob.name,
+            )
+        # No stored object, fall back to the file system.
+        filename = Path(doc.get_file_path()) / basename  # use basename for file
+        if filename.is_file():
+            return BlobSpec(
+                bucket=doc.type_id,
+                name=str(blob_stem.with_suffix(filename.suffix)),
+            )
+        else:
+            return None
+
+    # No extension has been specified so far, so look one up.
+    matching_stored_objects = (
+        StoredObject.objects.exclude_deleted()
+        .filter(
+            store=doc.type_id,
+            name__startswith=f"{blob_stem}.",  # anchor to end with trailing "."
+        )
+        .order_by("name")
+    )  # orders by suffix
+    blob_ext_choices = {
+        Path(stored_obj.name).suffix: stored_obj
+        for stored_obj in matching_stored_objects
+    }
+
+    # Short-circuit to return pdf if present
+    if ".pdf" in blob_ext_choices:
+        pdf_blob = blob_ext_choices[".pdf"]
+        return BlobSpec(
+            bucket=pdf_blob.store,
+            name=str(blob_stem.with_suffix(".pdf")),
+        )
+
+    # Now look for files
+    filename = Path(doc.get_file_path()) / basename
+    file_ext_choices = {
+        # Construct a map from suffix to full filename
+        fn.suffix: fn.name
+        for fn in sorted(filename.parent.glob(filename.stem + ".*"))
+    }
+
+    # Short-circuit to return pdf if we have the file
+    if ".pdf" in file_ext_choices:
+        return BlobSpec(
+            bucket=doc.type_id,
+            name=str(blob_stem.with_suffix(".pdf")),
+        )
+
+    all_exts = set(blob_ext_choices.keys()).union(file_ext_choices.keys())
+    if len(all_exts) > 0:
+        preferred_ext = sorted(all_exts)[0]
+        if preferred_ext in blob_ext_choices:
+            preferred_blob = blob_ext_choices[preferred_ext]
+            return BlobSpec(
+                bucket=preferred_blob.store,
+                name=preferred_blob.name,
+            )
+        else:
+            return BlobSpec(
+                bucket=doc.type_id,
+                name=str(blob_stem.with_suffix(preferred_ext)),
+            )
+
+    return None
+
+
+def resolve_materials_for_one_meeting(meeting: Meeting):
+    start_time = timezone.now()
+    meeting_documents = (
+        Document.objects.filter(
+            type_id__in=settings.MATERIALS_TYPES_SERVED_BY_WORKER
+        ).filter(
+            Q(session__meeting=meeting) | Q(proceedingsmaterial__meeting=meeting)
+        )
+    ).distinct()
+    
+    resolved = []
+    for doc in meeting_documents:
+        # request by doc name with no rev
+        blob = resolve_one_material(doc, rev=None, ext=None)
+        if blob is not None:
+            resolved.append(
+                ResolvedMaterial(
+                    name=doc.name,
+                    meeting_number=meeting.number,
+                    bucket=blob.bucket,
+                    blob=blob.name,
+                )
+            )
+        # request by doc name + rev
+        blob = resolve_one_material(doc, rev=doc.rev, ext=None)
+        if blob is not None:
+            resolved.append(
+                ResolvedMaterial(
+                    name=f"{doc.name}-{doc.rev:02}",
+                    meeting_number=meeting.number,
+                    bucket=blob.bucket,
+                    blob=blob.name,
+                )
+            )
+        # for other revisions, only need request by doc name + rev
+        other_revisions = doc.revisions_by_newrevisionevent()
+        other_revisions.remove(doc.rev)
+        for rev in other_revisions:
+            blob = resolve_one_material(doc, rev=rev, ext=None)
+            if blob is not None:
+                resolved.append(
+                    ResolvedMaterial(
+                        name=f"{doc.name}-{rev:02}",
+                        meeting_number=meeting.number,
+                        bucket=blob.bucket,
+                        blob=blob.name,
+                    )
+                )
+    ResolvedMaterial.objects.bulk_create(
+        resolved,
+        update_conflicts=True,
+        unique_fields=["name", "meeting_number"],
+        update_fields=["bucket", "blob"],
+    )
+    # Warn if any files were updated during the above process
+    last_update = meeting_documents.aggregate(Max("time"))["time__max"]
+    if last_update and last_update > start_time:
+        log(
+            f"Warning: materials for meeting {meeting.number} "
+            "changed during ResolvedMaterial update"
+        )
+
+def resolve_uploaded_material(meeting: Meeting, doc: Document):
+    resolved = []
+    blob = resolve_one_material(doc, rev=None, ext=None)
+    if blob is not None:
+        resolved.append(
+            ResolvedMaterial(
+                name=doc.name,
+                meeting_number=meeting.number,
+                bucket=blob.bucket,
+                blob=blob.name,
+            )
+        )
+    # request by doc name + rev
+    blob = resolve_one_material(doc, rev=doc.rev, ext=None)
+    if blob is not None:
+        resolved.append(
+            ResolvedMaterial(
+                name=f"{doc.name}-{doc.rev:02}",
+                meeting_number=meeting.number,
+                bucket=blob.bucket,
+                blob=blob.name,
+            )
+        )
+    ResolvedMaterial.objects.bulk_create(
+        resolved,
+        update_conflicts=True,
+        unique_fields=["name", "meeting_number"],
+        update_fields=["bucket", "blob"],
+    )
+
+
+def store_blob_for_one_material_file(doc: Document, rev: str, filepath: Path):
+    if not settings.ENABLE_BLOBSTORAGE:
+        raise RuntimeError("Cannot store blobs: ENABLE_BLOBSTORAGE is False")
+
+    bucket = doc.type_id
+    if bucket not in settings.MATERIALS_TYPES_SERVED_BY_WORKER:
+        raise ValueError(f"Bucket {bucket} not found for doc {doc.name}.")
+    blob_stem = f"{doc.name}-{rev}"
+    suffix = filepath.suffix  # includes leading "."
+
+    # Store the file
+    try:
+        file_bytes = filepath.read_bytes()
+    except Exception as err:
+        log(f"Failed to read {filepath}: {err}")
+        raise
+    with suppress(AlreadyExistsError):
+        store_bytes(
+            kind=bucket,
+            name= blob_stem + suffix,
+            content=file_bytes,
+            mtime=datetime.datetime.fromtimestamp(
+                filepath.stat().st_mtime,
+                tz=datetime.UTC,
+            ),
+            allow_overwrite=False,
+            doc_name=doc.name,
+            doc_rev=rev,
+        )
+
+    # Special case: pre-render markdown into HTML as .md.html
+    if suffix == ".md":
+        try:
+            markdown_source = file_bytes.decode("utf-8")
+        except UnicodeDecodeError as err:
+            log(f"Unable to decode {filepath} as UTF-8, treating as latin-1: {err}")
+            markdown_source = file_bytes.decode("latin-1")
+        # render the markdown
+        try:
+            html = render_to_string(
+                "minimal.html",
+                {
+                    "content": markdown.markdown(markdown_source),
+                    "title": blob_stem,
+                    "static_ietf_org": settings.STATIC_IETF_ORG,
+                },
+            )
+        except Exception as err:
+            log(f"Failed to render markdown for {filepath}: {err}")
+        else:
+            # Don't overwrite, but don't fail if the blob exists
+            with suppress(AlreadyExistsError):
+                store_str(
+                    kind=bucket,
+                    name=blob_stem + ".md.html",
+                    content=html,
+                    allow_overwrite=False,
+                    doc_name=doc.name,
+                    doc_rev=rev,
+                    content_type="text/html;charset=utf-8",
+                )
+
+
+def store_blobs_for_one_material_doc(doc: Document):
+    """Ensure that all files related to a materials Document are in the blob store"""
+    if doc.type_id not in settings.MATERIALS_TYPES_SERVED_BY_WORKER:
+        log(f"This method does not handle docs of type {doc.name}")
+        return
+
+    # Store files for current Document / rev
+    file_path = Path(doc.get_file_path())
+    base_name = Path(doc.get_base_name())
+    # .stem would remove directories, so use .with_suffix("")
+    base_name_stem = str(base_name.with_suffix(""))
+    if base_name_stem.endswith(".") and base_name.suffix == "":
+        # In Python 3.14, a trailing "." is a valid suffix, but in prior versions
+        # it is left as part of the stem. The suffix check ensures that either way,
+        # only a single "." will be removed.
+        base_name_stem = base_name_stem[:-1]
+    # Add any we find without the rev
+    for file_to_store in file_path.glob(base_name_stem + ".*"):
+        if not (file_to_store.is_file()):
+            continue
+        try:
+            store_blob_for_one_material_file(doc, doc.rev, file_to_store)
+        except Exception as err:
+            log(
+                f"Failed to store blob for {doc} rev {doc.rev} "
+                f"from {file_to_store}: {err}"
+            )
+
+    # Get other revisions
+    for rev in doc.revisions_by_newrevisionevent():
+        if rev == doc.rev:
+            continue  # already handled this
+
+        # Add some that have the rev
+        for file_to_store in file_path.glob(doc.name + f"-{rev}.*"):
+            if not file_to_store.is_file():
+                continue
+            try:
+                store_blob_for_one_material_file(doc, rev, file_to_store)
+            except Exception as err:
+                log(
+                    f"Failed to store blob for {doc} rev {rev} "
+                    f"from {file_to_store}: {err}"
+                )
+
+
+def store_blobs_for_one_meeting(meeting: Meeting):
+    meeting_documents = (
+        Document.objects.filter(
+            type_id__in=settings.MATERIALS_TYPES_SERVED_BY_WORKER
+        ).filter(
+            Q(session__meeting=meeting) | Q(proceedingsmaterial__meeting=meeting)
+        )
+    ).distinct()
+
+    for doc in meeting_documents:
+        store_blobs_for_one_material_doc(doc)
+
+
 def create_recording(session, url, title=None, user=None):
     '''
     Creates the Document type=recording, setting external_url and creating
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index fcc9312609..cf6fed596b 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -9,6 +9,7 @@
 import json
 import math
 import os
+
 import pytz
 import re
 import tarfile
@@ -27,10 +28,12 @@
 
 from django import forms
 from django.core.cache import caches
+from django.core.files.storage import storages
 from django.shortcuts import render, redirect, get_object_or_404
 from django.http import (HttpResponse, HttpResponseRedirect, HttpResponseForbidden,
                          HttpResponseNotFound, Http404, HttpResponseBadRequest,
-                         JsonResponse, HttpResponseGone, HttpResponseNotAllowed)
+                         JsonResponse, HttpResponseGone, HttpResponseNotAllowed,
+                         FileResponse)
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
@@ -48,18 +51,25 @@
 from django.views.decorators.cache import cache_page
 from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt
 from django.views.generic import RedirectView
+from rest_framework.status import HTTP_404_NOT_FOUND
 
 import debug                            # pyflakes:ignore
 
 from ietf.doc.fields import SearchableDocumentsField
 from ietf.doc.models import Document, State, DocEvent, NewRevisionDocEvent
-from ietf.doc.storage_utils import remove_from_storage, retrieve_bytes, store_file
+from ietf.doc.storage_utils import (
+    remove_from_storage,
+    retrieve_bytes,
+    store_file,
+)
 from ietf.group.models import Group
 from ietf.group.utils import can_manage_session_materials, can_manage_some_groups, can_manage_group
 from ietf.person.models import Person, User
 from ietf.ietfauth.utils import role_required, has_role, user_is_person
 from ietf.mailtrigger.utils import gather_address_lists
-from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, SessionPresentation, TimeSlot, SlideSubmission, Attended
+from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, \
+    SessionPresentation, TimeSlot, SlideSubmission, Attended
+from ..blobdb.models import ResolvedMaterial
 from ietf.meeting.models import ImportantDate, SessionStatusName, SchedulingEvent, SchedTimeSessAssignment, Room, TimeSlotTypeName
 from ietf.meeting.models import Registration
 from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm, ImportMinutesForm,
@@ -83,7 +93,8 @@
     finalize,
     generate_proceedings_content,
     organize_proceedings_sessions,
-    sort_accept_tuple,
+    resolve_uploaded_material,
+    sort_accept_tuple, store_blobs_for_one_material_doc,
 )
 from ietf.meeting.utils import add_event_info_to_session_qs
 from ietf.meeting.utils import session_time_for_sorting
@@ -120,6 +131,8 @@
 
 from icalendar import Calendar, Event
 from ietf.doc.templatetags.ietf_filters import absurl
+from ..api.ietf_utils import requires_api_token
+from ..blobdb.storage import BlobdbStorage, BlobFile
 
 request_summary_exclude_group_types = ['team']
 
@@ -245,21 +258,32 @@ def current_materials(request):
         raise Http404('No such meeting')
 
 
-def _get_materials_doc(meeting, name):
+def _get_materials_doc(name, meeting=None):
     """Get meeting materials document named by name
 
-    Raises Document.DoesNotExist if a match cannot be found.
+    Raises Document.DoesNotExist if a match cannot be found. If meeting is None,
+    matches a name that is associated with _any_ meeting.
     """
+
+    def _matches_meeting(doc, meeting=None):
+        if meeting is None:
+            return doc.get_related_meeting() is not None
+        return doc.get_related_meeting() == meeting
+
     # try an exact match first
     doc = Document.objects.filter(name=name).first()
-    if doc is not None and doc.get_related_meeting() == meeting:
+    if doc is not None and _matches_meeting(doc, meeting):
         return doc, None
+
     # try parsing a rev number
     if "-" in name:
         docname, rev = name.rsplit("-", 1)
         if len(rev) == 2 and rev.isdigit():
             doc = Document.objects.get(name=docname)  # may raise Document.DoesNotExist
-            if doc.get_related_meeting() == meeting and rev in doc.revisions_by_newrevisionevent():
+            if (
+                _matches_meeting(doc, meeting)
+                and rev in doc.revisions_by_newrevisionevent()
+            ):
                 return doc, rev
     # give up
     raise Document.DoesNotExist
@@ -277,7 +301,7 @@ def materials_document(request, document, num=None, ext=None):
     meeting = get_meeting(num, type_in=["ietf", "interim"])
     num = meeting.number
     try:
-        doc, rev = _get_materials_doc(meeting=meeting, name=document)
+        doc, rev = _get_materials_doc(name=document, meeting=meeting)
     except Document.DoesNotExist:
         raise Http404("No such document for meeting %s" % num)
 
@@ -320,6 +344,7 @@ def materials_document(request, document, num=None, ext=None):
                         {
                             "content": markdown.markdown(bytes.decode(encoding=chset)),
                             "title": filename.name,
+                            "static_ietf_org": settings.STATIC_IETF_ORG,
                         },
                     )
                     content_type = content_type.replace("plain", "html", 1)
@@ -334,6 +359,133 @@ def materials_document(request, document, num=None, ext=None):
         return HttpResponseRedirect(redirect_to=doc.get_href(meeting=meeting))
 
 
+@requires_api_token("ietf.meeting.views.api_resolve_materials_name")
+def api_resolve_materials_name_cached(request, document, num=None, ext=None):
+    """Resolve materials name into document to a blob spec
+
+    Returns the bucket/name of a blob in the blob store that corresponds to the named
+    document. Handles resolution of revision if it is not specified and determines the
+    best extension if one is not provided. Response is JSON.
+
+    As of 2025-10-10 we do not have blobs for all materials documents or for every
+    format of every document. This API still returns the bucket/name as if the blob
+    exists. Another API will allow the caller to obtain the file contents using that
+    name if it cannot be retrieved from the blob store.
+    """
+
+    def _error_response(status: int, detail: str):
+        return JsonResponse(
+            {
+                "status": status,
+                "title": "Error",
+                "detail": detail,
+            },
+            status=status,
+        )
+
+    def _response(bucket: str, name: str):
+        return JsonResponse(
+            {
+                "bucket": bucket,
+                "name": name,
+            }
+        )
+
+    try:
+        resolved = ResolvedMaterial.objects.get(
+            meeting_number=num, name=document
+        )
+    except ResolvedMaterial.DoesNotExist:
+        return _error_response(
+            HTTP_404_NOT_FOUND, f"No suitable file for {document} for meeting {num}"
+        )
+    return _response(bucket=resolved.bucket, name=resolved.blob)
+
+
+@requires_api_token
+def api_retrieve_materials_blob(request, bucket, name):
+    """Retrieve contents of a meeting materials blob
+    
+    This is intended as a fallback if the web worker cannot retrieve a blob from
+    the blobstore itself. The most likely cause is retrieving an old materials document
+    that has not been backfilled.
+    
+    If a blob is requested that does not exist, this checks for it on the filesystem
+    and if found, adds it to the blobstore, creates a StoredObject record, and returns
+    the contents as it would have done if the blob was already present.
+    
+    As a special case, if a requested file with extension `.md.html` does not exist
+    but a file with the same name but extension `.md` does, `.md` file will be rendered
+    from markdown to html and returned / stored.
+    """
+    DEFAULT_CONTENT_TYPES = {
+        ".html": "text/html;charset=utf-8",
+        ".md": "text/markdown;charset=utf-8",
+        ".pdf": "application/pdf",
+        ".txt": "text/plain;charset=utf-8",
+    }
+    
+    def _default_content_type(blob_name: str):
+        return DEFAULT_CONTENT_TYPES.get(Path(name).suffix, "application/octet-stream") 
+
+    if not (
+        settings.ENABLE_BLOBSTORAGE
+        and bucket in settings.MATERIALS_TYPES_SERVED_BY_WORKER
+    ):
+        return HttpResponseNotFound(f"Bucket {bucket} not found.")
+    storage = storages[bucket]  # if not configured, a server error will result
+    assert isinstance(storage, BlobdbStorage)
+    try:
+        blob = storage.open(name, "rb")
+    except FileNotFoundError:
+        pass
+    else:
+        # found the blob - return it
+        assert isinstance(blob, BlobFile)
+        return FileResponse(
+            blob,
+            filename=name,
+            content_type=blob.content_type or _default_content_type(name),
+        )
+    
+    # Did not find the blob. Create it if we can 
+    name_as_path = Path(name)
+    if name_as_path.suffixes == [".md", ".html"]:
+        # special case: .md.html means we want to create the .md and the .md.html
+        # will come along as a bonus
+        name_to_store = name_as_path.stem  # removes the .html
+    else:
+        name_to_store = name
+    
+    # See if we have a meeting-related document that matches the requested bucket and
+    # name.
+    try: 
+        doc, rev = _get_materials_doc(Path(name_to_store).stem)
+        if doc.type_id != bucket:
+            raise Document.DoesNotExist
+    except Document.DoesNotExist:
+        return HttpResponseNotFound(
+            f"Document corresponding to {bucket}:{name} not found."
+        )
+    else:
+        # create all missing blobs for the doc while we're at it
+        store_blobs_for_one_material_doc(doc)
+    
+    # If we can make the blob at all, it now exists, so return it or a 404
+    try:
+        blob = storage.open(name, "rb")
+    except FileNotFoundError:
+        return HttpResponseNotFound(f"Object {bucket}:{name} not found.")
+    else:
+        # found the blob - return it
+        assert isinstance(blob, BlobFile)
+        return FileResponse(
+            blob,
+            filename=name,
+            content_type=blob.content_type or _default_content_type(name),
+        )
+    
+
 @login_required
 def materials_editable_groups(request, num=None):
     meeting = get_meeting(num)
@@ -2949,6 +3101,7 @@ def upload_session_minutes(request, session_id, num):
                 form.add_error(None, str(err))
             else:
                 # no exception -- success!
+                resolve_uploaded_material(meeting=session.meeting, doc=session.minutes())
                 messages.success(request, f'Successfully uploaded minutes as revision {session.minutes().rev}.')
                 return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym)
     else:
@@ -3008,6 +3161,7 @@ def upload_session_narrativeminutes(request, session_id, num):
                 form.add_error(None, str(err))
             else:
                 # no exception -- success!
+                resolve_uploaded_material(meeting=session.meeting, doc=session.narrative_minutes())
                 messages.success(request, f'Successfully uploaded narrative minutes as revision {session.narrative_minutes().rev}.')
                 return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym)
     else:
@@ -3154,6 +3308,7 @@ def upload_session_agenda(request, session_id, num):
                 form.add_error(None, save_error)
             else:
                 doc.save_with_history([e])
+                resolve_uploaded_material(meeting=session.meeting, doc=doc)
                 messages.success(request, f'Successfully uploaded agenda as revision {doc.rev}.')
                 return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
     else: 
@@ -3337,6 +3492,7 @@ def upload_session_slides(request, session_id, num, name=None):
             else:
                 doc.save_with_history([e])
                 post_process(doc)
+                resolve_uploaded_material(meeting=session.meeting, doc=doc)
 
             # Send MeetEcho updates even if we had a problem saving - that will keep it in sync with the
             # SessionPresentation, which was already saved regardless of problems saving the file.
@@ -4737,6 +4893,7 @@ def err(code, text):
     write_doc_for_session(session, 'chatlog', filename, json.dumps(apidata['chatlog']))
     e = NewRevisionDocEvent.objects.create(doc=doc, rev=doc.rev, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev)
     doc.save_with_history([e])
+    resolve_uploaded_material(meeting=session.meeting, doc=doc)
     return HttpResponse(
         "Done",
         status=200,
@@ -4785,6 +4942,7 @@ def err(code, text):
     write_doc_for_session(session, 'polls', filename, json.dumps(apidata['polls']))
     e = NewRevisionDocEvent.objects.create(doc=doc, rev=doc.rev, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev)
     doc.save_with_history([e])
+    resolve_uploaded_material(meeting=session.meeting, doc=doc)
     return HttpResponse(
         "Done",
         status=200,
@@ -5167,6 +5325,7 @@ def approve_proposed_slides(request, slidesubmission_id, num):
                 doc.store_bytes(target_filename, retrieve_bytes("staging", submission.filename))
                 remove_from_storage("staging", submission.filename)
                 post_process(doc)
+                resolve_uploaded_material(meeting=submission.session.meeting, doc=doc)
                 DocEvent.objects.create(type="approved_slides", doc=doc, rev=doc.rev, by=request.user.person, desc="Slides approved")
 
                 # update meetecho slide info if configured
diff --git a/ietf/settings.py b/ietf/settings.py
index 5e576430ed..eb5f9d2161 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -786,29 +786,29 @@ def skip_unreadable_post(record):
 
 # Storages for artifacts stored as blobs
 ARTIFACT_STORAGE_NAMES: list[str] = [
-    "bofreq",
-    "charter",
-    "conflrev",
     "active-draft",
-    "draft",
-    "slides",
-    "minutes",
     "agenda",
+    "bibxml-ids",
     "bluesheets",
-    "procmaterials",
-    "narrativeminutes",
-    "statement",
-    "statchg",
-    "liai-att",
+    "bofreq",
+    "charter",
     "chatlog",
-    "polls",
-    "staging",
-    "bibxml-ids",
-    "indexes",
+    "conflrev",
+    "draft",
     "floorplan",
+    "indexes",
+    "liai-att",
     "meetinghostlogo",
+    "minutes",
+    "narrativeminutes",
     "photo",
+    "polls",
+    "procmaterials",
     "review",
+    "slides",
+    "staging",
+    "statchg",
+    "statement",
 ]
 for storagename in ARTIFACT_STORAGE_NAMES:
     STORAGES[storagename] = {
@@ -816,6 +816,20 @@ def skip_unreadable_post(record):
         "OPTIONS": {"bucket_name": storagename},
     }
 
+# Buckets / doc types of meeting materials the CF worker is allowed to serve. This
+# differs from the list in Session.meeting_related() by the omission of "recording"
+MATERIALS_TYPES_SERVED_BY_WORKER = [
+    "agenda",
+    "bluesheets",
+    "chatlog",
+    "minutes",
+    "narrativeminutes",
+    "polls",
+    "procmaterials",
+    "slides",
+]
+
+
 # Override this in settings_local.py if needed
 # *_PATH variables ends with a slash/ .
 
diff --git a/ietf/templates/minimal.html b/ietf/templates/minimal.html
index 87f661f501..15c432505e 100644
--- a/ietf/templates/minimal.html
+++ b/ietf/templates/minimal.html
@@ -9,8 +9,8 @@
         <meta http-equiv="X-UA-Compatible" content="IE=edge">
         <title>{{ title }}</title>
         <meta name="viewport" content="width=device-width, initial-scale=1">
-        <link href="{{ settings.STATIC_IETF_ORG }}/fonts/inter/import.css" rel="stylesheet">
-        <link href="{{ settings.STATIC_IETF_ORG }}/fonts/noto-sans-mono/import.css" rel="stylesheet">
+        <link href="{{ static_ietf_org }}/fonts/inter/import.css" rel="stylesheet">
+        <link href="{{ static_ietf_org }}/fonts/noto-sans-mono/import.css" rel="stylesheet">
         <link rel="stylesheet" href="{% static 'ietf/css/ietf.css' %}">
         {# load this in the head, to prevent flickering #}
         <script src="{% static 'ietf/js/theme.js' %}"></script>

From af0bcc743f6e449f93e0c7a7e4f2e2eec3ec76ae Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 23 Oct 2025 17:14:39 -0400
Subject: [PATCH 482/656] docs: Update PostgreSQL version badge in README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4e1b7e1a45..dfaf871052 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@
 [![Python Version](https://img.shields.io/badge/python-3.9-blue?logo=python&logoColor=white)](#prerequisites)
 [![Django Version](https://img.shields.io/badge/django-4.x-51be95?logo=django&logoColor=white)](#prerequisites)
 [![Node Version](https://img.shields.io/badge/node.js-16.x-green?logo=node.js&logoColor=white)](#prerequisites)
-[![MariaDB Version](https://img.shields.io/badge/postgres-16-blue?logo=postgresql&logoColor=white)](#prerequisites)
+[![MariaDB Version](https://img.shields.io/badge/postgres-17-blue?logo=postgresql&logoColor=white)](#prerequisites)
 
 ##### The day-to-day front-end to the IETF database for people who work on IETF standards.
 

From f9dea7df9d562ba818cf9224c1594f0e0983cdbe Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 23 Oct 2025 17:24:58 -0400
Subject: [PATCH 483/656] docs: Update Python version badge to 3.12 in README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index dfaf871052..baffc311e7 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@
 [![Release](https://img.shields.io/github/release/ietf-tools/datatracker.svg?style=flat&maxAge=300)](https://github.com/ietf-tools/datatracker/releases)
 [![License](https://img.shields.io/github/license/ietf-tools/datatracker)](https://github.com/ietf-tools/datatracker/blob/main/LICENSE)
 [![Code Coverage](https://codecov.io/gh/ietf-tools/datatracker/branch/feat/bs5/graph/badge.svg?token=V4DXB0Q28C)](https://codecov.io/gh/ietf-tools/datatracker)  
-[![Python Version](https://img.shields.io/badge/python-3.9-blue?logo=python&logoColor=white)](#prerequisites)
+[![Python Version](https://img.shields.io/badge/python-3.12-blue?logo=python&logoColor=white)](#prerequisites)
 [![Django Version](https://img.shields.io/badge/django-4.x-51be95?logo=django&logoColor=white)](#prerequisites)
 [![Node Version](https://img.shields.io/badge/node.js-16.x-green?logo=node.js&logoColor=white)](#prerequisites)
 [![MariaDB Version](https://img.shields.io/badge/postgres-17-blue?logo=postgresql&logoColor=white)](#prerequisites)

From e0691c17121d2324d812bc68c3943d963d1c5d4d Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 23 Oct 2025 17:30:50 -0400
Subject: [PATCH 484/656] ci: remove assets rsync sync job from
 dev-assets-sync-nightly workflow

Removed the nightly sync job for assets in the workflow.
---
 .github/workflows/dev-assets-sync-nightly.yml | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml
index 19933bddfd..4cfbf6365b 100644
--- a/.github/workflows/dev-assets-sync-nightly.yml
+++ b/.github/workflows/dev-assets-sync-nightly.yml
@@ -47,17 +47,3 @@ jobs:
           file: dev/shared-assets-sync/Dockerfile
           push: true
           tags: ghcr.io/ietf-tools/datatracker-rsync-assets:latest
-          
-  sync:
-    name: Run assets rsync
-    if: ${{ always() }}
-    runs-on: [self-hosted, dev-server]
-    needs: [build]
-    steps:
-    - name: Run rsync
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      run: |
-        docker pull ghcr.io/ietf-tools/datatracker-rsync-assets:latest
-        docker run --rm -v dt-assets:/assets ghcr.io/ietf-tools/datatracker-rsync-assets:latest
-        docker image prune -a -f

From 354d83d2fa22f817384a792bcbdef9757771f70a Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 23 Oct 2025 17:34:00 -0400
Subject: [PATCH 485/656] ci: remove sandbox-refresh workflow

---
 .github/workflows/sandbox-refresh.yml | 35 ---------------------------
 1 file changed, 35 deletions(-)
 delete mode 100644 .github/workflows/sandbox-refresh.yml

diff --git a/.github/workflows/sandbox-refresh.yml b/.github/workflows/sandbox-refresh.yml
deleted file mode 100644
index 3ddb119e4f..0000000000
--- a/.github/workflows/sandbox-refresh.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-name: Sandbox Refresh
-
-on:
-  # Run every night
-  schedule:
-   - cron: '0 9 * * *'
-  
-  workflow_dispatch:
-
-jobs:
-  main:
-    name: Refresh DBs
-    runs-on: [self-hosted, dev-server]
-    permissions:
-      contents: read
-      
-    steps:
-    - uses: actions/checkout@v4
-        
-    - name: Refresh DBs
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      run: |
-        echo "Install Deploy to Container CLI dependencies..."
-        cd dev/deploy-to-container
-        npm ci
-        cd ../..
-        echo "Start Refresh..."
-        node ./dev/deploy-to-container/refresh.js
-        
-    - name: Cleanup old docker resources
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      run: |
-        docker image prune -a -f

From 4e6168607cb49abc9341b27049f458bc9363297a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 23 Oct 2025 20:43:04 -0300
Subject: [PATCH 486/656] ci: proceedings cache cfg for prod/tests (#9784)

---
 ietf/settings_testcrawl.py |  4 +++-
 k8s/settings_local.py      | 11 +++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/ietf/settings_testcrawl.py b/ietf/settings_testcrawl.py
index a1b5ce8946..40744a228d 100644
--- a/ietf/settings_testcrawl.py
+++ b/ietf/settings_testcrawl.py
@@ -27,9 +27,11 @@
             'MAX_ENTRIES': 10000,
         },
     },
+    'proceedings': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+    },
     'sessions': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-        # No version-specific VERSION setting.
     },
     'htmlized': {
         'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index c09bd70c86..f8ffacc83f 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -301,6 +301,17 @@ def _multiline_to_list(s):
         "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
         "VERSION": __version__,
         "KEY_PREFIX": "ietf:dt",
+        # Key function is default except with sha384-encoded key
+        "KEY_FUNCTION": lambda key, key_prefix, version: (
+            f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+        ),
+    },
+    "proceedings": {
+        "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+        "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+        # No release-specific VERSION setting.
+        "KEY_PREFIX": "ietf:dt:proceedings",
+        # Key function is default except with sha384-encoded key
         "KEY_FUNCTION": lambda key, key_prefix, version: (
             f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
         ),

From 6db7d4afbe2b876192d0aa4a63a0bbe98a3806be Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 28 Oct 2025 20:06:53 -0300
Subject: [PATCH 487/656] fix: don't trust libmagic charset recognition (#9815)

---
 ietf/meeting/views.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index cf6fed596b..d6b5a1c0db 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -329,7 +329,7 @@ def materials_document(request, document, num=None, ext=None):
     old_proceedings_format = meeting.number.isdigit() and int(meeting.number) <= 96
     if settings.MEETING_MATERIALS_SERVE_LOCALLY or old_proceedings_format:
         bytes = filename.read_bytes()
-        mtype, chset = get_mime_type(bytes)
+        mtype, chset = get_mime_type(bytes)  # chset does not consider entire file!
         content_type = "%s; charset=%s" % (mtype, chset)
 
         if filename.suffix == ".md" and mtype == "text/plain":
@@ -339,15 +339,24 @@ def materials_document(request, document, num=None, ext=None):
                     content_type = content_type.replace("plain", "markdown", 1)
                     break
                 elif atype[0] == "text/html":
+                    # Render markdown, allowing that charset may be inaccurate.
+                    try:
+                        md_src = bytes.decode(
+                            "utf-8" if chset in ["ascii", "us-ascii"] else chset
+                        )
+                    except UnicodeDecodeError:
+                        # latin-1, aka iso8859-1, accepts all 8-bit code points
+                        md_src = bytes.decode("latin-1")
+                    content = markdown.markdown(md_src)  # a string
                     bytes = render_to_string(
                         "minimal.html",
                         {
-                            "content": markdown.markdown(bytes.decode(encoding=chset)),
+                            "content": content,
                             "title": filename.name,
                             "static_ietf_org": settings.STATIC_IETF_ORG,
                         },
-                    )
-                    content_type = content_type.replace("plain", "html", 1)
+                    ).encode("utf-8")
+                    content_type = "text/html; charset=utf-8"
                     break
                 elif atype[0] == "text/plain":
                     break

From 3e34efe74950d7f237171e9ea5cedc24d8d08615 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 28 Oct 2025 20:09:27 -0300
Subject: [PATCH 488/656] chore: update names fixture (#9807)

* chore(dev): update names fixture

* chore(dev): update names fixture again
---
 ietf/name/fixtures/names.json | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json
index 58deb01f0c..64e26e503a 100644
--- a/ietf/name/fixtures/names.json
+++ b/ietf/name/fixtures/names.json
@@ -650,7 +650,7 @@
   },
   {
     "fields": {
-      "desc": "<a href=\"https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.1\" target=\"_blank\">4.2.1. Call for Adoption by WG Issued</a>\r\n\r\n The \"Call for Adoption by WG Issued\" state should be used to indicate when an I-D is being considered for adoption by an IETF WG. An I-D that is in this state is actively being considered for adoption and has not yet achieved consensus, preference, or selection in the WG.\r\n\r\n This state may be used to describe an I-D that someone has asked a WG to consider for adoption, if the WG Chair has agreed with the request. This state may also be used to identify an I-D that a WG Chair asked an author to write specifically for consideration as a candidate WG item [WGDTSPEC], and/or an I-D that is listed as a 'candidate draft' in the WG's charter.\r\n\r\n Under normal conditions, it should not be possible for an I-D to be in the \"Call for Adoption by WG Issued\" state in more than one working group at the same time. This said, it is not uncommon for authors to \"shop\" their I-Ds to more than one WG at a time, with the hope of getting their documents adopted somewhere.\r\n\r\n After this state is implemented in the Datatracker, an I-D that is in the \"Call for Adoption by WG Issued\" state will not be able to be \"shopped\" to any other WG without the consent of the WG Chairs and the responsible ADs impacted by the shopping.\r\n\r\n Note that Figure 1 includes an arc leading from this state to outside of the WG state machine. This illustrates that some I-Ds that are considered do not get adopted as WG drafts. An I-D that is not adopted as a WG draft will transition out of the WG state machine and revert back to having no stream-specific state; however, the status change history log of the I-D will record that the I-D was previously in the \"Call for Adoption by WG Issued\" state.",
+      "desc": "A call for adoption of the individual submission document has been issued by the Working Group (WG) chairs.  This call is still running but the WG has not yet reached consensus for adoption.",
       "name": "Call For Adoption By WG Issued",
       "next_states": [
         36,
@@ -666,7 +666,7 @@
   },
   {
     "fields": {
-      "desc": "<a href=\"https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.2\" target=\"_blank\">4.2.2. Adopted by a WG</a>\r\n\r\n The \"Adopted by a WG\" state describes an individual submission I-D that an IETF WG has agreed to adopt as one of its WG drafts.\r\n\r\n WG Chairs who use this state will be able to clearly indicate when their WGs adopt individual submission I-Ds. This will facilitate the Datatracker's ability to correctly capture \"Replaces\" information for WG drafts and correct \"Replaced by\" information for individual submission I-Ds that have been replaced by WG drafts.\r\n\r\n This state is needed because the Datatracker uses the filename of an I-D as a key to search its database for status information about the I-D, and because the filename of a WG I-D is supposed to be different from the filename of an individual submission I-D. The filename of an individual submission I-D will typically be formatted as 'draft-author-wgname-topic-nn'.\r\n\r\n The filename of a WG document is supposed to be formatted as 'draft- ietf-wgname-topic-nn'.\r\n\r\n An individual I-D that is adopted by a WG may take weeks or months to be resubmitted by the author as a new (version-00) WG draft. If the \"Adopted by a WG\" state is not used, the Datatracker has no way to determine that an I-D has been adopted until a new version of the I-D is submitted to the WG by the author and until the I-D is approved for posting by a WG Chair.",
+      "desc": "The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted.",
       "name": "Adopted by a WG",
       "next_states": [
         38
@@ -681,7 +681,7 @@
   },
   {
     "fields": {
-      "desc": "<a href=\"https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.3\" target=\"_blank\">4.2.3. Adopted for WG Info Only</a>\r\n\r\n The \"Adopted for WG Info Only\" state describes a document that contains useful information for the WG that adopted it, but the document is not intended to be published as an RFC. The WG will not actively develop the contents of the I-D or progress it for publication as an RFC. The only purpose of the I-D is to provide information for internal use by the WG.",
+      "desc": "The document is adopted by the Working Group (WG) for its internal use.  The WG has decided that it will not pursue publication of it as an RFC.",
       "name": "Adopted for WG Info Only",
       "next_states": [],
       "order": 3,
@@ -694,7 +694,7 @@
   },
   {
     "fields": {
-      "desc": "<a href=\"https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.4\" target=\"_blank\">4.2.4. WG Document</a>\r\n\r\n The \"WG Document\" state describes an I-D that has been adopted by an IETF WG and is being actively developed.\r\n\r\n A WG Chair may transition an I-D into the \"WG Document\" state at any time as long as the I-D is not being considered or developed in any other WG.\r\n\r\n Alternatively, WG Chairs may rely upon new functionality to be added to the Datatracker to automatically move version-00 drafts into the \"WG Document\" state as described in Section 4.1.\r\n\r\n Under normal conditions, it should not be possible for an I-D to be in the \"WG Document\" state in more than one WG at a time. This said, I-Ds may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs.",
+      "desc": "The document has been adopted by the Working Group (WG) and is under development.  A document can only be adopted by one WG at a time.  However, a document may be transferred between WGs.",
       "name": "WG Document",
       "next_states": [
         39,
@@ -712,7 +712,7 @@
   },
   {
     "fields": {
-      "desc": "<a href=\"https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.5\" target=\"_blank\">4.2.5. Parked WG Document</a>\r\n\r\n A \"Parked WG Document\" is an I-D that has lost its author or editor, is waiting for another document to be written or for a review to be completed, or cannot be progressed by the working group for some other reason.\r\n\r\n Some of the annotation tags described in Section 4.3 may be used in conjunction with this state to indicate why an I-D has been parked, and/or what may need to happen for the I-D to be un-parked.\r\n\r\n Parking a WG draft will not prevent it from expiring; however, this state can be used to indicate why the I-D has stopped progressing in the WG.\r\n\r\n A \"Parked WG Document\" that is not expired may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs.",
+      "desc": "The Working Group (WG) document is in a temporary state where it will not be actively developed.  The reason for the pause is explained via a datatracker comments section.",
       "name": "Parked WG Document",
       "next_states": [
         38
@@ -727,7 +727,7 @@
   },
   {
     "fields": {
-      "desc": "<a href=\"https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.6\" target=\"_blank\">4.2.6. Dead WG Document</a>\r\n\r\n A \"Dead WG Document\" is an I-D that has been abandoned. Note that 'Dead' is not always a final state for a WG I-D. If consensus is subsequently achieved, a \"Dead WG Document\" may be resurrected. A \"Dead WG Document\" that is not resurrected will eventually expire.\r\n\r\n Note that an I-D that is declared to be \"Dead\" in one WG and that is not expired may be transferred to a non-dead state in another WG with the consent of the WG Chairs and the responsible ADs.",
+      "desc": "The Working Group (WG) document has been abandoned by the WG.  No further development is planned in this WG.  A decision to resume work on this document and move it out of this state is possible.",
       "name": "Dead WG Document",
       "next_states": [
         38
@@ -742,7 +742,7 @@
   },
   {
     "fields": {
-      "desc": "<a href=\"https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.7\" target=\"_blank\">4.2.7. In WG Last Call</a>\r\n\r\n A document \"In WG Last Call\" is an I-D for which a WG Last Call (WGLC) has been issued and is in progress.\r\n\r\n Note that conducting a WGLC is an optional part of the IETF WG process, per Section 7.4 of RFC 2418 [RFC2418].\r\n\r\n If a WG Chair decides to conduct a WGLC on an I-D, the \"In WG Last Call\" state can be used to track the progress of the WGLC. The Chair may configure the Datatracker to send a WGLC message to one or more mailing lists when the Chair moves the I-D into this state. The WG Chair may also be able to select a different set of mailing lists for a different document undergoing a WGLC; some documents may deserve coordination with other WGs.\r\n\r\n A WG I-D in this state should remain \"In WG Last Call\" until the WG Chair moves it to another state. The WG Chair may configure the Datatracker to send an e-mail after a specified period of time to remind or 'nudge' the Chair to conclude the WGLC and to determine the next state for the document.\r\n\r\n It is possible for one WGLC to lead into another WGLC for the same document. For example, an I-D that completed a WGLC as an \"Informational\" document may need another WGLC if a decision is taken to convert the I-D into a Standards Track document.",
+      "desc": "The Working Group (WG) document is currently subject to an active WG Last Call (WGLC) review per Section 7.4 of RFC2418.",
       "name": "In WG Last Call",
       "next_states": [
         38,
@@ -759,7 +759,7 @@
   },
   {
     "fields": {
-      "desc": "<a href=\"https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.8\" target=\"_blank\">4.2.8. Waiting for WG Chair Go-Ahead</a>\r\n\r\n A WG Chair may wish to place an I-D that receives a lot of comments during a WGLC into the \"Waiting for WG Chair Go-Ahead\" state. This state describes an I-D that has undergone a WGLC; however, the Chair is not yet ready to call consensus on the document.\r\n\r\n If comments from the WGLC need to be responded to, or a revision to the I-D is needed, the Chair may place an I-D into this state until all of the WGLC comments are adequately addressed and the (possibly revised) document is in the I-D repository.",
+      "desc": "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed",
       "name": "Waiting for WG Chair Go-Ahead",
       "next_states": [
         41,
@@ -775,7 +775,7 @@
   },
   {
     "fields": {
-      "desc": "<a href=\"https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.9\" target=\"_blank\">4.2.9. WG Consensus: Waiting for Writeup</a>\r\n\r\n A document in the \"WG Consensus: Waiting for Writeup\" state has essentially completed its development within the working group, and is nearly ready to be sent to the IESG for publication. The last thing to be done is the preparation of a protocol writeup by a Document Shepherd. The IESG requires that a document shepherd writeup be completed before publication of the I-D is requested. The IETF document shepherding process and the role of a WG Document Shepherd is described in RFC 4858 [RFC4858]\r\n\r\n A WG Chair may call consensus on an I-D without a formal WGLC and transition an I-D that was in the \"WG Document\" state directly into this state.\r\n\r\n The name of this state includes the words \"Waiting for Writeup\" because a good document shepherd writeup takes time to prepare.",
+      "desc": "The Working Group (WG) document has consensus to proceed to publication.  However, the document is waiting for a document shepherd write-up per RFC4858.",
       "name": "WG Consensus: Waiting for Write-Up",
       "next_states": [
         44
@@ -790,7 +790,7 @@
   },
   {
     "fields": {
-      "desc": "<a href=\"https://www.rfc-editor.org/rfc/rfc6174.html#section-4.2.10\" target=\"_blank\">4.2.10. Submitted to IESG for Publication</a>\r\n\r\n This state describes a WG document that has been submitted to the IESG for publication and that has not been sent back to the working group for revision.\r\n\r\n An I-D in this state may be under review by the IESG, it may have been approved and be in the RFC Editor's queue, or it may have been published as an RFC. Other possibilities exist too. The document may be \"Dead\" (in the IESG state machine) or in a \"Do Not Publish\" state.",
+      "desc": "The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication.  See the “IESG State” or “RFC Editor State” for further details on the state of the document.",
       "name": "Submitted to IESG for Publication",
       "next_states": [
         38
@@ -2020,7 +2020,7 @@
   },
   {
     "fields": {
-      "desc": "The document has been marked as a candidate for WG adoption by the WG Chair.  This state can be used before a call for adoption is issued (and the document is put in the \"Call For Adoption By WG Issued\" state), to indicate that the document is in the queue for a call for adoption, even if none has been issued yet.",
+      "desc": "The individual submission document has been marked by the Working Group (WG) chairs as a candidate for adoption by the WG, but no adoption call has been started.",
       "name": "Candidate for WG Adoption",
       "next_states": [
         35
@@ -2152,7 +2152,7 @@
   },
   {
     "fields": {
-      "desc": "In some areas, it can be desirable to wait for multiple interoperable implementations before progressing a draft to be an RFC, and in some WGs this is required.  This state should be entered after WG Last Call has completed.",
+      "desc": "The progression of this Working Group (WG) document towards publication is paused as it awaits implementation. The process governing the approach to implementations is WG-specific.",
       "name": "Waiting for Implementation",
       "next_states": [],
       "order": 8,
@@ -2165,7 +2165,7 @@
   },
   {
     "fields": {
-      "desc": "Held by WG, see document history for details.",
+      "desc": "Held by Working Group (WG) chairs for administrative reasons.  See document history for details.",
       "name": "Held by WG",
       "next_states": [],
       "order": 9,
@@ -4473,6 +4473,7 @@
       ],
       "session_purposes": [
         "coding",
+        "open_meeting",
         "presentation",
         "social",
         "tutorial"
@@ -5535,7 +5536,6 @@
       ],
       "desc": "Recipients for a message when a new incoming liaison statement is posted",
       "to": [
-        "liaison_from_contact",
         "liaison_to_contacts"
       ]
     },

From 145b9f76c19030b67628432b5f811a1c3c55c749 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 28 Oct 2025 20:11:52 -0300
Subject: [PATCH 489/656] chore(dev): bump dev blobdb to pg17 (#9806)

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 8c6e0ea486..2440faf121 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -116,7 +116,7 @@ services:
             - "minio-data:/data"
 
     blobdb:
-        image: postgres:16
+        image: postgres:17
         restart: unless-stopped
         environment:
             POSTGRES_DB: blob

From cbb0e2e3db4cc9e591b4397b7bc6cdebb51cfc8c Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 29 Oct 2025 11:18:47 -0300
Subject: [PATCH 490/656] feat: logs in api_retrieve_materials_blob() (#9818)

---
 ietf/meeting/views.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index d6b5a1c0db..69635d6219 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -451,6 +451,7 @@ def _default_content_type(blob_name: str):
     else:
         # found the blob - return it
         assert isinstance(blob, BlobFile)
+        log(f"Materials blob: directly returning {bucket}:{name}")
         return FileResponse(
             blob,
             filename=name,
@@ -473,17 +474,20 @@ def _default_content_type(blob_name: str):
         if doc.type_id != bucket:
             raise Document.DoesNotExist
     except Document.DoesNotExist:
+        log(f"Materials blob: no doc for {bucket}:{name}")
         return HttpResponseNotFound(
             f"Document corresponding to {bucket}:{name} not found."
         )
     else:
         # create all missing blobs for the doc while we're at it
+        log(f"Materials blob: storing blobs for {doc.name}-{doc.rev}")
         store_blobs_for_one_material_doc(doc)
     
     # If we can make the blob at all, it now exists, so return it or a 404
     try:
         blob = storage.open(name, "rb")
     except FileNotFoundError:
+        log(f"Materials blob: no blob for {bucket}:{name}")
         return HttpResponseNotFound(f"Object {bucket}:{name} not found.")
     else:
         # found the blob - return it

From c47fe34b0e409f4811e2f96fc45ec87bc1b7931f Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 3 Nov 2025 09:05:30 -0500
Subject: [PATCH 491/656] fix: include punctuation when tablesorting (#9855)

---
 ietf/static/js/list.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/static/js/list.js b/ietf/static/js/list.js
index 756a75001a..c03368cd72 100644
--- a/ietf/static/js/list.js
+++ b/ietf/static/js/list.js
@@ -16,7 +16,7 @@ function text_sort(a, b, options) {
     // sort by text content
     return prep(a, options).localeCompare(prep(b, options), "en", {
         sensitivity: "base",
-        ignorePunctuation: true,
+        ignorePunctuation: false,
         numeric: true
     });
 }

From 87c3a9db06b784d2cf1484a547171a9783e50fdc Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Mon, 3 Nov 2025 09:08:53 -0500
Subject: [PATCH 492/656] feat(agenda): Show calendar links to all the events
 (#9843)

* feat(agenda): Show calendar links to all the events

* test: Update playwright tests
---
 client/agenda/AgendaScheduleList.vue    | 20 ++++++++++----------
 playwright/tests/meeting/agenda.spec.js |  7 ++++++-
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue
index fc8b5fd30f..bbe5dfee8b 100644
--- a/client/agenda/AgendaScheduleList.vue
+++ b/client/agenda/AgendaScheduleList.vue
@@ -398,16 +398,6 @@ const meetingEvents = computed(() => {
             color: 'teal'
           })
         }
-        // -> Calendar item
-        if (item.links.calendar) {
-          links.push({
-            id: `lnk-${item.id}-calendar`,
-            label: 'Calendar (.ics) entry for this session',
-            icon: 'calendar-check',
-            href: item.links.calendar,
-            color: 'pink'
-          })
-        }
       } else {
         // -> Post event
         if (meetingNumberInt >= 60) {
@@ -484,6 +474,16 @@ const meetingEvents = computed(() => {
         }
       }
     }
+    // Add Calendar item for all events that has a calendar link
+    if (item.adjustedEnd > current && item.links.calendar) {
+      links.push({
+        id: `lnk-${item.id}-calendar`,
+        label: 'Calendar (.ics) entry for this session',
+        icon: 'calendar-check',
+        href: item.links.calendar,
+        color: 'pink'
+      })
+    }
 
     // Event icon
     let icon = null
diff --git a/playwright/tests/meeting/agenda.spec.js b/playwright/tests/meeting/agenda.spec.js
index 412a3fe9b8..2248027a38 100644
--- a/playwright/tests/meeting/agenda.spec.js
+++ b/playwright/tests/meeting/agenda.spec.js
@@ -1219,7 +1219,12 @@ test.describe('future - desktop', () => {
             await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar > i.bi`)).toBeVisible()
           }
         } else {
-          await expect(eventButtons).toHaveCount(0)
+          if (event.links.calendar) {
+            await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar`)).toHaveAttribute('href', event.links.calendar)
+            await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar > i.bi`)).toBeVisible()
+          } else {
+            await expect(eventButtons).toHaveCount(0)
+          }
         }
       }
     }

From 8da45cb8488345a1f449e6fc7442098cff81e3ff Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 3 Nov 2025 09:10:59 -0500
Subject: [PATCH 493/656] feat: optionally hide room-only schedule diffs
 (#9861)

* feat: optionally hide room-only schedule diffs

* test: update test
---
 ietf/meeting/tests_views.py | 194 +++++++++++++++++++++++++-----------
 ietf/meeting/views.py       |  13 +++
 2 files changed, 151 insertions(+), 56 deletions(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index b1bbc62907..50960b5143 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -49,7 +49,11 @@
 from ietf.meeting.helpers import send_interim_minutes_reminder, populate_important_dates, update_important_dates
 from ietf.meeting.models import Session, TimeSlot, Meeting, SchedTimeSessAssignment, Schedule, SessionPresentation, SlideSubmission, SchedulingEvent, Room, Constraint, ConstraintName
 from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting, make_interim_test_data
-from ietf.meeting.utils import condition_slide_order, generate_proceedings_content
+from ietf.meeting.utils import (
+    condition_slide_order,
+    generate_proceedings_content,
+    diff_meeting_schedules,
+)
 from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting
 from ietf.meeting.utils import create_recording, delete_recording, get_next_sequence, bluesheet_data
 from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule
@@ -4765,73 +4769,151 @@ def test_list_schedules(self):
         self.assertTrue(r.status_code, 200)
 
     def test_diff_schedules(self):
-        meeting = make_meeting_test_data()
-
-        url = urlreverse('ietf.meeting.views.diff_schedules',kwargs={'num':meeting.number})
-        login_testing_unauthorized(self,"secretary", url)
-        r = self.client.get(url)
-        self.assertTrue(r.status_code, 200)
-
-        from_schedule = Schedule.objects.get(meeting=meeting, name="test-unofficial-schedule")
-
-        session1 = Session.objects.filter(meeting=meeting, group__acronym='mars').first()
-        session2 = Session.objects.filter(meeting=meeting, group__acronym='ames').first()
-        session3 = SessionFactory(meeting=meeting, group=Group.objects.get(acronym='mars'),
-                                  attendees=10, requested_duration=datetime.timedelta(minutes=70),
-                                  add_to_schedule=False)
-        SchedulingEvent.objects.create(session=session3, status_id='schedw', by=Person.objects.first())
-
-        slot2 = TimeSlot.objects.filter(meeting=meeting, type='regular').order_by('-time').first()
-        slot3 = TimeSlot.objects.create(
-            meeting=meeting, type_id='regular', location=slot2.location,
-            duration=datetime.timedelta(minutes=60),
-            time=slot2.time + datetime.timedelta(minutes=60),
+        # Create meeting and some time slots
+        meeting = MeetingFactory(type_id="ietf", populate_schedule=False)
+        rooms = RoomFactory.create_batch(2, meeting=meeting)
+        # first index is room, second is time
+        timeslots = [
+            [
+                TimeSlotFactory(
+                    location=room,
+                    meeting=meeting,
+                    time=datetime.datetime.combine(
+                        meeting.date, datetime.time(9, 0, tzinfo=datetime.UTC)
+                    )
+                ),
+                TimeSlotFactory(
+                    location=room,
+                    meeting=meeting,
+                    time=datetime.datetime.combine(
+                        meeting.date, datetime.time(10, 0, tzinfo=datetime.UTC)
+                    )
+                ),
+                TimeSlotFactory(
+                    location=room,
+                    meeting=meeting,
+                    time=datetime.datetime.combine(
+                        meeting.date, datetime.time(11, 0, tzinfo=datetime.UTC)
+                    )
+                ),
+            ]
+            for room in rooms
+        ]
+        sessions = SessionFactory.create_batch(
+            5, meeting=meeting, add_to_schedule=False
         )
 
-        # copy
-        new_url = urlreverse("ietf.meeting.views.new_meeting_schedule", kwargs=dict(num=meeting.number, owner=from_schedule.owner_email(), name=from_schedule.name))
-        r = self.client.post(new_url, {
-            'name': "newtest",
-            'public': "on",
-        })
-        self.assertNoFormPostErrors(r)
+        from_schedule = ScheduleFactory(meeting=meeting)
+        to_schedule = ScheduleFactory(meeting=meeting)
 
-        to_schedule = Schedule.objects.get(meeting=meeting, name='newtest')
+        # sessions[0]: not scheduled in from_schedule, scheduled in to_schedule
+        SchedTimeSessAssignment.objects.create(
+            schedule=to_schedule,
+            session=sessions[0],
+            timeslot=timeslots[0][0],
+        )
+        # sessions[1]: scheduled in from_schedule, not scheduled in to_schedule
+        SchedTimeSessAssignment.objects.create(
+            schedule=from_schedule,
+            session=sessions[1],
+            timeslot=timeslots[0][0],
+        )
+        # sessions[2]: moves rooms, not time
+        SchedTimeSessAssignment.objects.create(
+            schedule=from_schedule,
+            session=sessions[2],
+            timeslot=timeslots[0][1],
+        )
+        SchedTimeSessAssignment.objects.create(
+            schedule=to_schedule,
+            session=sessions[2],
+            timeslot=timeslots[1][1],
+        )
+        # sessions[3]: moves time, not room
+        SchedTimeSessAssignment.objects.create(
+            schedule=from_schedule,
+            session=sessions[3],
+            timeslot=timeslots[1][1],
+        )
+        SchedTimeSessAssignment.objects.create(
+            schedule=to_schedule,
+            session=sessions[3],
+            timeslot=timeslots[1][2],
+        )
+        # sessions[4]: moves room and time
+        SchedTimeSessAssignment.objects.create(
+            schedule=from_schedule,
+            session=sessions[4],
+            timeslot=timeslots[1][0],
+        )
+        SchedTimeSessAssignment.objects.create(
+            schedule=to_schedule,
+            session=sessions[4],
+            timeslot=timeslots[0][2],
+        )
 
-        # make some changes
+        # Check the raw diffs
+        raw_diffs = diff_meeting_schedules(from_schedule, to_schedule)
+        self.assertCountEqual(
+            raw_diffs,
+            [
+                {
+                    "change": "schedule",
+                    "session": sessions[0].pk,
+                    "to": timeslots[0][0].pk,
+                },
+                {
+                    "change": "unschedule",
+                    "session": sessions[1].pk,
+                    "from": timeslots[0][0].pk,
+                },
+                {
+                    "change": "move",
+                    "session": sessions[2].pk,
+                    "from": timeslots[0][1].pk,
+                    "to": timeslots[1][1].pk,
+                },
+                {
+                    "change": "move",
+                    "session": sessions[3].pk,
+                    "from": timeslots[1][1].pk,
+                    "to": timeslots[1][2].pk,
+                },
+                {
+                    "change": "move",
+                    "session": sessions[4].pk,
+                    "from": timeslots[1][0].pk,
+                    "to": timeslots[0][2].pk,
+                },
+            ]
+        )
 
-        edit_url = urlreverse("ietf.meeting.views.edit_meeting_schedule", kwargs=dict(num=meeting.number, owner=to_schedule.owner_email(), name=to_schedule.name))
+        # Check the view
+        url = urlreverse("ietf.meeting.views.diff_schedules",
+                         kwargs={"num": meeting.number})
+        login_testing_unauthorized(self, "secretary", url)
+        r = self.client.get(url)
+        self.assertTrue(r.status_code, 200)
 
-        # schedule session
-        r = self.client.post(edit_url, {
-            'action': 'assign',
-            'timeslot': slot3.pk,
-            'session': session3.pk,
-        })
-        self.assertEqual(json.loads(r.content)['success'], True)
-        # unschedule session
-        r = self.client.post(edit_url, {
-            'action': 'unassign',
-            'session': session1.pk,
-        })
-        self.assertEqual(json.loads(r.content)['success'], True)
-        # move session
-        r = self.client.post(edit_url, {
-            'action': 'assign',
-            'timeslot': slot2.pk,
-            'session': session2.pk,
+        # with show room changes disabled - does not show sessions[2] because it did 
+        # not change time
+        r = self.client.get(url, {
+            "from_schedule": from_schedule.name,
+            "to_schedule": to_schedule.name,
         })
-        self.assertEqual(json.loads(r.content)['success'], True)
+        self.assertTrue(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q(".schedule-diffs tr")), 4 + 1)
 
-        # now get differences
+        # with show room changes enabled - shows all changes
         r = self.client.get(url, {
-            'from_schedule': from_schedule.name,
-            'to_schedule': to_schedule.name,
+            "from_schedule": from_schedule.name,
+            "to_schedule": to_schedule.name,
+            "show_room_changes": "on",
         })
         self.assertTrue(r.status_code, 200)
-
         q = PyQuery(r.content)
-        self.assertEqual(len(q(".schedule-diffs tr")), 3+1)
+        self.assertEqual(len(q(".schedule-diffs tr")), 5 + 1)
 
     def test_delete_schedule(self):
         url = urlreverse('ietf.meeting.views.delete_schedule',
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 69635d6219..b0c46cb05a 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -1675,6 +1675,11 @@ def list_schedules(request, num):
 class DiffSchedulesForm(forms.Form):
     from_schedule = forms.ChoiceField()
     to_schedule = forms.ChoiceField()
+    show_room_changes = forms.BooleanField(
+        initial=False,
+        required=False,
+        help_text="Include changes to room without a date or time change",
+    )
 
     def __init__(self, meeting, user, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -1707,6 +1712,14 @@ def diff_schedules(request, num):
             raw_diffs = diff_meeting_schedules(from_schedule, to_schedule)
 
             diffs = prefetch_schedule_diff_objects(raw_diffs)
+            if not form.cleaned_data["show_room_changes"]:
+                # filter out room-only changes
+                diffs = [
+                    d
+                    for d in diffs
+                    if (d["change"] != "move") or (d["from"].time != d["to"].time)
+                ]
+
             for d in diffs:
                 s = d['session']
                 s.session_label = s.short_name

From 9546e15224df7d8d9f385a8f670cd27012d7aee5 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 3 Nov 2025 09:11:32 -0500
Subject: [PATCH 494/656] fix: no autoescape for bluesheet template (#9858)

---
 ietf/templates/meeting/bluesheet.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ietf/templates/meeting/bluesheet.txt b/ietf/templates/meeting/bluesheet.txt
index dd3bf36ac7..5b3960f3aa 100644
--- a/ietf/templates/meeting/bluesheet.txt
+++ b/ietf/templates/meeting/bluesheet.txt
@@ -1,7 +1,8 @@
-Bluesheet for {{session}}
+{% autoescape off %}Bluesheet for {{session}}
 ========================================================================
 
 {{ data|length }} attendees.
 
 {% for item in data %}
 {{ item.name }}	{{ item.affiliation }}{% endfor %}
+{% endautoescape %}

From 7b4035d7fcd1130cdf8e08b3aa54efda35087a8a Mon Sep 17 00:00:00 2001
From: Tero Kivinen <kivinen@iki.fi>
Date: Mon, 3 Nov 2025 18:16:33 +0200
Subject: [PATCH 495/656] fix: Change add period button to save new period.
 (#9847)

---
 ietf/templates/group/change_reviewer_settings.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/templates/group/change_reviewer_settings.html b/ietf/templates/group/change_reviewer_settings.html
index 9ecec5633c..75451fdd75 100644
--- a/ietf/templates/group/change_reviewer_settings.html
+++ b/ietf/templates/group/change_reviewer_settings.html
@@ -89,7 +89,7 @@ <h2 class="mt-5">Unavailable periods</h2>
             <button type="submit"
                     class="btn btn-primary"
                     name="action"
-                    value="add_period">Add period</button>
+                    value="add_period">Save new period</button>
         </form>
     </div>
     <h2 class="mt-5">History of settings</h2>

From 1ba63977c00121572048c506289f88d41ce67291 Mon Sep 17 00:00:00 2001
From: Matthew Holloway <matthew@holloway.co.nz>
Date: Tue, 4 Nov 2025 06:26:25 +1300
Subject: [PATCH 496/656] fix: ask google not to index noscript content (#9844)

---
 ietf/templates/base.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/templates/base.html b/ietf/templates/base.html
index aa44955527..d8ff85f86e 100644
--- a/ietf/templates/base.html
+++ b/ietf/templates/base.html
@@ -96,7 +96,7 @@
                     </div>
                 {% endif %}
                 <div class="col mx-lg-3 ietf-auto-nav" id="content">
-                    <noscript>
+                    <noscript data-nosnippet>
                         <div class="alert alert-danger alert-ignore my-3">
                             <b>Javascript disabled?</b> Like other modern websites, the IETF Datatracker relies on Javascript.
                             Please enable Javascript for full functionality.

From 8d876c0bd422de4f297a4064119b0377e9dde659 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 3 Nov 2025 17:50:39 -0500
Subject: [PATCH 497/656] feat: remove restrictions on sending incoming
 liaisons (#9859)

* feat: remove restrictions on sending incoming liaisons

* chore: remove flake and ruff imports
---
 ietf/liaisons/forms.py     | 16 +---------------
 ietf/liaisons/tests.py     | 33 ---------------------------------
 ietf/liaisons/views.py     | 36 ++++++++++--------------------------
 ietf/static/js/liaisons.js | 13 +------------
 4 files changed, 12 insertions(+), 86 deletions(-)

diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py
index 1747e55571..6ceda5ad38 100644
--- a/ietf/liaisons/forms.py
+++ b/ietf/liaisons/forms.py
@@ -23,7 +23,7 @@
 from ietf.liaisons.fields import SearchableLiaisonStatementsField
 from ietf.liaisons.models import (LiaisonStatement,
                                   LiaisonStatementEvent, LiaisonStatementAttachment, LiaisonStatementPurposeName)
-from ietf.liaisons.utils import get_person_for_user, is_authorized_individual, OUTGOING_LIAISON_ROLES, \
+from ietf.liaisons.utils import get_person_for_user, OUTGOING_LIAISON_ROLES, \
     INCOMING_LIAISON_ROLES
 from ietf.liaisons.widgets import ButtonWidget, ShowAttachmentsWidget
 from ietf.name.models import DocRelationshipName
@@ -466,25 +466,11 @@ def set_to_fields(self):
         assert NotImplemented
 
 class IncomingLiaisonForm(LiaisonModelForm):
-    def clean(self):
-        if 'send' in list(self.data.keys()) and self.get_post_only():
-            raise forms.ValidationError('As an IETF Liaison Manager you can not send incoming liaison statements, you only can post them')
-        return super(IncomingLiaisonForm, self).clean()
 
     def is_approved(self):
         '''Incoming Liaison Statements do not required approval'''
         return True
 
-    def get_post_only(self):
-        from_groups = self.cleaned_data.get("from_groups")
-        if (
-            has_role(self.user, "Secretariat")
-            or has_role(self.user, "Liaison Coordinator")
-            or is_authorized_individual(self.user, from_groups)
-        ):
-            return False
-        return True
-
     def set_from_fields(self):
         """Configure from "From" fields based on user roles"""
         qs = external_groups_for_person(self.person)
diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py
index 5478f6c302..1209d7f401 100644
--- a/ietf/liaisons/tests.py
+++ b/ietf/liaisons/tests.py
@@ -203,7 +203,6 @@ def test_ajax(self):
         self.assertEqual(r.status_code, 200)
         data = r.json()
         self.assertEqual(data["error"], False)
-        self.assertEqual(data["post_only"], False)
         self.assertTrue('cc' in data)
         self.assertTrue('needs_approval' in data)
         self.assertTrue('to_contacts' in data)
@@ -871,38 +870,6 @@ def test_add_outgoing_liaison(self):
         self.assertTrue("Liaison Statement" in outbox[-1]["Subject"])
         self.assertTrue('aread@' in outbox[-1]['To'])
         self.assertTrue(submitter.email_address(), outbox[-1]['Cc'])
-        
-
-    def test_add_outgoing_liaison_unapproved_post_only(self):
-        RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman')
-        mars = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group
-        RoleFactory(name_id='ad',group=mars)
-
-        url = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'outgoing'})
-        login_testing_unauthorized(self, "secretary", url)
-
-        # add new
-        mailbox_before = len(outbox)
-        from_group = Group.objects.get(acronym="mars")
-        to_group = Group.objects.filter(type="sdo")[0]
-        submitter = Person.objects.get(user__username="marschairman")
-        today = date_today(datetime.UTC)
-        r = self.client.post(url,
-                             dict(from_groups=str(from_group.pk),
-                                  from_contact=submitter.email_address(),
-                                  to_groups=str(to_group.pk),
-                                  to_contacts='to_contacts@example.com',
-                                  approved="",
-                                  purpose="info",
-                                  title="title",
-                                  submitted_date=today.strftime("%Y-%m-%d"),
-                                  body="body",
-                                  post_only="1",
-                                  ))
-        self.assertEqual(r.status_code, 302)
-        l = LiaisonStatement.objects.all().order_by("-id")[0]
-        self.assertEqual(l.state.slug,'pending')
-        self.assertEqual(len(outbox), mailbox_before + 1)
 
     def test_liaison_add_attachment(self):
         liaison = LiaisonStatementFactory(deadline=date_today(DEADLINE_TZINFO)+datetime.timedelta(days=1))
diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py
index f54a023357..e2abff52ac 100644
--- a/ietf/liaisons/views.py
+++ b/ietf/liaisons/views.py
@@ -118,23 +118,6 @@ def normalize_sort(request):
 
     return sort, order_by
 
-def post_only(group,person):
-    '''Returns true if the user is restricted to post_only (vs. post_and_send) for this
-    group.  This is for incoming liaison statements.
-    - Secretariat have full access.
-    - Authorized Individuals have full access for the group they are associated with
-    - Liaison Managers can post only
-    '''
-    if group.type_id == "sdo" and (
-        not (
-            has_role(person.user, "Secretariat")
-            or has_role(person.user, "Liaison Coordinator")
-            or group.role_set.filter(name="auth", person=person)
-        )
-    ):
-        return True
-    else:
-        return False
 
 # -------------------------------------------------
 # Ajax Functions
@@ -159,15 +142,13 @@ def ajax_get_liaison_info(request):
 
     cc = []
     does_need_approval = []
-    can_post_only = []
     to_contacts = []
     response_contacts = []
-    result = {'response_contacts':[],'to_contacts': [], 'cc': [], 'needs_approval': False, 'post_only': False, 'full_list': []}
+    result = {'response_contacts':[],'to_contacts': [], 'cc': [], 'needs_approval': False, 'full_list': []}
 
     for group in from_groups:
         cc.extend(get_contacts_for_liaison_messages_for_group_primary(group))
         does_need_approval.append(needs_approval(group,person))
-        can_post_only.append(post_only(group,person))
         response_contacts.append(get_contacts_for_liaison_messages_for_group_secondary(group))
 
     for group in to_groups:
@@ -183,12 +164,15 @@ def ajax_get_liaison_info(request):
     else:
         does_need_approval = True
 
-    result.update({'error': False,
-                   'cc': list(set(cc)),
-                   'response_contacts':list(set(response_contacts)),
-                   'to_contacts': list(set(to_contacts)),
-                   'needs_approval': does_need_approval,
-                   'post_only': any(can_post_only)})
+    result.update(
+        {
+            "error": False,
+            "cc": list(set(cc)),
+            "response_contacts": list(set(response_contacts)),
+            "to_contacts": list(set(to_contacts)),
+            "needs_approval": does_need_approval,
+        }
+    )
 
     json_result = json.dumps(result)
     return HttpResponse(json_result, content_type='application/json')
diff --git a/ietf/static/js/liaisons.js b/ietf/static/js/liaisons.js
index e7a76660c9..6c46cb6dfc 100644
--- a/ietf/static/js/liaisons.js
+++ b/ietf/static/js/liaisons.js
@@ -197,16 +197,6 @@ var liaisonForm = {
         }
     },
 
-    checkPostOnly: function (post_only) {
-        if (post_only) {
-            $("button[name=send]")
-                .hide();
-        } else {
-            $("button[name=send]")
-                .show();
-        }
-    },
-
     updateInfo: function (first_time, sender) {
         // don't overwrite fields when editing existing liaison
         if (liaisonForm.is_edit_form) {
@@ -239,7 +229,6 @@ var liaisonForm = {
                         liaisonForm.toggleApproval(response.needs_approval);
                         liaisonForm.response_contacts.val(response.response_contacts);
                     }
-                    liaisonForm.checkPostOnly(response.post_only);
                 }
             }
         });
@@ -326,4 +315,4 @@ $(document)
             .each(liaisonForm.init);
         $('#liaison_search_form')
             .each(searchForm.init);
-    });
\ No newline at end of file
+    });

From a3836949eb827b7b70b22fb09187ba5c202a4d60 Mon Sep 17 00:00:00 2001
From: Peter Yee <github@houseofyee.com>
Date: Mon, 3 Nov 2025 18:11:21 -0500
Subject: [PATCH 498/656] feat(meetings): Use "Decline and Delete" for slide
 disapprovals (#9842)

* feat(meetings): Use "Decline and Delete" for slide disapprovals

Changes use of "disapprove" and "rejected" to "decline" and "declined"
when dealing with slides that have been submitted for approval.

Responds to #9242

* feat(meetings): Use "Decline and Delete" for slide disapprovals

Now with a fix for a test that broke owing to the original change.

Still responds to #9242
---
 ietf/meeting/tests_views.py                            | 2 +-
 ietf/meeting/views.py                                  | 2 +-
 ietf/templates/meeting/approve_proposed_slides.html    | 4 ++--
 ietf/templates/meeting/previously_approved_slides.html | 6 +++---
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 50960b5143..bab2de5ec7 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -7148,7 +7148,7 @@ def test_disapprove_proposed_slides(self):
             self.assertFalse(exists_in_storage("staging", submission.filename))
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-        self.assertRegex(r.content.decode(), r"These\s+slides\s+have\s+already\s+been\s+rejected")
+        self.assertRegex(r.content.decode(), r"These\s+slides\s+have\s+already\s+been\s+declined")
 
     @override_settings(MEETECHO_API_CONFIG="fake settings")  # enough to trigger API calls
     @patch("ietf.meeting.views.SlidesManager")
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index b0c46cb05a..3484a85784 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -5304,7 +5304,7 @@ def approve_proposed_slides(request, slidesubmission_id, num):
             if request.POST.get('approve'):
                 # Ensure that we have a file to approve.  The system gets cranky otherwise.
                 if submission.filename is None or submission.filename == '' or not os.path.isfile(submission.staged_filepath()):
-                    return HttpResponseNotFound("The slides you attempted to approve could not be found.  Please disapprove and delete them instead.")
+                    return HttpResponseNotFound("The slides you attempted to approve could not be found.  Please decline and delete them instead.")
                 title = form.cleaned_data['title']
                 if existing_doc:
                    doc = Document.objects.get(name=name)
diff --git a/ietf/templates/meeting/approve_proposed_slides.html b/ietf/templates/meeting/approve_proposed_slides.html
index 37fb523394..204473f455 100644
--- a/ietf/templates/meeting/approve_proposed_slides.html
+++ b/ietf/templates/meeting/approve_proposed_slides.html
@@ -34,6 +34,6 @@ <h2>
         <button type="submit"
                 class="btn btn-warning"
                 name="disapprove"
-                value="disapprove">Disapprove and Delete</button>
+                value="disapprove">Decline and Delete</button>
     </form>
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/ietf/templates/meeting/previously_approved_slides.html b/ietf/templates/meeting/previously_approved_slides.html
index 25a4c97863..95975cb63f 100644
--- a/ietf/templates/meeting/previously_approved_slides.html
+++ b/ietf/templates/meeting/previously_approved_slides.html
@@ -11,7 +11,7 @@ <h1 class="mb-3">
         {% if submission.status.slug == 'approved' %}
             approved
         {% else %}
-            rejected
+            declined
         {% endif %}
     </h1>
     <p>
@@ -19,7 +19,7 @@ <h1 class="mb-3">
         {% if submission.status.slug == 'approved' %}
             approved.
         {% else %}
-            rejected.
+            declined.
         {% endif %}
         No further action is needed.
     </p>
@@ -33,4 +33,4 @@ <h1 class="mb-3">
             return to this meeting session
         </a>.
     </p>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

From 3c12880650b6f468ab41ec5782f791c3e019b6ef Mon Sep 17 00:00:00 2001
From: Russ Housley <housley@vigilsec.com>
Date: Wed, 5 Nov 2025 15:27:52 -0500
Subject: [PATCH 499/656] feat: Add tab for IESG view of working groups (#9841)

* Add tab for WGs. Fixes#9730.

* Update utils.py

Only add WG tabe for the IESG (not the IAB)
---
 ietf/group/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/group/utils.py b/ietf/group/utils.py
index 29cfff2c2d..6777ed1933 100644
--- a/ietf/group/utils.py
+++ b/ietf/group/utils.py
@@ -237,9 +237,10 @@ def construct_group_menu_context(request, group, selected, group_type, others):
         entries.append(("Review requests", urlreverse(ietf.group.views.review_requests, kwargs=kwargs)))
         entries.append(("Reviewers", urlreverse(ietf.group.views.reviewer_overview, kwargs=kwargs)))
         entries.append(("Reviews History", urlreverse(ietf.group.views.review_requests_history, kwargs=kwargs)))
-
     if group.features.has_meetings:
         entries.append(("Meetings", urlreverse("ietf.group.views.meetings", kwargs=kwargs)))
+    if group.acronym in ["iesg"]:
+        entries.append(("Working Groups", urlreverse("ietf.iesg.views.working_groups")))
     if group.acronym in ["iab", "iesg"]:
         entries.append(("Statements", urlreverse("ietf.group.views.statements", kwargs=kwargs)))
         entries.append(("Appeals", urlreverse("ietf.group.views.appeals", kwargs=kwargs)))
@@ -250,7 +251,6 @@ def construct_group_menu_context(request, group, selected, group_type, others):
         if is_valid_url(group.list_archive):
             entries.append((mark_safe("List archive &raquo;"), group.list_archive))
 
-
     # actions
     actions = []
 

From d796aa0a9eee4b705bc072d546b9ce634eb7dd92 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 5 Nov 2025 15:29:08 -0500
Subject: [PATCH 500/656] chore: remove OBE test (#9885)

---
 ietf/liaisons/tests.py | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py
index 1209d7f401..d2f4fafca3 100644
--- a/ietf/liaisons/tests.py
+++ b/ietf/liaisons/tests.py
@@ -1119,17 +1119,6 @@ def test_redirect_for_approval(self):
     # -------------------------------------------------
     # Form validations
     # -------------------------------------------------
-    def test_post_and_send_fail(self):
-        RoleFactory(name_id='liaiman',person__user__username='ulm-liaiman',group__type_id='sdo',group__acronym='ulm')
-        GroupFactory(type_id='wg',acronym='mars')
-
-        url = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'incoming'})
-        login_testing_unauthorized(self, "ulm-liaiman", url)
-
-        r = self.client.post(url,get_liaison_post_data(),follow=True)
-
-        self.assertEqual(r.status_code, 200)
-        self.assertContains(r, 'As an IETF Liaison Manager you can not send incoming liaison statements')
 
     def test_deadline_field(self):
         '''Required for action, comment, not info, response'''

From 2ac61062b4776933a3bb73554963095bf1503760 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 6 Nov 2025 10:33:38 -0500
Subject: [PATCH 501/656] feat: show available sdo groups to liaison
 coordinators and managers (#9856)

* feat: show available sdo groups to liaison coordinators and mamagers

* fix: better typing delcaration

* fix: avoid unneeded whitespace in class declarations
---
 ietf/liaisons/tests.py                      |  11 +++
 ietf/liaisons/urls.py                       |   1 +
 ietf/liaisons/views.py                      |  14 +++
 ietf/templates/base/menu.html               | 102 +++++++++++---------
 ietf/templates/liaisons/list_other_sdo.html |  41 ++++++++
 5 files changed, 122 insertions(+), 47 deletions(-)
 create mode 100644 ietf/templates/liaisons/list_other_sdo.html

diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py
index d2f4fafca3..5e0a237575 100644
--- a/ietf/liaisons/tests.py
+++ b/ietf/liaisons/tests.py
@@ -110,6 +110,17 @@ def test_help_pages(self):
         self.assertEqual(self.client.get('/liaison/help/from_ietf/').status_code, 200)
         self.assertEqual(self.client.get('/liaison/help/to_ietf/').status_code, 200)
 
+    def test_list_other_sdo(self):
+        GroupFactory(type_id="sdo", state_id="conclude", acronym="third")
+        GroupFactory(type_id="sdo", state_id="active", acronym="second")
+        GroupFactory(type_id="sdo", state_id="active", acronym="first")
+        url = urlreverse("ietf.liaisons.views.list_other_sdo")
+        login_testing_unauthorized(self, "secretary", url)
+        r = self.client.get(url)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q("h1")), 2)
+        first_td_elements_text = [e.text for e in q("tr").find("td:first-child")]
+        self.assertEqual(first_td_elements_text, ["first", "second", "third"])
 
 class UnitTests(TestCase):
     def test_get_contacts_for_liaison_messages_for_group_primary(self):
diff --git a/ietf/liaisons/urls.py b/ietf/liaisons/urls.py
index 0fbd29425e..498df3b965 100644
--- a/ietf/liaisons/urls.py
+++ b/ietf/liaisons/urls.py
@@ -37,4 +37,5 @@
     url(r'^add/$', views.redirect_add),
     url(r'^for_approval/$', views.redirect_for_approval),
     url(r'^for_approval/(?P<object_id>\d+)/$', views.redirect_for_approval),
+    url(r"^list_other_sdo/$", views.list_other_sdo),
 ]
diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py
index e2abff52ac..59c6ea69fc 100644
--- a/ietf/liaisons/views.py
+++ b/ietf/liaisons/views.py
@@ -509,3 +509,17 @@ def liaison_resend(request, object_id):
     messages.success(request,'Liaison Statement resent')
     return redirect('ietf.liaisons.views.liaison_list')
 
+
+@role_required("Secretariat", "IAB", "Liaison Coordinator", "Liaison Manager")
+def list_other_sdo(request):
+    def _sdo_order_key(obj:Group)-> tuple[str,str]:
+        state_order = {
+            "active" : "a",
+            "conclude": "b",
+        }
+        return (state_order.get(obj.state.slug,f"c{obj.state.slug}"), obj.acronym)
+
+    sdos = sorted(list(Group.objects.filter(type="sdo")),key = _sdo_order_key)
+    for sdo in sdos:
+        sdo.liaison_managers =[r.person for r in sdo.role_set.filter(name="liaiman")]
+    return render(request,"liaisons/list_other_sdo.html",dict(sdos=sdos))
diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html
index 1e7c1688ff..8ff6e952da 100644
--- a/ietf/templates/base/menu.html
+++ b/ietf/templates/base/menu.html
@@ -31,19 +31,19 @@
     {% if flavor == 'top' %}<li><hr class="dropdown-divider"></li>{% endif %}
     <li {% if flavor == 'top' %}class="dropdown-header"{% else %}class="nav-item fw-bolder"{% endif %}>New work</li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.group.views.chartering_groups" %}">
             Chartering groups
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.group.views.bofs" group_type="wg" %}">
             BOFs
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.doc.views_bofreq.bof_requests" %}">
             BOF Requests
         </a>
@@ -51,13 +51,13 @@
     {% if flavor == 'top' %}<li><hr class="dropdown-divider"></li>{% endif %}
     <li {% if flavor == 'top' %}class="dropdown-header"{% else %}class="nav-item fw-bolder"{% endif %}>Other groups</li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.group.views.concluded_groups" %}">
             Concluded groups
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.mailinglists.views.nonwg' %}">
             Non-WG lists
         </a>
@@ -81,26 +81,26 @@
         </li>
     {% endif %}
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.doc.views_search.search" %}">
             Search
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.doc.views_search.recent_drafts" %}">
             Recent I-Ds
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.submit.views.upload_submission" %}">
             Submit an Internet-Draft
         </a>
     </li>
     {% if user and user.is_authenticated %}
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
             href="{% url "ietf.community.views.view_list" user.username %}">
             My tracked docs
         </a>
@@ -108,7 +108,7 @@
     {% if user|has_role:"Area Director,Secretariat" %}
         {% if flavor == 'top' %}<li><hr class="dropdown-divider"></li>{% endif %}
         <li>
-            <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+            <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
                 href="{% url 'ietf.doc.views_status_change.rfc_status_changes' %}">
                 RFC status changes
             </a>
@@ -121,19 +121,19 @@
       </a>
       <ul class="dropdown-menu">
             <li>
-                <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
                 href="{% url 'ietf.group.views.appeals' acronym='iesg' %}">
                     IESG appeals
                 </a>
             </li>
             <li>
-                <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
                 href="{% url 'ietf.doc.views_search.ad_workload' %}">
                     IESG dashboard
                 </a>
             </li>
             <li>
-                <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
                 href="{% url 'ietf.group.views.statements' acronym='iesg' %}">
                     IESG statements
                 </a>
@@ -142,7 +142,7 @@
         </li>
         <li><hr class="dropdown-divider"></li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
             href="{% url 'ietf.doc.views_ballot.irsg_ballot_status' %}">
             IRSG ballot status
         </a>
@@ -151,14 +151,14 @@
             {% if flavor == 'top' %}<li><hr class="dropdown-divider"></li>{% endif %}
             <li {% if flavor == 'top' %}class="dropdown-header"{% else %}class="nav-item fw-bolder"{% endif %}>Manage</li>
             <li>
-                <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
                    href="{% url "ietf.submit.views.approvals" %}">
                     Approve an I-D
                 </a>
             </li>
             {% for g in user|docman_groups %}
                 <li>
-                    <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                    <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
                        href="{% url "ietf.group.views.group_documents" g.acronym %}">
                         {{ g.acronym }} {{ g.type_id }} docs
                     </a>
@@ -175,7 +175,7 @@
             </li>
             {% for g in user|managed_review_groups %}
                 <li>
-                    <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                    <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
                        href="{% url "ietf.group.views.review_requests" g.acronym %}">
                         {{ g.acronym }} reviews
                     </a>
@@ -192,7 +192,7 @@
             </li>
             {% for nomcom in nomcoms %}
                 <li>
-                    <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                    <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
                        href="{% url "ietf.nomcom.views.private_index" nomcom.year %}">
                         {{ nomcom|capfirst }}
                     </a>
@@ -208,25 +208,25 @@
         RFC streams
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.group.views.stream_documents" acronym="iab" %}">
             IAB
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.group.views.stream_documents" acronym="irtf" %}">
             IRTF
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.group.views.stream_documents" acronym="ise" %}">
             ISE
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.group.views.stream_documents" acronym="editorial" %}">
             Editorial
         </a>
@@ -239,15 +239,15 @@
         Subseries
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.doc.views_search.index_subseries" type_id="std" %}">
             STD
         </a>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.doc.views_search.index_subseries" type_id="bcp" %}">
             BCP
         </a>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.doc.views_search.index_subseries" type_id="fyi" %}">
             FYI
         </a>
@@ -273,37 +273,37 @@
         </li>
     {% endif %}
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'agenda' %}">
             Agenda
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.meeting.views.materials' %}">
             Materials
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'floor-plan' %}">
             Floor plan
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="https://www.ietf.org/how/meetings/register/">
             Registration
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.meeting.views.important_dates' %}">
             Important dates
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.meeting.views_session_request.list_view' %}">
             Request a session
         </a>
@@ -317,7 +317,7 @@
         </li>
     {% endif %}
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.meeting.views.meeting_requests' %}">
             Session requests
         </a>
@@ -344,7 +344,7 @@
         </li>
     {% endif %}
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.meeting.views.upcoming' %}">
             Upcoming meetings
         </a>
@@ -359,13 +359,13 @@
         </li>
     {% endif %}
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.meeting.views.past' %}">
             Past meetings
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="https://www.ietf.org/how/meetings/past/">
             Meeting proceedings
         </a>
@@ -391,56 +391,64 @@
         </li>
     {% endif %}
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.ipr.views.showlist' %}">
             IPR disclosures
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.liaisons.views.liaison_list' %}">
             Liaison statements
         </a>
     </li>
+    {% if user|has_role:"Secretariat,IAB,Liaison Manager,Liaison Coordinator" %}
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
+           href="{% url 'ietf.liaisons.views.list_other_sdo' %}">
+            List of other SDO groups
+        </a>
+    </li>
+    {% endif %}
+    <li>
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.iesg.views.agenda' %}">
             IESG agenda
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.nomcom.views.index' %}">
             NomComs
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.doc.views_downref.downref_registry' %}">
             Downref registry
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url "ietf.stats.views.stats_index" %}">
             Statistics
         </a>
         <ul class="dropdown-menu {% if flavor == 'top' %}mt-n1{% else %}ms-n1{% endif %}">
             <li>
-                <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
                    href="{% url 'ietf.stats.views.document_stats' %}">
                     I-Ds/RFCs
                 </a>
             </li>
             <li>
-                <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
                    href="{% url 'ietf.stats.views.meeting_stats' %}">
                     Meetings
                 </a>
             </li>
             {% if user and user.is_authenticated %}
                 <li>
-                    <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+                    <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
                        href="{% url 'ietf.stats.views.review_stats' %}">
                         Reviews
                     </a>
@@ -449,19 +457,19 @@
         </ul>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.api.views.api_help' %}">
             API Help
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            href="{% url 'ietf.release.views.release' %}">
             Release notes
         </a>
     </li>
     <li>
-        <a class="dropdown-item {% if flavor != 'top' %}text-wrap{% endif %}"
+        <a class="dropdown-item{% if flavor != 'top' %} text-wrap{% endif %}"
            target="_blank" href="https://status.ietf.org">
             System status
         </a>
diff --git a/ietf/templates/liaisons/list_other_sdo.html b/ietf/templates/liaisons/list_other_sdo.html
new file mode 100644
index 0000000000..23f2d94af3
--- /dev/null
+++ b/ietf/templates/liaisons/list_other_sdo.html
@@ -0,0 +1,41 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% load origin static person_filters %}
+{% block pagehead %}
+    <link rel="stylesheet" href="{% static 'ietf/css/list.css' %}">
+{% endblock %}
+{% block title %}
+   Other SDO groups
+{% endblock %}
+{% block content %}
+  {% origin %}
+  {% regroup sdos by state.name as sdos_by_state %}
+  {% for sdos_in_state in sdos_by_state %}
+    <h1>{{sdos_in_state.grouper|capfirst}} SDO groups</h1>
+      <table class="table tablesorter table-borderless table-striped table-sm mt-3">
+        <thead>
+          <tr>
+            <th scope="col" data-sort="acronym">Acronym</th>
+            <th scope="col" data-sort="name">Name</th>
+            <th scope="col" data-sort="managers">Liaison Managers</th>
+          </tr>
+        </thead>
+        <tbody>
+          {% for group in sdos_in_state.list%}
+          <tr>
+            <td>{{ group.acronym }}</td>
+            <td>{{ group.name }}</td>
+            <td>
+              {% for person in group.liaison_managers %}
+                {% person_link person %}{% if not forloop.last %}, {% endif %}
+              {% endfor %}
+            </td>
+          </tr>
+          {% endfor %}
+        </tbody>
+    </table>
+  {% endfor %}
+{% endblock %}
+{% block js %}
+    <script src="{% static "ietf/js/list.js" %}"></script>
+{% endblock %}
\ No newline at end of file

From ce91feba9529ed74b8751269e70701b0ebeb70a4 Mon Sep 17 00:00:00 2001
From: Absit Iniuria <absniuria@nours.page>
Date: Fri, 7 Nov 2025 13:22:11 +0000
Subject: [PATCH 502/656] fix: onclick disable submit button (#9878)

---
 ietf/templates/submit/upload_submission.html | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ietf/templates/submit/upload_submission.html b/ietf/templates/submit/upload_submission.html
index 7313d8f000..b8b1aca29c 100644
--- a/ietf/templates/submit/upload_submission.html
+++ b/ietf/templates/submit/upload_submission.html
@@ -73,6 +73,11 @@
         $(document).ready(function() {
             if ($("#checkbox").is(':checked'))
             $("#other-formats").collapse('show')
+
+            $("form").one('submit', function() {
+                $("button").attr('disabled', 'disabled');
+                return true;
+            })
         });
     </script>
 {% endblock %}

From bd6a160508422da82ca24bdeabe08a9b7bcdb16f Mon Sep 17 00:00:00 2001
From: Russ Housley <housley@vigilsec.com>
Date: Fri, 7 Nov 2025 08:24:11 -0500
Subject: [PATCH 503/656] fix: add most recent ballot hint to History tab. 
 (#9851)

* Add most recent ballot hint to History tab. Fixes#9696.

* chore: style nit

* fix: explicitly order queryset

* fix: more precise description of the revision at the time of ballot

* refactor: is None instead of == None

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
Co-authored-by: Jennifer Richards <jeni@borkbork.org>
---
 ietf/doc/tests_utils.py                  | 28 +++++++++++++++++++++++-
 ietf/doc/utils.py                        | 19 +++++++++++++++-
 ietf/doc/views_doc.py                    |  6 +++++
 ietf/templates/doc/document_history.html |  5 +++++
 4 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py
index 7db59819da..ef71f6ae6e 100644
--- a/ietf/doc/tests_utils.py
+++ b/ietf/doc/tests_utils.py
@@ -13,13 +13,15 @@
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.name.models import DocTagName
 from ietf.person.factories import PersonFactory
+from ietf.doc.factories import BallotPositionDocEventFactory
 from ietf.utils.test_utils import TestCase, name_of_file_containing, reload_db_objects
 from ietf.person.models import Person
 from ietf.doc.factories import DocumentFactory, WgRfcFactory, WgDraftFactory
 from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor
 from ietf.doc.utils import (update_action_holders, add_state_change_event, update_documentauthors,
                             fuzzy_find_documents, rebuild_reference_relations, build_file_urls,
-                            ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file)
+                            ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file,
+                            last_ballot_doc_revision)
 from ietf.utils.draft import Draft, PlaintextDraft
 from ietf.utils.xmldraft import XMLDraft
 
@@ -533,3 +535,27 @@ def test_update_draft_bibxml_file(self, mock):
         self.assertEqual(mock.call_count, 1)
         self.assertEqual(mock.call_args, call(doc, "26"))
         self.assertEqual(ref_path.read_text(), "This\nis\nmy\nbibxml")
+
+
+class LastBallotDocRevisionTests(TestCase):
+    def test_last_ballot_doc_revision(self):
+        now = timezone.now()
+        ad = Person.objects.get(user__username="ad")
+        bpde_with_null_send_email = BallotPositionDocEventFactory(
+            time=now - datetime.timedelta(minutes=30),
+            send_email=None,
+        )
+        ballot = bpde_with_null_send_email.ballot
+        BallotPositionDocEventFactory(
+            ballot=ballot,
+            balloter=ad,
+            pos_id='noobj',
+            comment='Commentary',
+            comment_time=timezone.now(),
+            send_email=None,
+        )
+        doc = bpde_with_null_send_email.doc
+        rev = bpde_with_null_send_email.rev
+        nobody = PersonFactory()
+        self.assertIsNone(last_ballot_doc_revision(doc, nobody))
+        self.assertEqual(rev, last_ballot_doc_revision(doc, ad))
diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index 115b28b09b..2bd9a3d314 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -36,11 +36,12 @@
 from ietf.doc.models import Document, DocHistory, State, DocumentAuthor, DocHistoryAuthor
 from ietf.doc.models import RelatedDocument, RelatedDocHistory, BallotType, DocReminder
 from ietf.doc.models import DocEvent, ConsensusDocEvent, BallotDocEvent, IRSGBallotDocEvent, NewRevisionDocEvent, StateDocEvent
-from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent
+from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent, BallotPositionDocEvent
 from ietf.name.models import DocReminderTypeName, DocRelationshipName
 from ietf.group.models import Role, Group, GroupFeatures
 from ietf.ietfauth.utils import has_role, is_authorized_in_doc_stream, is_individual_draft_author, is_bofreq_editor
 from ietf.person.models import Email, Person
+from ietf.person.utils import get_active_balloters
 from ietf.review.models import ReviewWish
 from ietf.utils import draft, log
 from ietf.utils.mail import parseaddr, send_mail
@@ -687,6 +688,22 @@ def nice_consensus(consensus):
         }
     return mapping[consensus]
 
+def last_ballot_doc_revision(doc, person):
+    """ Return the document revision for the most recent ballot position
+        by the provided user. """
+    ballot = doc.active_ballot()
+    if ballot is None or person is None:
+        return None    
+    balloters = get_active_balloters(ballot.ballot_type)
+    if person not in balloters:
+        return None
+    position_queryset = BallotPositionDocEvent.objects.filter(type="changed_ballot_position", balloter=person, ballot=ballot).order_by("-time")
+    if not position_queryset.exists():
+        return None
+    ballot_time = position_queryset.first().time
+    doc_rev = NewRevisionDocEvent.objects.filter(doc=doc, time__lte=ballot_time).order_by('-time').first().rev
+    return doc_rev
+
 def has_same_ballot(doc, date1, date2=None):
     """ Test if the most recent ballot created before the end of date1
         is the same as the most recent ballot created before the
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 4a20db3c89..50070734f9 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -79,6 +79,7 @@
 from ietf.doc.views_ballot import parse_ballot_edit_return_point
 from ietf.doc.forms import InvestigateForm, TelechatForm, NotifyForm, ActionHoldersForm, DocAuthorForm, DocAuthorChangeBasisForm
 from ietf.doc.mails import email_comment, email_remind_action_holders
+from ietf.doc.utils import last_ballot_doc_revision
 from ietf.mailtrigger.utils import gather_relevant_expansions
 from ietf.meeting.models import Session, SessionPresentation
 from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions, sort_sessions, add_event_info_to_session_qs
@@ -1225,6 +1226,10 @@ def document_history(request, name):
             request.user, ("Area Director", "Secretariat", "IRTF Chair")
         )
 
+    # if the current user has balloted on this document, give them a revision hint
+    ballot_doc_rev = None
+    if request.user.is_authenticated:
+        ballot_doc_rev = last_ballot_doc_revision(doc, request.user.person)
 
     return render(
         request,
@@ -1235,6 +1240,7 @@ def document_history(request, name):
             "diff_revisions": diff_revisions,
             "events": events,
             "can_add_comment": can_add_comment,
+            "ballot_doc_rev": ballot_doc_rev,
         },
     )
 
diff --git a/ietf/templates/doc/document_history.html b/ietf/templates/doc/document_history.html
index a9ee3e9a36..78471e08d6 100644
--- a/ietf/templates/doc/document_history.html
+++ b/ietf/templates/doc/document_history.html
@@ -20,6 +20,11 @@
         <h2 class="my-3">Revision differences</h2>
         {% include "doc/document_history_form.html" with doc=doc diff_revisions=diff_revisions action=rfcdiff_base_url snapshot=snapshot only %}
     {% endif %}
+    {% if ballot_doc_rev %}
+        <p class="alert alert-info my-3">
+            Your most recent ballot position was provided when the document was at version {{ ballot_doc_rev }}.
+        </p>
+    {% endif %}
     <h2 class="my-3">Document history</h2>
     {% if doc.came_from_draft %}
         <ul class="nav nav-tabs my-3">

From 1249ace05bc700c1b80e80aee30992eac0595e89 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 7 Nov 2025 14:32:08 -0500
Subject: [PATCH 504/656] feat: add OpenTelemetry integration (#9886)

* feat: initial otel integration

* feat: add traceparent propagation via headers and meta tag

* feat: add memcached instrumentation

* fix: add more otel service attributes

* style: Update dev/build/gunicorn.conf.py

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>

* style: Update dev/build/gunicorn.conf.py

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>

* style: Update dev/build/gunicorn.conf.py

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>

* fix: Update traceparent meta tag to use otel variable

* fix: Update traceparent_id context processor to use sub-object

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 dev/build/gunicorn.conf.py | 36 +++++++++++++++++++++++++++++++++++-
 ietf/context_processors.py |  6 ++++++
 ietf/middleware.py         | 10 ++++++++++
 ietf/settings.py           |  2 ++
 ietf/templates/base.html   |  3 ++-
 requirements.txt           |  6 ++++++
 6 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py
index 032d95ee0d..c54b24a054 100644
--- a/dev/build/gunicorn.conf.py
+++ b/dev/build/gunicorn.conf.py
@@ -1,4 +1,16 @@
-# Copyright The IETF Trust 2024, All Rights Reserved
+# Copyright The IETF Trust 2024-2025, All Rights Reserved
+
+import os
+import ietf
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
+from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
 
 # Configure security scheme headers for forwarded requests. Cloudflare sets X-Forwarded-Proto 
 # for us. Don't trust any of the other similar headers. Only trust the header if it's coming
@@ -119,3 +131,25 @@ def post_request(worker, req, environ, resp):
     in_flight = in_flight_by_pid.get(worker.pid, [])
     if request_description in in_flight:
         in_flight.remove(request_description)
+
+def post_fork(server, worker):
+    server.log.info("Worker spawned (pid: %s)", worker.pid)
+
+    resource = Resource.create(attributes={
+        "service.name": "datatracker",
+        "service.version": ietf.__version__,
+        "service.instance.id": worker.pid,
+        "service.namespace": "datatracker",
+        "deployment.environment.name": os.environ.get("DATATRACKER_SERVICE_ENV", "dev")
+    })
+
+    trace.set_tracer_provider(TracerProvider(resource=resource))
+    otlp_exporter = OTLPSpanExporter(endpoint="https://heimdall-otlp.ietf.org/v1/traces")
+
+    trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter))
+
+    # Instrumentations
+    DjangoInstrumentor().instrument()
+    Psycopg2Instrumentor().instrument()
+    PymemcacheInstrumentor().instrument()
+    RequestsInstrumentor().instrument()
diff --git a/ietf/context_processors.py b/ietf/context_processors.py
index baa8d7a5d2..5aaa4ab256 100644
--- a/ietf/context_processors.py
+++ b/ietf/context_processors.py
@@ -5,6 +5,7 @@
 from django.conf import settings
 from django.utils import timezone
 from ietf import __version__, __patch__, __release_branch__, __release_hash__
+from opentelemetry.propagate import inject
 
 def server_mode(request):
     return {'server_mode': settings.SERVER_MODE}
@@ -51,3 +52,8 @@ def timezone_now(request):
     return {
         'timezone_now': timezone.now(),
     }
+
+def traceparent_id(request):
+    context_extras = {}
+    inject(context_extras)
+    return { "otel": context_extras }
diff --git a/ietf/middleware.py b/ietf/middleware.py
index a4b7a0d24c..fa2e8efd0c 100644
--- a/ietf/middleware.py
+++ b/ietf/middleware.py
@@ -8,6 +8,7 @@
 from django.http import HttpResponsePermanentRedirect
 from ietf.utils.log import log, exc_parts
 from ietf.utils.mail import log_smtp_exception
+from opentelemetry.propagate import inject
 import re
 import smtplib
 import unicodedata
@@ -99,3 +100,12 @@ def add_header(request):
         return response
 
     return add_header
+
+def add_otel_traceparent_header(get_response):
+    """Middleware to add the OpenTelemetry traceparent id header to the response"""
+    def add_header(request):
+        response = get_response(request)
+        inject(response)
+        return response
+
+    return add_header
diff --git a/ietf/settings.py b/ietf/settings.py
index eb5f9d2161..f8d8a28d65 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -411,6 +411,7 @@ def skip_unreadable_post(record):
         ],
         'OPTIONS': {
             'context_processors': [
+                'ietf.context_processors.traceparent_id',
                 'django.contrib.auth.context_processors.auth',
                 'django.template.context_processors.debug',     # makes 'sql_queries' available in templates
                 'django.template.context_processors.i18n',
@@ -443,6 +444,7 @@ def skip_unreadable_post(record):
 
 
 MIDDLEWARE = [
+    "ietf.middleware.add_otel_traceparent_header",
     "django.middleware.csrf.CsrfViewMiddleware",
     "corsheaders.middleware.CorsMiddleware", # see docs on CORS_REPLACE_HTTPS_REFERER before using it
     "django.middleware.common.CommonMiddleware",
diff --git a/ietf/templates/base.html b/ietf/templates/base.html
index d8ff85f86e..25ce50c467 100644
--- a/ietf/templates/base.html
+++ b/ietf/templates/base.html
@@ -15,6 +15,7 @@
             {% block title %}No title{% endblock %}
         </title>
         <meta name="viewport" content="width=device-width, initial-scale=1">
+        <meta name="traceparent" content="{{ otel.traceparent }}">
         <link href="{{ settings.STATIC_IETF_ORG }}/fonts/inter/import.css" rel="stylesheet">
         <link href="{{ settings.STATIC_IETF_ORG }}/fonts/noto-sans-mono/import.css" rel="stylesheet">
         <link rel="stylesheet" href="{% static 'ietf/css/ietf.css' %}">
@@ -170,4 +171,4 @@
         </script>
       {% analytical_body_bottom %}
     </body>
-</html>
\ No newline at end of file
+</html>
diff --git a/requirements.txt b/requirements.txt
index cf7c920fa3..02a4cf5fd0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -52,6 +52,12 @@ mock>=5.2.0  # should replace with unittest.mock and remove dependency
 types-mock>=5.2.0
 mypy~=1.7.0  # Version requirements determined by django-stubs.
 oic>=1.7.0  # Used only by tests
+opentelemetry-sdk>=1.38.0
+opentelemetry-instrumentation-django>=0.59b0
+opentelemetry-instrumentation-psycopg2>=0.59b0
+opentelemetry-instrumentation-pymemcache>=0.59b0
+opentelemetry-instrumentation-requests>=0.59b0
+opentelemetry-exporter-otlp-proto-http>=1.38.0
 pillow>=11.3.0
 psycopg2>=2.9.10
 pyang>=2.6.1

From de08e5f0d2e9b862e2a642a7241f62c22be05d38 Mon Sep 17 00:00:00 2001
From: Eric Vyncke <eric@vyncke.org>
Date: Fri, 7 Nov 2025 14:45:02 -0500
Subject: [PATCH 505/656] fix: Milestones date sorting out of order (#9868)

* Add hidden YYYY-MM-DD field to sort

* Also hide for ARIA readers
---
 ietf/templates/group/milestones.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ietf/templates/group/milestones.html b/ietf/templates/group/milestones.html
index df03d370fd..51579ce80a 100644
--- a/ietf/templates/group/milestones.html
+++ b/ietf/templates/group/milestones.html
@@ -20,7 +20,7 @@ <h2 class="my-3">
     <table class="table table-sm table-striped tablesorter">
         <thead>
             <tr>
-                <th class="col-2" scope="col"{% if group.uses_milestone_dates %} data-sort="num"{% endif %}>
+                <th class="col-2" scope="col"{% if group.uses_milestone_dates %} data-sort="num"{% endif %} data-default-sort="asc">
                     {% if group.uses_milestone_dates %}
                         Date
                     {% else %}
@@ -39,7 +39,7 @@ <h2 class="my-3">
                             <span class="badge rounded-pill {% if milestone.resolved|slugify == 'done' %}text-bg-success{% else %}text-bg-warning{% endif %}">{{ milestone.resolved|title }}</span>
                         {% else %}
                             {% if group.uses_milestone_dates %}
-                                {{ milestone.due|date:"M Y" }}
+                                 <span class="d-none" aria-hidden="true">{{ milestone.due }}</span>{{ milestone.due|date:"M Y" }}
                             {% else %}
                                 {% if forloop.first %}Last{% endif %}
                                 {% if forloop.last %}Next{% endif %}

From c8cf9e1e32c087b03b1fc9ce169a3c6f213c4e6a Mon Sep 17 00:00:00 2001
From: Rudi Matz <rudi@staff.ietf.org>
Date: Fri, 7 Nov 2025 14:45:43 -0500
Subject: [PATCH 506/656] refactor: use ical for important-dates (#9849)

* refactor: use ical for important-dates

* chore: format and cleanup

* fix: remove unwanted newline
---
 ietf/meeting/views.py                         | 74 +++++++++++++++++--
 ietf/templates/meeting/important_dates.ics    |  5 --
 .../meeting/important_dates_for_meeting.ics   | 24 ------
 3 files changed, 69 insertions(+), 34 deletions(-)
 delete mode 100644 ietf/templates/meeting/important_dates.ics
 delete mode 100644 ietf/templates/meeting/important_dates_for_meeting.ics

diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 3484a85784..a4fe498152 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -4445,6 +4445,73 @@ def upcoming_ical(request):
     response['Content-Disposition'] = 'attachment; filename="upcoming.ics"'
     return response
     
+def render_important_dates_ical(meetings, request):
+    """Generate important dates using the icalendar library"""
+    cal = Calendar()
+    cal.add("prodid", "-//IETF//datatracker.ietf.org ical importantdates//EN")
+    cal.add("version", "2.0")
+    cal.add("method", "PUBLISH")
+
+    for meeting in meetings:
+        for important_date in meeting.important_dates:
+            event = Event()
+            event.add("uid", f"ietf-{meeting.number}-{important_date.name_id}-"
+                     f"{important_date.date.isoformat()}")
+            event.add("summary", f"IETF {meeting.number}: {important_date.name.name}")
+            event.add("class", "PUBLIC")
+
+            if not important_date.midnight_cutoff:
+                event.add("dtstart", important_date.date)
+            else:
+                event.add("dtstart", datetime.datetime.combine(
+                    important_date.date, 
+                    datetime.time(23, 59, 0, tzinfo=pytz.UTC))
+                )
+
+            event.add("transp", "TRANSPARENT")
+            event.add("dtstamp", meeting.cached_updated)
+            description_lines = [important_date.name.desc]
+            if important_date.name.slug in ('openreg', 'earlybird'):
+                description_lines.append(
+                    "Register here: https://www.ietf.org/how/meetings/register/")
+            if important_date.name.slug == 'opensched':
+                description_lines.append("To request a Working Group session, use the "
+                                         "IETF Meeting Session Request Tool:")
+                description_lines.append(f"{request.scheme}://{request.get_host()}"
+                    f"{reverse('ietf.meeting.views_session_request.list_view')}")
+                description_lines.append("If you are working on a BOF request, it is "
+                                         "highly recommended to tell the IESG")
+                description_lines.append("now by sending an email to iesg@ietf.org "
+                                         "to get advance help with the request.")
+            if important_date.name.slug == 'cutoffwgreq':
+                description_lines.append("To request a Working Group session, use the "
+                                         "IETF Meeting Session Request Tool:")
+                description_lines.append(f"{request.scheme}://{request.get_host()}"
+                    f"{reverse('ietf.meeting.views_session_request.list_view')}")
+            if important_date.name.slug == 'cutoffbofreq':
+                description_lines.append("To request a BOF, please see instructions on "
+                                         "Requesting a BOF:")
+                description_lines.append("https://www.ietf.org/how/bofs/bof-procedures/")
+            if important_date.name.slug == 'idcutoff':
+                description_lines.append("Upload using the I-D Submission Tool:")
+                description_lines.append(f"{request.scheme}://{request.get_host()}"
+                    f"{reverse('ietf.submit.views.upload_submission')}")
+            if important_date.name.slug in (
+                    'draftwgagenda', 
+                    'revwgagenda', 
+                    'procsub', 
+                    'revslug'
+                ):
+                description_lines.append("Upload using the Meeting Materials "
+                                         "Management Tool:")
+                description_lines.append(f"{request.scheme}://{request.get_host()}"
+                    f"{reverse('ietf.meeting.views.materials', 
+                               kwargs={'num': meeting.number})}")
+
+            event.add("description", "\n".join(description_lines))
+            cal.add_component(event)
+
+    return cal.to_ical().decode("utf-8")
 
 def upcoming_json(request):
     '''Return Upcoming meetings in json format'''
@@ -5063,11 +5130,8 @@ def important_dates(request, num=None, output_format=None):
     if output_format == 'ics':
         preprocess_meeting_important_dates(meetings)
 
-        ics = render_to_string('meeting/important_dates.ics', {
-            'meetings': meetings,
-        }, request=request)
-        # icalendar response file should have '\r\n' line endings per RFC5545
-        response = HttpResponse(parse_ical_line_endings(ics), content_type='text/calendar')
+        response = HttpResponse(render_important_dates_ical(meetings, request), 
+                                content_type='text/calendar')
         response['Content-Disposition'] = 'attachment; filename="important-dates.ics"'
         return response
 
diff --git a/ietf/templates/meeting/important_dates.ics b/ietf/templates/meeting/important_dates.ics
deleted file mode 100644
index 35079e01eb..0000000000
--- a/ietf/templates/meeting/important_dates.ics
+++ /dev/null
@@ -1,5 +0,0 @@
-{% load humanize %}{% autoescape off %}{% load ietf_filters %}BEGIN:VCALENDAR
-VERSION:2.0
-METHOD:PUBLISH
-PRODID:-//IETF//datatracker.ietf.org ical importantdates//EN
-{% for meeting in meetings %}{% include "meeting/important_dates_for_meeting.ics" %}{% endfor %}END:VCALENDAR{% endautoescape %}
diff --git a/ietf/templates/meeting/important_dates_for_meeting.ics b/ietf/templates/meeting/important_dates_for_meeting.ics
deleted file mode 100644
index e6d403da93..0000000000
--- a/ietf/templates/meeting/important_dates_for_meeting.ics
+++ /dev/null
@@ -1,24 +0,0 @@
-{# Copyright The IETF Trust 2025, All Rights Reserved #}
-{% load tz ietf_filters %}{% for d in meeting.important_dates %}BEGIN:VEVENT
-UID:ietf-{{ meeting.number }}-{{ d.name_id }}-{{ d.date.isoformat }}
-SUMMARY:IETF {{ meeting.number }}: {{ d.name.name }}
-CLASS:PUBLIC
-DTSTART{% if not d.midnight_cutoff %};VALUE=DATE{% endif %}:{{ d.date|date:"Ymd" }}{% if d.midnight_cutoff %}235900Z{% endif %}
-DTSTAMP{% ics_date_time meeting.cached_updated|utc 'utc' %}
-TRANSP:TRANSPARENT
-DESCRIPTION:{{ d.name.desc }}{% if first and d.name.slug == 'openreg' or first and d.name.slug == 'earlybird' %}\n
- Register here: https://www.ietf.org/how/meetings/register/{% endif %}{% if d.name.slug == 'opensched' %}\n
- To request a Working Group session, use the IETF Meeting Session Request Tool:\n
- {{ request.scheme }}://{{ request.get_host}}{% url 'ietf.meeting.views_session_request.list_view' %}\n
- If you are working on a BOF request, it is highly recommended to tell the IESG\n
- now by sending an email to iesg@ietf.org to get advance help with the request.{% endif %}{% if d.name.slug == 'cutoffwgreq' %}\n
- To request a Working Group session, use the IETF Meeting Session Request Tool:\n
- {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.meeting.views_session_request.list_view' %}{% endif %}{% if d.name.slug == 'cutoffbofreq' %}\n
- To request a BOF, please see instructions on Requesting a BOF:\n
- https://www.ietf.org/how/bofs/bof-procedures/{% endif %}{% if d.name.slug == 'idcutoff' %}\n
- Upload using the I-D Submission Tool:\n
- {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.submit.views.upload_submission' %}{% endif %}{% if d.name.slug == 'draftwgagenda' or d.name.slug == 'revwgagenda' or d.name.slug == 'procsub' or d.name.slug == 'revslug' %}\n
- Upload using the Meeting Materials Management Tool:\n
- {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.meeting.views.materials' num=meeting.number %}{% endif %}
-END:VEVENT
-{% endfor %}

From 2ea08c696a7317e74e24f50c8b85ffbf91cdfa7b Mon Sep 17 00:00:00 2001
From: Eric Vyncke <eric@vyncke.org>
Date: Fri, 7 Nov 2025 14:46:46 -0500
Subject: [PATCH 507/656] Sort ADs by full name (#9839)

---
 ietf/doc/views_search.py       | 3 ++-
 ietf/utils/unicodenormalize.py | 9 +++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)
 create mode 100644 ietf/utils/unicodenormalize.py

diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py
index 2144c23e06..3b67061b05 100644
--- a/ietf/doc/views_search.py
+++ b/ietf/doc/views_search.py
@@ -74,7 +74,7 @@
 from ietf.utils.log import log
 from ietf.doc.utils_search import prepare_document_table, doc_type, doc_state, doc_type_name, AD_WORKLOAD
 from ietf.ietfauth.utils import has_role
-
+from ietf.utils.unicodenormalize import normalize_for_sorting
 
 class SearchForm(forms.Form):
     name = forms.CharField(required=False)
@@ -480,6 +480,7 @@ def _state_to_doc_type(state):
     ).distinct():
         if p in get_active_ads():
             ads.append(p)
+    ads.sort(key=lambda p: normalize_for_sorting(p.plain_name()))
 
     bucket_template = {
         dt: {state: [[] for _ in range(days)] for state in STATE_SLUGS[dt].values()}
diff --git a/ietf/utils/unicodenormalize.py b/ietf/utils/unicodenormalize.py
new file mode 100644
index 0000000000..8644dbdb79
--- /dev/null
+++ b/ietf/utils/unicodenormalize.py
@@ -0,0 +1,9 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+import unicodedata
+
+def normalize_for_sorting(text):
+    """Normalize text for proper accent-aware sorting."""
+    # Normalize the text to NFD (decomposed form)
+    decomposed = unicodedata.normalize('NFD', text)
+    # Filter out combining diacritical marks
+    return ''.join(char for char in decomposed if not unicodedata.combining(char))

From bc4752d8a09070680efe0f198fb4f8ac7b125233 Mon Sep 17 00:00:00 2001
From: Eric Vyncke <eric@vyncke.org>
Date: Fri, 7 Nov 2025 14:47:08 -0500
Subject: [PATCH 508/656] fix: AD order in IESG ballots (and other places) does
 not handle UTF-8 collation (#9836)

* Fix the ballot AD order by name

* Also fix the IESG WG dashboard

* Fix IESG milestones under review

* Better normalization by removing completely the diacritical marks

* Align comments

* Test whether areas are alphabetically sorted
---
 ietf/doc/views_doc.py |  4 +-
 ietf/iesg/tests.py    | 89 ++++++++++++++++++++++---------------------
 ietf/iesg/utils.py    |  7 ++--
 ietf/iesg/views.py    |  3 +-
 4 files changed, 53 insertions(+), 50 deletions(-)

diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 50070734f9..5210317325 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -92,7 +92,7 @@
 from ietf.utils.response import permission_denied
 from ietf.utils.text import maybe_split
 from ietf.utils.timezone import date_today
-
+from ietf.utils.unicodenormalize import normalize_for_sorting
 
 def render_document_top(request, doc, tab, name):
     tabs = []
@@ -1506,7 +1506,7 @@ def document_ballot_content(request, doc, ballot_id, editable=True):
     position_groups = []
     for n in BallotPositionName.objects.filter(slug__in=[p.pos_id for p in positions]).order_by('order'):
         g = (n, [p for p in positions if p.pos_id == n.slug])
-        g[1].sort(key=lambda p: (p.is_old_pos, p.balloter.plain_name()))
+        g[1].sort(key=lambda p: (p.is_old_pos, normalize_for_sorting(p.balloter.plain_name())))
         if n.blocking:
             position_groups.insert(0, g)
         else:
diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py
index f3778d1ded..e5fbe5da7b 100644
--- a/ietf/iesg/tests.py
+++ b/ietf/iesg/tests.py
@@ -281,12 +281,13 @@ def test_working_groups(self):
             wg_summary,
         ) = get_wg_dashboard_info()
 
+        # checks for the expected result with area sorted by name
         self.assertEqual(
             area_summary,
             [
                 {
-                    "area": "red",
-                    "groups_in_area": 1,
+                    "area": "blue",
+                    "groups_in_area": 3,
                     "groups_with_docs": 0,
                     "doc_count": 0,
                     "page_count": 0,
@@ -295,8 +296,8 @@ def test_working_groups(self):
                     "page_percent": 0,
                 },
                 {
-                    "area": "orange",
-                    "groups_in_area": 4,
+                    "area": "green",
+                    "groups_in_area": 3,
                     "groups_with_docs": 0,
                     "doc_count": 0,
                     "page_count": 0,
@@ -305,8 +306,8 @@ def test_working_groups(self):
                     "page_percent": 0,
                 },
                 {
-                    "area": "yellow",
-                    "groups_in_area": 2,
+                    "area": "orange",
+                    "groups_in_area": 4,
                     "groups_with_docs": 0,
                     "doc_count": 0,
                     "page_count": 0,
@@ -315,8 +316,8 @@ def test_working_groups(self):
                     "page_percent": 0,
                 },
                 {
-                    "area": "green",
-                    "groups_in_area": 3,
+                    "area": "red",
+                    "groups_in_area": 1,
                     "groups_with_docs": 0,
                     "doc_count": 0,
                     "page_count": 0,
@@ -325,8 +326,8 @@ def test_working_groups(self):
                     "page_percent": 0,
                 },
                 {
-                    "area": "blue",
-                    "groups_in_area": 3,
+                    "area": "violet",
+                    "groups_in_area": 4,
                     "groups_with_docs": 0,
                     "doc_count": 0,
                     "page_count": 0,
@@ -335,8 +336,8 @@ def test_working_groups(self):
                     "page_percent": 0,
                 },
                 {
-                    "area": "violet",
-                    "groups_in_area": 4,
+                    "area": "yellow",
+                    "groups_in_area": 2,
                     "groups_with_docs": 0,
                     "doc_count": 0,
                     "page_count": 0,
@@ -1192,34 +1193,14 @@ def test_working_groups(self):
             area_summary,
             [
                 {
-                    "area": "red",
-                    "groups_in_area": 1,
-                    "groups_with_docs": 1,
-                    "doc_count": 1,
-                    "page_count": 7,
-                    "group_percent": 6.25,
-                    "doc_percent": 3.571428571428571,
-                    "page_percent": 3.5897435897435894,
-                },
-                {
-                    "area": "orange",
-                    "groups_in_area": 4,
+                    "area": "blue",
+                    "groups_in_area": 3,
                     "groups_with_docs": 3,
-                    "doc_count": 4,
-                    "page_count": 29,
+                    "doc_count": 6,
+                    "page_count": 40,
                     "group_percent": 18.75,
-                    "doc_percent": 14.285714285714285,
-                    "page_percent": 14.871794871794872,
-                },
-                {
-                    "area": "yellow",
-                    "groups_in_area": 2,
-                    "groups_with_docs": 2,
-                    "doc_count": 2,
-                    "page_count": 17,
-                    "group_percent": 12.5,
-                    "doc_percent": 7.142857142857142,
-                    "page_percent": 8.717948717948717,
+                    "doc_percent": 21.428571428571427,
+                    "page_percent": 20.51282051282051,
                 },
                 {
                     "area": "green",
@@ -1232,14 +1213,24 @@ def test_working_groups(self):
                     "page_percent": 11.282051282051283,
                 },
                 {
-                    "area": "blue",
-                    "groups_in_area": 3,
+                    "area": "orange",
+                    "groups_in_area": 4,
                     "groups_with_docs": 3,
-                    "doc_count": 6,
-                    "page_count": 40,
+                    "doc_count": 4,
+                    "page_count": 29,
                     "group_percent": 18.75,
-                    "doc_percent": 21.428571428571427,
-                    "page_percent": 20.51282051282051,
+                    "doc_percent": 14.285714285714285,
+                    "page_percent": 14.871794871794872,
+                },
+                {
+                    "area": "red",
+                    "groups_in_area": 1,
+                    "groups_with_docs": 1,
+                    "doc_count": 1,
+                    "page_count": 7,
+                    "group_percent": 6.25,
+                    "doc_percent": 3.571428571428571,
+                    "page_percent": 3.5897435897435894,
                 },
                 {
                     "area": "violet",
@@ -1251,6 +1242,16 @@ def test_working_groups(self):
                     "doc_percent": 35.714285714285715,
                     "page_percent": 41.02564102564102,
                 },
+                {
+                    "area": "yellow",
+                    "groups_in_area": 2,
+                    "groups_with_docs": 2,
+                    "doc_count": 2,
+                    "page_count": 17,
+                    "group_percent": 12.5,
+                    "doc_percent": 7.142857142857142,
+                    "page_percent": 8.717948717948717,
+                },
             ],
         )
         self.assertEqual(
diff --git a/ietf/iesg/utils.py b/ietf/iesg/utils.py
index 9051cf92b2..1d24ecac8e 100644
--- a/ietf/iesg/utils.py
+++ b/ietf/iesg/utils.py
@@ -12,7 +12,7 @@
 from ietf.group.models import Group
 from ietf.iesg.agenda import get_doc_section
 from ietf.person.utils import get_active_ads
-
+from ietf.utils.unicodenormalize import normalize_for_sorting
 
 TelechatPageCount = namedtuple(
     "TelechatPageCount",
@@ -192,6 +192,7 @@ def get_wg_dashboard_info():
                 else 0,
             }
         )
+    area_summary.sort(key=lambda r: r["area"])
     area_totals = {
         "group_count": groups_total,
         "doc_count": docs_total,
@@ -238,7 +239,7 @@ def get_wg_dashboard_info():
                     else 0,
                 }
             )
-    noad_summary.sort(key=lambda r: (r["ad"], r["area"]))
+    noad_summary.sort(key=lambda r: (normalize_for_sorting(r["ad"]), r["area"]))
 
     ad_summary = []
     ad_totals = {
@@ -278,7 +279,7 @@ def get_wg_dashboard_info():
                     else 0,
                 }
             )
-    ad_summary.sort(key=lambda r: (r["ad"], r["area"]))
+    ad_summary.sort(key=lambda r: (normalize_for_sorting(r["ad"]), r["area"]))
 
     rfc_counter = Counter(
         Document.objects.filter(type="rfc").values_list("group__acronym", flat=True)
diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py
index 014b290425..f03afb9fc1 100644
--- a/ietf/iesg/views.py
+++ b/ietf/iesg/views.py
@@ -68,6 +68,7 @@
 from ietf.meeting.utils import get_activity_stats
 from ietf.doc.utils_search import fill_in_document_table_attributes, fill_in_telechat_date
 from ietf.utils.timezone import date_today, datetime_from_date
+from ietf.utils.unicodenormalize import normalize_for_sorting
 
 def review_decisions(request, year=None):
     events = DocEvent.objects.filter(type__in=("iesg_disapproved", "iesg_approved"))
@@ -547,7 +548,7 @@ def milestones_needing_review(request):
             )
 
     return render(request, 'iesg/milestones_needing_review.html',
-                  dict(ads=sorted(ad_list, key=lambda ad: ad.plain_name()),))
+                  dict(ads=sorted(ad_list, key=lambda ad: normalize_for_sorting(ad.plain_name())),))
 
 def photos(request):
     roles = sorted(Role.objects.filter(group__type='area', group__state='active', name_id='ad'),key=lambda x: "" if x.group.acronym=="gen" else x.group.acronym)

From c43092560c50363f59914205d059fec103350720 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Sun, 9 Nov 2025 10:51:51 -0500
Subject: [PATCH 509/656] fix: add resolver records when importing minutes
 (#9907)

---
 ietf/meeting/views.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index a4fe498152..2ab670a04d 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -5531,6 +5531,7 @@ def import_session_minutes(request, session_id, num):
             except SaveMaterialsError as err:
                 form.add_error(None, str(err))
             else:
+                resolve_uploaded_material(meeting=session.meeting, doc=session.minutes())
                 messages.success(request, f'Successfully imported minutes as revision {session.minutes().rev}.')
                 return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym)
     else:

From d252e85642261e8881e5fbec2d051e06c1616e07 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 11 Nov 2025 18:38:48 -0500
Subject: [PATCH 510/656] chore(deps): bump actions/setup-node from 4 to 6
 (#9758)

Bumps [actions/setup-node](https://github.com/actions/setup-node) from 4 to 6.
- [Release notes](https://github.com/actions/setup-node/releases)
- [Commits](https://github.com/actions/setup-node/compare/v4...v6)

---
updated-dependencies:
- dependency-name: actions/setup-node
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build.yml | 2 +-
 .github/workflows/tests.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7eac7b1c64..24ab51ebcf 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -170,7 +170,7 @@ jobs:
         fetch-tags: false
 
     - name: Setup Node.js environment
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v6
       with:
         node-version: 18.x
 
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index f10c1db9a3..836314bac0 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -104,7 +104,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - uses: actions/setup-node@v4
+    - uses: actions/setup-node@v6
       with:
         node-version: '18'
 

From 3f6addc6ebba12536d941b51c252168c9fc4f182 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 12 Nov 2025 10:30:43 -0600
Subject: [PATCH 511/656] fix: link to sdo group page from other sdo list
 (#9920)

---
 ietf/templates/liaisons/list_other_sdo.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/templates/liaisons/list_other_sdo.html b/ietf/templates/liaisons/list_other_sdo.html
index 23f2d94af3..e6a567ae50 100644
--- a/ietf/templates/liaisons/list_other_sdo.html
+++ b/ietf/templates/liaisons/list_other_sdo.html
@@ -23,7 +23,7 @@ <h1>{{sdos_in_state.grouper|capfirst}} SDO groups</h1>
         <tbody>
           {% for group in sdos_in_state.list%}
           <tr>
-            <td>{{ group.acronym }}</td>
+            <td><a href="{% url 'ietf.group.views.group_about' acronym=group.acronym %}">{{ group.acronym }}</a></td>
             <td>{{ group.name }}</td>
             <td>
               {% for person in group.liaison_managers %}

From 8cabf4d6c41ce1a73602b5ac1e1d372086266f71 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 12 Nov 2025 14:55:21 -0600
Subject: [PATCH 512/656] fix(dev): Disable python terminal integration as it
 corrupts the terminal (#9924)

---
 .vscode/settings.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.vscode/settings.json b/.vscode/settings.json
index b0ceba5c9d..b323cd02f7 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -56,5 +56,6 @@
     "python.linting.pylintArgs": ["--load-plugins", "pylint_django"],
     "python.testing.pytestEnabled": false,
     "python.testing.unittestEnabled": false,
-    "python.linting.enabled": true
+    "python.linting.enabled": true,
+    "python.terminal.shellIntegration.enabled": false
 }

From 92f23c0066c439ae7a696ddd8929418698cd9185 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 13 Nov 2025 09:36:52 -0600
Subject: [PATCH 513/656] fix: adjust test for other sdo list page (#9929)

---
 ietf/liaisons/tests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py
index 5e0a237575..e29045443f 100644
--- a/ietf/liaisons/tests.py
+++ b/ietf/liaisons/tests.py
@@ -119,7 +119,7 @@ def test_list_other_sdo(self):
         r = self.client.get(url)
         q = PyQuery(r.content)
         self.assertEqual(len(q("h1")), 2)
-        first_td_elements_text = [e.text for e in q("tr").find("td:first-child")]
+        first_td_elements_text = [e.text for e in q("tr").find("td:first-child a")]
         self.assertEqual(first_td_elements_text, ["first", "second", "third"])
 
 class UnitTests(TestCase):

From 9244e6107c4923fb6186126302bd6b71000680e3 Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Fri, 14 Nov 2025 17:37:14 -0500
Subject: [PATCH 514/656] ci(staging): Increase wait-for-completion timeout to
 60 minutes

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 24ab51ebcf..df218930ab 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -453,7 +453,7 @@ jobs:
           token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
           inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "waitClusterReady":true }'
           wait-for-completion: true
-          wait-for-completion-timeout: 30m
+          wait-for-completion-timeout: 60m
           wait-for-completion-interval: 20s
           display-workflow-run-url: false
 

From 70067702da41414710151e176ddaf57a3bf8ef64 Mon Sep 17 00:00:00 2001
From: Rudi Matz <rudi@staff.ietf.org>
Date: Tue, 18 Nov 2025 09:19:59 -0500
Subject: [PATCH 515/656] refactor: use ical for upcoming meetings (#9845)

* refactor: use ical for upcoming meetings

* fix: unfold to pass tests

* fix: make material title, type and url on one line
---
 ietf/meeting/tests_views.py         |  7 ++-
 ietf/meeting/views.py               | 69 ++++++++++++++++++++++++-----
 ietf/templates/meeting/upcoming.ics | 32 -------------
 3 files changed, 64 insertions(+), 44 deletions(-)
 delete mode 100644 ietf/templates/meeting/upcoming.ics

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index bab2de5ec7..b94229d969 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -5302,7 +5302,9 @@ def test_upcoming_ical(self):
         assert_ical_response_is_valid(self, r,
                                       expected_event_summaries=expected_event_summaries,
                                       expected_event_count=len(expected_event_summaries))
-        self.assertContains(r, 'Remote instructions: https://someurl.example.com')
+        # Unfold long lines that might have been folded by iCal
+        content_unfolded = r.content.decode('utf-8').replace('\r\n ', '')
+        self.assertIn('Remote instructions: https://someurl.example.com', content_unfolded)
 
         Session.objects.filter(meeting__type_id='interim').update(remote_instructions='')
         r = self.client.get(url)
@@ -5310,7 +5312,8 @@ def test_upcoming_ical(self):
         assert_ical_response_is_valid(self, r,
                                       expected_event_summaries=expected_event_summaries,
                                       expected_event_count=len(expected_event_summaries))
-        self.assertNotContains(r, 'Remote instructions:')
+        content_unfolded = r.content.decode('utf-8').replace('\r\n ', '')
+        self.assertNotIn('Remote instructions:', content_unfolded)
 
         updated = meeting.updated()
         self.assertIsNotNone(updated)
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 2ab670a04d..903e3c7e79 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -4430,16 +4430,7 @@ def upcoming_ical(request):
     else:
         ietfs = []
 
-    meeting_vtz = {meeting.vtimezone() for meeting in meetings}
-    meeting_vtz.discard(None)
-
-    # icalendar response file should have '\r\n' line endings per RFC5545
-    response = render_to_string('meeting/upcoming.ics', {
-        'vtimezones': ''.join(sorted(meeting_vtz)),
-        'assignments': assignments,
-        'ietfs': ietfs,
-    }, request=request)
-    response = parse_ical_line_endings(response)
+    response = render_upcoming_ical(assignments, ietfs, request)
 
     response = HttpResponse(response, content_type='text/calendar')
     response['Content-Disposition'] = 'attachment; filename="upcoming.ics"'
@@ -4513,6 +4504,64 @@ def render_important_dates_ical(meetings, request):
 
     return cal.to_ical().decode("utf-8")
 
+def render_upcoming_ical(assignments, meetings, request):
+    """Generate upcoming using the icalendar library"""
+
+    cal = Calendar()
+    cal.add("prodid", "-//IETF//datatracker.ietf.org ical upcoming//EN")
+    cal.add("version", "2.0")
+    cal.add("method", "PUBLISH")
+
+    for item in assignments:
+        event = Event()
+
+        event.add("uid", f"ietf-{item.session.meeting.number}-{item.timeslot.pk}")
+        event.add("summary", f"{item.session.group.acronym.lower()} - {item.session.name if item.session.name else item.session.group.name}")
+
+        if item.schedule.meeting.city:
+            event.add("location", f"{item.schedule.meeting.city},{item.schedule.meeting.country}")
+
+        event.add("status", item.session.ical_status)
+        event.add("class", "PUBLIC")
+
+        event.add("dtstart", item.timeslot.utc_start_time())
+        event.add("dtend", item.timeslot.utc_end_time())
+        event.add("dtstamp", item.timeslot.modified)
+        if item.session.agenda():
+            event.add("url", item.session.agenda().get_href())
+
+        description_lines = []
+        if item.timeslot.name:
+            description_lines.append(f"{item.timeslot.name}")
+        if item.session.agenda_note:
+            description_lines.append(f"Note: {item.session.agenda_note}")
+        
+        for material in item.session.materials.all():
+            title_part = f" ({material.title})" if material.type.name != "Agenda" else ""
+            description_lines.append(f"{material.type}{title_part}: {material.get_href()}")
+
+        if item.session.remote_instructions:
+            description_lines.append(f"Remote instructions: {item.session.remote_instructions}")
+
+        event.add("description", "\n".join(description_lines))
+        cal.add_component(event)
+
+    for meeting in meetings:
+        event = Event()
+        event.add("uid", f"ietf-{meeting.number}")
+        event.add("summary", f"IETF {meeting.number}")
+        if meeting.city:
+            event.add("location", f"{meeting.city},{meeting.country}")
+        event.add("class", "PUBLIC")
+        event.add("dtstart", meeting.date)
+        event.add("dtend", meeting.end_date() + datetime.timedelta(days=1))
+        event.add("dtstamp", meeting.cached_updated)
+        event.add("url", f"{request.scheme}://{request.get_host()}{reverse('agenda', kwargs={'num': meeting.number})}")
+
+        cal.add_component(event)
+
+    return cal.to_ical().decode("utf-8")
+
 def upcoming_json(request):
     '''Return Upcoming meetings in json format'''
     today = date_today()
diff --git a/ietf/templates/meeting/upcoming.ics b/ietf/templates/meeting/upcoming.ics
deleted file mode 100644
index 5eca7ec81d..0000000000
--- a/ietf/templates/meeting/upcoming.ics
+++ /dev/null
@@ -1,32 +0,0 @@
-{% load humanize tz %}{% autoescape off %}{% load ietf_filters textfilters %}BEGIN:VCALENDAR
-VERSION:2.0
-METHOD:PUBLISH
-PRODID:-//IETF//datatracker.ietf.org ical upcoming//EN
-{{vtimezones}}{% for item in assignments %}BEGIN:VEVENT
-UID:ietf-{{item.session.meeting.number}}-{{item.timeslot.pk}}
-SUMMARY:{% if item.session.name %}{{item.session.group.acronym|lower}} - {{item.session.name|ics_esc}}{% else %}{{item.session.group.acronym|lower}} - {{item.session.group.name}}{%endif%}
-{% if item.schedule.meeting.city %}LOCATION:{{item.schedule.meeting.city}},{{item.schedule.meeting.country}}
-{% endif %}STATUS:{{item.session.ical_status}}
-CLASS:PUBLIC
-DTSTART{% ics_date_time item.timeslot.local_start_time item.schedule.meeting.time_zone %}
-DTEND{% ics_date_time item.timeslot.local_end_time item.schedule.meeting.time_zone %}
-DTSTAMP{% ics_date_time item.timeslot.modified|utc 'utc' %}{% if item.session.agenda %}
-URL:{{item.session.agenda.get_href}}{% endif %}
-DESCRIPTION:{% if item.timeslot.name %}{{item.timeslot.name|ics_esc}}\n{% endif %}{% if item.session.agenda_note %}
- Note: {{item.session.agenda_note|ics_esc}}\n{% endif %}{% for material in item.session.materials.all %}
- \n{{material.type}}{% if material.type.name != "Agenda" %}
-  ({{material.title|ics_esc}}){% endif %}:
-  {{material.get_href}}\n{% endfor %}{% if item.session.remote_instructions %}
- Remote instructions: {{ item.session.remote_instructions }}\n{% endif %}
-END:VEVENT
-{% endfor %}{% for meeting in ietfs %}BEGIN:VEVENT
-UID:ietf-{{ meeting.number }}
-SUMMARY:IETF {{ meeting.number }}{% if meeting.city %}
-LOCATION:{{ meeting.city }},{{ meeting.country }}{% endif %}
-CLASS:PUBLIC
-DTSTART;VALUE=DATE{% if meeting.time_zone %};TZID={{ meeting.time_zone|ics_esc }}{% endif %}:{{ meeting.date|date:"Ymd" }}
-DTEND;VALUE=DATE{% if meeting.time_zone %};TZID={{ meeting.time_zone|ics_esc }}{% endif %}:{{ meeting.end_date|next_day|date:"Ymd" }}
-DTSTAMP{% ics_date_time meeting.cached_updated|utc 'utc' %}
-URL:{{ request.scheme }}://{{ request.get_host }}{% url 'agenda' num=meeting.number %}
-END:VEVENT
-{% endfor %}END:VCALENDAR{% endautoescape %}

From 33c0a894b3c1a4122f128117c295709e6f3c83ca Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 18 Nov 2025 10:01:16 -0600
Subject: [PATCH 516/656] test: fix selenium selector syntax (#9963)

---
 ietf/meeting/tests_js.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py
index 262b47652c..3269342924 100644
--- a/ietf/meeting/tests_js.py
+++ b/ietf/meeting/tests_js.py
@@ -1260,7 +1260,7 @@ def _assert_ietf_tz_correct(meetings, tz):
         # to inherit Django's settings.TIME_ZONE but I don't know whether that's guaranteed to be consistent.
         # To avoid test fragility, ask Moment what it considers local and expect that.
         local_tz = self.driver.execute_script('return moment.tz.guess();')
-        local_tz_opt = tz_select_input.find_element(By.CSS_SELECTOR, 'option[value=%s]' % local_tz)
+        local_tz_opt = tz_select_input.find_element(By.CSS_SELECTOR, 'option[value="%s"]' % local_tz)
         local_tz_bottom_opt = tz_select_bottom_input.find_element(By.CSS_SELECTOR, 'option[value="%s"]' % local_tz)
 
         # Should start off in local time zone

From 0dd84be354aa872e0bc2582295a97f7491dc7b21 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 18 Nov 2025 13:28:30 -0400
Subject: [PATCH 517/656] fix: recognize Editorial Stream in queue2.xml (#9965)

---
 ietf/sync/rfceditor.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 76357ffbcc..b3234a87e2 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -115,6 +115,8 @@ def parse_queue(response):
                     stream = "irtf"
                 elif name.startswith("INDEPENDENT"):
                     stream = "ise"
+                elif name.startswith("Editorial Stream"):
+                    stream = "editorial"
                 else:
                     stream = None
                     warnings.append("unrecognized section " + name)

From a007f50518d027e5edcf2325a4b6de1c347a1a02 Mon Sep 17 00:00:00 2001
From: Rudi Matz <rudi@staff.ietf.org>
Date: Tue, 18 Nov 2025 16:53:51 -0500
Subject: [PATCH 518/656] fix(dev): remove deprecated ext (#9966)

---
 .devcontainer/devcontainer.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index bf28550084..2cfff78853 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -33,7 +33,6 @@
                 "oderwat.indent-rainbow",
                 "redhat.vscode-yaml",
                 "spmeesseman.vscode-taskexplorer",
-                "visualstudioexptteam.vscodeintellicode",
                 "ms-python.pylint",
                 "charliermarsh.ruff"
             ],

From c79b35dd1ddbdadd78481023fa6e6dd876a3db3c Mon Sep 17 00:00:00 2001
From: Absit Iniuria <nouralma68@gmail.com>
Date: Tue, 25 Nov 2025 20:12:18 +0000
Subject: [PATCH 519/656] fix: make chatlog readable in dark mode (#9995)

---
 client/components/ChatLog.vue | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/client/components/ChatLog.vue b/client/components/ChatLog.vue
index f9dc382bfe..b3a4f7b40f 100644
--- a/client/components/ChatLog.vue
+++ b/client/components/ChatLog.vue
@@ -159,4 +159,18 @@ onMounted(() => {
     }
   }
 }
+
+[data-bs-theme="dark"] .chatlog {
+  .n-timeline-item-content__title {
+    color: #d63384 !important;
+  }
+  
+  .n-timeline-item-content__content {
+    color: #fff !important;
+  }
+
+  .n-timeline-item-content__meta {
+    color: #0569ffd9 !important;
+  }
+}
 </style>

From 3afa3d0e79fe98dd8f1780a6649340acbf0f317b Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Wed, 26 Nov 2025 00:05:07 -0500
Subject: [PATCH 520/656] ci: enable restoreToLastFullSnapshot for staging db
 deploy

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index df218930ab..5e91445202 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -451,7 +451,7 @@ jobs:
           repo: ietf-tools/infra-k8s
           ref: main
           token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
-          inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "waitClusterReady":true }'
+          inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }'
           wait-for-completion: true
           wait-for-completion-timeout: 60m
           wait-for-completion-interval: 20s

From 03e84773497715d5e082f8b08d8aaa90e45d2a73 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 1 Dec 2025 11:02:08 -0400
Subject: [PATCH 521/656] chore: drop apt-key + old pg utilities (#10029)

* chore: refactor to drop apt-key + consistency

Updates postgres apt setup to use current recommendation + not use
apt-key.

Moves all installed keyrings to /etc/apt/keyrings instead of
/usr/share/keyrings. Either is a reasonable place, but let's just use
one.

* chore: no duplicate key install, drop pg client 14

* chore: drop pgloader, too
---
 docker/app.Dockerfile    |  5 -----
 docker/base.Dockerfile   | 27 ++++++++++++++-------------
 docker/celery.Dockerfile |  5 -----
 3 files changed, 14 insertions(+), 23 deletions(-)

diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile
index fee3833733..dd4cf72ffd 100644
--- a/docker/app.Dockerfile
+++ b/docker/app.Dockerfile
@@ -10,12 +10,7 @@ ARG USER_GID=$USER_UID
 COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh
 RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh
 
-# Add Postgresql Apt Repository to get 14    
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
-RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \
     # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
     && apt-get purge -y imagemagick imagemagick-6-common \
     # Install common packages, non-root user
diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile
index c1fe5b093e..2501636049 100644
--- a/docker/base.Dockerfile
+++ b/docker/base.Dockerfile
@@ -11,21 +11,22 @@ RUN apt-get update \
 
 # Add Node.js Source
 RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \
-    && mkdir -p /etc/apt/keyrings\
-    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
-RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
-RUN echo "Package: nodejs" >> /etc/apt/preferences.d/preferences && \
-    echo "Pin: origin deb.nodesource.com" >> /etc/apt/preferences.d/preferences && \
-    echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/preferences
+    && mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+RUN echo "Package: nodejs" >> /etc/apt/preferences.d/preferences \
+    && echo "Pin: origin deb.nodesource.com" >> /etc/apt/preferences.d/preferences \
+    && echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/preferences
 
 # Add Docker Source
-RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
-    $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
-
-# Add PostgreSQL Source 
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
-RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+RUN mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker-archive-keyring.gpg \
+    && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list
+
+# Add PostgreSQL Source
+RUN mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/keyrings/apt.postgresql.org.gpg \
+    && echo "deb [signed-by=/etc/apt/keyrings/apt.postgresql.org.gpg] https://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
 
 # Install the packages we need
 RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends \
diff --git a/docker/celery.Dockerfile b/docker/celery.Dockerfile
index e7c7b9cc3f..e93ca3cf77 100644
--- a/docker/celery.Dockerfile
+++ b/docker/celery.Dockerfile
@@ -10,12 +10,7 @@ ARG USER_GID=$USER_UID
 COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh
 RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh
 
-# Add Postgresql Apt Repository to get 14    
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
-RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \
     # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
     && apt-get purge -y imagemagick imagemagick-6-common \
     # Install common packages, non-root user

From 47c15df84ca8a3f401239a41214b66839b16ecc3 Mon Sep 17 00:00:00 2001
From: Absit Iniuria <nouralma68@gmail.com>
Date: Mon, 1 Dec 2025 15:13:37 +0000
Subject: [PATCH 522/656] fix: render polls correctly in darkmode (#10027)

---
 client/components/Polls.vue | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/client/components/Polls.vue b/client/components/Polls.vue
index 30cc9e8f36..0846d4ed16 100644
--- a/client/components/Polls.vue
+++ b/client/components/Polls.vue
@@ -90,3 +90,21 @@ onMounted(() => {
 })
 </script>
 
+<style lang="scss">
+@import '../shared/colors.scss';
+
+[data-bs-theme="dark"] .polls {
+    tr,
+    th, 
+    td {
+        background-color: $gray-900 !important;
+        color: $gray-300 !important;
+        border-color: $gray-500 !important;
+        border-width: 0.05 !important;
+    }
+
+    td:hover {
+      background-color: $cyan-600 !important;
+    }
+}
+</style>

From 9d10995dbc6d84bcce7712dc2b2b849286925223 Mon Sep 17 00:00:00 2001
From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com>
Date: Mon, 1 Dec 2025 16:02:38 +0000
Subject: [PATCH 523/656] ci: update base image target version to 20251201T1548

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index d3b186e1f5..ae59ba1440 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20250903T2216
+FROM ghcr.io/ietf-tools/datatracker-app-base:20251201T1548
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 9d8427efdb..726f080c67 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20250903T2216
+20251201T1548

From ef4e0958e40380954577d9a4026a12bb9f8bf99e Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 2 Dec 2025 19:07:29 -0400
Subject: [PATCH 524/656] fix: adjust patch for Django 4.2.27 (#10045)

---
 patch/django-cookie-delete-with-all-settings.patch | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/patch/django-cookie-delete-with-all-settings.patch b/patch/django-cookie-delete-with-all-settings.patch
index fb8bbbe4fe..4ceaf8fceb 100644
--- a/patch/django-cookie-delete-with-all-settings.patch
+++ b/patch/django-cookie-delete-with-all-settings.patch
@@ -9,9 +9,9 @@
                  samesite=settings.SESSION_COOKIE_SAMESITE,
              )
  
---- django/http/response.py.orig	2020-08-13 11:16:04.060627793 +0200
-+++ django/http/response.py	2020-08-13 11:54:03.482476973 +0200
-@@ -282,20 +282,28 @@
+--- django/http/response.py.orig	2025-12-02 22:12:05.197283001 +0000
++++ django/http/response.py	2025-12-02 22:26:01.396576013 +0000
+@@ -286,20 +286,28 @@
          value = signing.get_cookie_signer(salt=key + salt).sign(value)
          return self.set_cookie(key, value, **kwargs)
 

From 9a4ad7223155814cc8cf8e7b6c4e581b37e7d119 Mon Sep 17 00:00:00 2001
From: Absit Iniuria <nouralma68@gmail.com>
Date: Mon, 15 Dec 2025 14:46:12 +0000
Subject: [PATCH 525/656] fix: adjust datepicker css in darkmode (#10095)

---
 ietf/static/css/datepicker.scss | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/ietf/static/css/datepicker.scss b/ietf/static/css/datepicker.scss
index 88f9e835fd..b193ccda3a 100644
--- a/ietf/static/css/datepicker.scss
+++ b/ietf/static/css/datepicker.scss
@@ -4,3 +4,29 @@
 $dp-cell-focus-background-color: $dropdown-link-hover-bg !default;
 
 @import "vanillajs-datepicker/sass/datepicker-bs5";
+
+[data-bs-theme="dark"] .datepicker-picker {
+    .datepicker-header, 
+    .datepicker-controls .btn,
+    .datepicker-main,
+    .datepicker-footer {
+        background-color: $gray-800;
+    }
+
+    .datepicker-cell:hover {
+        background-color: $gray-700;
+    }
+
+    .datepicker-cell.day.focused {
+        background-color: $gray-600;
+    }
+
+     .datepicker-cell.day.selected.focused {
+        background-color: $blue;
+    }
+
+     .datepicker-controls .btn:hover {
+        background-color:$gray-700;
+        color: $gray-400;
+     }
+}

From e775af342a3ff7a90d33166ac653ec99b8952d4e Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 16 Dec 2025 09:10:56 -0600
Subject: [PATCH 526/656] feat: editable wglc messages (#9607) (#9616)

* feat: editable wglc messages (#9607)

* feat: split complex actions out of change state view

* feat: allow editing call for adoption messages (#9702)

* feat: (WIP) sketch of workflow changes to allow editing call for adoption message

* feat: improved tests and related code adjustments

* feat: improve the change state interstitial workflow

* fix: simplify call for adoption workflow (#9964)

* fix: simplify call for adoption workflow

* fix: guard against nonwg target groups

* refactor: make reject conditions easier to understand

* feat: add issuewglc form automatic date replacement (#9969)

* fix: add date adjusting js to the wglc email form (#9976)

* fix: populate and use cfa and wglc forms correctly (#9981)

* fix: populate cfa form correctly. use cfa form for email.

* fix: restrict wglc to drafts

* fix: use wglc form for email.

* fix: tweak wording on forms and email (#9998)

* fix: adjust when call for adoption is shown, allow direct setting of c-adopt and wg-lc (#10051)

* fix: adjust when to show the call for adoption button

* fix: allow direct setting of c-adopt and wg-lc

* fix: adjust call for adoption wording (#10076)

---------

Co-authored-by: Nicolas Giard <github@ngpixel.com>
---
 ietf/doc/mails.py                             |  55 --
 ietf/doc/templatetags/ietf_filters.py         |  58 ++
 ietf/doc/tests_draft.py                       | 501 ++++++++++++++----
 ietf/doc/urls.py                              |   5 +
 ietf/doc/views_doc.py                         |   8 +-
 ietf/doc/views_draft.py                       | 386 +++++++++++++-
 ietf/ietfauth/utils.py                        |   4 +-
 ietf/templates/doc/document_draft.html        |   7 +-
 .../doc/draft/ask_about_ietf_adoption.html    |  22 +
 .../doc/draft/change_stream_state.html        |  19 +-
 ...issue_working_group_call_for_adoption.html |  61 +++
 .../draft/issue_working_group_last_call.html  |  61 +++
 .../doc/draft/wg_action_helpers.html          |  25 +
 .../doc/mail/wg_call_for_adoption_issued.txt  |  24 +-
 .../doc/mail/wg_last_call_issued.txt          |  22 +-
 15 files changed, 1066 insertions(+), 192 deletions(-)
 create mode 100644 ietf/templates/doc/draft/ask_about_ietf_adoption.html
 create mode 100644 ietf/templates/doc/draft/issue_working_group_call_for_adoption.html
 create mode 100644 ietf/templates/doc/draft/issue_working_group_last_call.html
 create mode 100644 ietf/templates/doc/draft/wg_action_helpers.html

diff --git a/ietf/doc/mails.py b/ietf/doc/mails.py
index f20d398c3c..ddecbb6b54 100644
--- a/ietf/doc/mails.py
+++ b/ietf/doc/mails.py
@@ -103,61 +103,6 @@ def email_stream_changed(request, doc, old_stream, new_stream, text=""):
               dict(text=text,
                    url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url()),
               cc=cc)
-    
-def email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=None):
-    if cfa_duration_weeks is None:
-        cfa_duration_weeks=2
-    (to, cc) = gather_address_lists("doc_wg_call_for_adoption_issued", doc=doc)
-    frm = request.user.person.formatted_email()
-
-    end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=7 * cfa_duration_weeks)
-
-    subject = f"Call for adoption: {doc.name}-{doc.rev}  (Ends {end_date})"
-
-    send_mail(
-        request,
-        to,
-        frm,
-        subject,
-        "doc/mail/wg_call_for_adoption_issued.txt",
-        dict(
-            doc=doc,
-            subject=subject,
-            url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(),
-            end_date=end_date,
-            cfa_duration_weeks=cfa_duration_weeks,
-            wg_list=doc.group.list_email,
-        ),
-        cc=cc,
-    )
-
-
-def email_wg_last_call_issued(request, doc, wglc_duration_weeks=None):
-    if wglc_duration_weeks is None:
-        wglc_duration_weeks = 2
-    (to, cc) = gather_address_lists("doc_wg_last_call_issued", doc=doc)
-    frm = request.user.person.formatted_email()
-
-
-    end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=7 * wglc_duration_weeks)
-    subject =  f"WG Last Call: {doc.name}-{doc.rev} (Ends {end_date})"
-
-    send_mail(
-        request,
-        to,
-        frm,
-        subject,
-        "doc/mail/wg_last_call_issued.txt",
-        dict(
-            doc=doc,
-            subject=subject,
-            url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(),
-            end_date=end_date,
-            wglc_duration_weeks=wglc_duration_weeks,
-            wg_list=doc.group.list_email,
-        ),
-        cc=cc,
-    )
 
 def email_pulled_from_rfc_queue(request, doc, comment, prev_state, next_state):
     extra=extra_automation_headers(doc)
diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py
index 5cabe1728d..ae5df641c2 100644
--- a/ietf/doc/templatetags/ietf_filters.py
+++ b/ietf/doc/templatetags/ietf_filters.py
@@ -1017,3 +1017,61 @@ def is_in_stream(doc):
     elif stream == "editorial":
         return True
     return False
+
+
+@register.filter
+def is_doc_ietf_adoptable(doc):
+    return doc.stream_id is None or all(
+        [
+            doc.stream_id == "ietf",
+            doc.get_state_slug("draft-stream-ietf")
+            not in [
+                "c-adopt",
+                "adopt-wg",
+                "info",
+                "wg-doc",
+                "parked",
+                "dead",
+                "wg-lc",
+                "waiting-for-implementation",
+                "chair-w",
+                "writeupw",
+                "sub-pub",
+            ],
+            doc.get_state_slug("draft") != "rfc",
+            doc.became_rfc() is None,
+        ]
+    )
+
+
+@register.filter
+def can_issue_ietf_wg_lc(doc):
+    return all(
+        [
+            doc.stream_id == "ietf",
+            doc.get_state_slug("draft-stream-ietf")
+            not in ["wg-cand", "c-adopt", "wg-lc"],
+            doc.get_state_slug("draft") != "rfc",
+            doc.became_rfc() is None,
+        ]
+    )
+
+
+@register.filter
+def can_submit_to_iesg(doc):
+    return all(
+        [
+            doc.stream_id == "ietf",
+            doc.get_state_slug("draft-iesg") == "idexists",
+            doc.get_state_slug("draft-stream-ietf") not in ["wg-cand", "c-adopt"],
+        ]
+    )
+
+
+@register.filter
+def has_had_ietf_wg_lc(doc):
+    return (
+        doc.stream_id == "ietf"
+        and doc.docevent_set.filter(statedocevent__state__slug="wg-lc").exists()
+    )
+
diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py
index 4d262c5a2f..cebeac1f27 100644
--- a/ietf/doc/tests_draft.py
+++ b/ietf/doc/tests_draft.py
@@ -21,13 +21,14 @@
 import debug                            # pyflakes:ignore
 
 from ietf.doc.expire import expirable_drafts, get_expired_drafts, send_expire_notice_for_draft, expire_draft
-from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, WgDraftFactory, RgDraftFactory, DocEventFactory
+from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, StateDocEventFactory, WgDraftFactory, RgDraftFactory, DocEventFactory, WgRfcFactory
 from ietf.doc.models import ( Document, DocReminder, DocEvent,
     ConsensusDocEvent, LastCallDocEvent, RelatedDocument, State, TelechatDocEvent, 
     WriteupDocEvent, DocRelationshipName, IanaExpertDocEvent )
 from ietf.doc.storage_utils import exists_in_storage, store_str
 from ietf.doc.utils import get_tags_for_stream_id, create_ballot_if_not_open
-from ietf.doc.views_draft import AdoptDraftForm
+from ietf.doc.views_draft import AdoptDraftForm, IssueCallForAdoptionForm, IssueWorkingGroupLastCallForm
+from ietf.ietfauth.utils import has_role
 from ietf.name.models import DocTagName, RoleName
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.group.models import Group, Role
@@ -86,7 +87,7 @@ def test_ad_approved(self):
         self.assertTrue("Approved: " in outbox[-1]['Subject'])
         self.assertTrue(draft.name in outbox[-1]['Subject'])
         self.assertTrue('iesg@' in outbox[-1]['To'])
-        
+
     def test_change_state(self):
         ad = Person.objects.get(user__username="ad")
         draft = WgDraftFactory(
@@ -1708,11 +1709,7 @@ def test_adopt_document(self):
                 self.assertEqual(draft.group, chair_role.group)
                 self.assertEqual(draft.stream_id, stream_state_type_slug[type_id][13:]) # trim off "draft-stream-"
                 self.assertEqual(draft.docevent_set.count() - events_before, 5)
-                self.assertEqual(len(outbox), 2)
-                self.assertTrue("Call For Adoption" in outbox[0]["Subject"])
-                self.assertTrue(f"{chair_role.group.acronym}-chairs@" in outbox[0]['To'])
-                self.assertTrue(f"{draft.name}@" in outbox[0]['To'])
-                self.assertTrue(f"{chair_role.group.acronym}@" in outbox[0]['To'])
+                self.assertEqual(len(outbox), 1)
                 # contents of outbox[1] are tested elsewhere
 
             # adopt
@@ -2003,6 +2000,40 @@ def test_set_state(self):
         self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string())
         self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string())
 
+    def test_set_stream_state_to_wglc(self):
+        def _form_presents_state_option(response, state):
+            q = PyQuery(response.content)
+            option = q(f"select#id_new_state option[value='{state.pk}']")
+            return len(option) != 0 
+        
+        doc = WgDraftFactory()
+        chair = RoleFactory(name_id="chair", group=doc.group).person
+        url = urlreverse(
+            "ietf.doc.views_draft.change_stream_state",
+            kwargs=dict(name=doc.name, state_type="draft-stream-ietf"),
+        )
+        login_testing_unauthorized(self, chair.user.username, url)
+        r = self.client.get(url)
+        wglc_state = State.objects.get(type="draft-stream-ietf", slug="wg-lc")
+        doc.set_state(wglc_state)
+        StateDocEventFactory(
+            doc=doc,
+            state_type_id="draft-stream-ietf",
+            state=("draft-stream-ietf", "wg-lc"),
+        )
+        self.assertEqual(doc.docevent_set.count(), 2)
+        r = self.client.get(url)
+        self.assertTrue(_form_presents_state_option(r, wglc_state))
+        other_doc = WgDraftFactory()
+        self.client.logout()
+        url = urlreverse(
+            "ietf.doc.views_draft.change_stream_state",
+            kwargs=dict(name=other_doc.name, state_type="draft-stream-ietf"),
+        )
+        login_testing_unauthorized(self, "secretary", url)
+        r = self.client.get(url)
+        self.assertTrue(_form_presents_state_option(r, wglc_state))
+
     def test_wg_call_for_adoption_issued(self):
         role = RoleFactory(
             name_id="chair",
@@ -2029,12 +2060,7 @@ def test_wg_call_for_adoption_issued(self):
             ),
         )
         self.assertEqual(r.status_code, 302)
-        self.assertEqual(len(outbox), 2)
-        self.assertIn("mars-wg@ietf.org", outbox[1]["To"])
-        self.assertIn("Call for adoption", outbox[1]["Subject"])
-        body = get_payload_text(outbox[1])
-        self.assertIn("disclosure obligations", body)
-        self.assertIn("starts a 10-week", body)
+        self.assertEqual(len(outbox), 1)
         # Test not entering a duration on the form
         draft = IndividualDraftFactory()
         url = urlreverse(
@@ -2051,12 +2077,7 @@ def test_wg_call_for_adoption_issued(self):
             ),
         )
         self.assertEqual(r.status_code, 302)
-        self.assertEqual(len(outbox), 2)
-        self.assertIn("mars-wg@ietf.org", outbox[1]["To"])
-        self.assertIn("Call for adoption", outbox[1]["Subject"])
-        body = get_payload_text(outbox[1])
-        self.assertIn("disclosure obligations", body)
-        self.assertIn("starts a 2-week", body)
+        self.assertEqual(len(outbox), 1)
 
         # Test the less usual workflow of issuing a call for adoption 
         # of a document that's already in the ietf stream
@@ -2086,12 +2107,7 @@ def test_wg_call_for_adoption_issued(self):
             ),
         )
         self.assertEqual(r.status_code, 302)
-        self.assertEqual(len(outbox), 2)
-        self.assertIn("mars-wg@ietf.org", outbox[1]["To"])
-        self.assertIn("Call for adoption", outbox[1]["Subject"])
-        body = get_payload_text(outbox[1])
-        self.assertIn("disclosure obligations", body)
-        self.assertIn("starts a 10-week", body)
+        self.assertEqual(len(outbox), 1)
         draft = WgDraftFactory(group=role.group)
         url = urlreverse(
             "ietf.doc.views_draft.change_stream_state",
@@ -2117,85 +2133,210 @@ def test_wg_call_for_adoption_issued(self):
             ),
         )
         self.assertEqual(r.status_code, 302)
-        self.assertEqual(len(outbox), 2)
-        self.assertIn("mars-wg@ietf.org", outbox[1]["To"])
-        self.assertIn("Call for adoption", outbox[1]["Subject"])
-        body = get_payload_text(outbox[1])
-        self.assertIn("disclosure obligations", body)
-        self.assertIn("starts a 2-week", body)
+        self.assertEqual(len(outbox), 1)
 
-    def test_wg_last_call_issued(self):
-        role = RoleFactory(
-            name_id="chair",
-            group__acronym="mars",
-            group__list_email="mars-wg@ietf.org",
-            person__user__username="marschairman",
-            person__name="WG Cháir Man",
+    def test_issue_wg_lc_form(self):
+        end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1)
+        post = dict(
+            end_date=end_date,
+            to="foo@example.net, bar@example.com",
+            # Intentionally not passing cc
+            subject=f"garbage {end_date.isoformat()}",
+            body=f"garbage {end_date.isoformat()}",
         )
-        draft = WgDraftFactory(group=role.group)
-        url = urlreverse(
-            "ietf.doc.views_draft.change_stream_state",
-            kwargs=dict(name=draft.name, state_type="draft-stream-ietf"),
+        form = IssueWorkingGroupLastCallForm(post)
+        self.assertTrue(form.is_valid())
+        post["end_date"] = date_today(DEADLINE_TZINFO)
+        form = IssueWorkingGroupLastCallForm(post)
+        self.assertFalse(form.is_valid())
+        self.assertIn(
+            "End date must be later than today",
+            form.errors["end_date"],
+            "Form accepted a too-early date",
         )
-        login_testing_unauthorized(self, "marschairman", url)
-        old_state = draft.get_state("draft-stream-%s" % draft.stream_id)
-        new_state = State.objects.get(
-            used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-lc"
+        post["end_date"] = end_date + datetime.timedelta(days=2)
+        form = IssueWorkingGroupLastCallForm(post)
+        self.assertFalse(form.is_valid())
+        self.assertIn(
+            f"Last call end date ({post['end_date'].isoformat()}) not found in subject",
+            form.errors["subject"],
+            "form allowed subject without end_date",
         )
-        self.assertNotEqual(old_state, new_state)
+        self.assertIn(
+            f"Last call end date ({post['end_date'].isoformat()}) not found in body",
+            form.errors["body"],
+            "form allowed body without end_date",
+        )
+
+    def test_issue_wg_lc(self):
+        def _assert_rejected(testcase, doc, person):
+            url = urlreverse(
+                "ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name)
+            )
+            login_testing_unauthorized(testcase, person.user.username, url)
+            r = testcase.client.get(url)
+            testcase.assertEqual(r.status_code, 404)
+            testcase.client.logout()
+
+        already_rfc = WgDraftFactory(states=[("draft", "rfc")])
+        rfc_chair = RoleFactory(name_id="chair", group=already_rfc.group).person
+        _assert_rejected(self, already_rfc, rfc_chair)
+        rg_doc = RgDraftFactory()
+        rg_chair = RoleFactory(name_id="chair", group=rg_doc.group).person
+        _assert_rejected(self, rg_doc, rg_chair)
+        inwglc_doc = WgDraftFactory(states=[("draft-stream-ietf", "wg-lc")])
+        inwglc_chair = RoleFactory(name_id="chair", group=inwglc_doc.group).person
+        _assert_rejected(self, inwglc_doc, inwglc_chair)
+        doc = WgDraftFactory()
+        chair = RoleFactory(name_id="chair", group=doc.group).person
+        url = urlreverse("ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name))
+        login_testing_unauthorized(self, chair.user.username, url)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        postdict = dict()
+        postdict["end_date"] = q("input#id_end_date").attr("value")
+        postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.org"
+        cc = q("input#id_cc").attr("value")
+        if cc is not None:
+            postdict["cc"] = cc + ", extracc@example.org"
+        else:
+            postdict["cc"] = "extracc@example.org"
+        postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words"
+        postdict["body"] = q("textarea#id_body").text() + "FGgqbQ$UNeXs"
         empty_outbox()
         r = self.client.post(
             url,
-            dict(
-                new_state=new_state.pk,
-                comment="some comment",
-                weeks="10",
-                tags=[
-                    t.pk
-                    for t in draft.tags.filter(
-                        slug__in=get_tags_for_stream_id(draft.stream_id)
-                    )
-                ],
-            ),
+            postdict,
         )
         self.assertEqual(r.status_code, 302)
+        self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "wg-lc")
         self.assertEqual(len(outbox), 2)
-        self.assertIn("mars-wg@ietf.org", outbox[1]["To"])
+        self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"])
+        self.assertIn("extrato@example.org", outbox[1]["To"])
+        self.assertIn("extracc@example.org", outbox[1]["Cc"])
+        self.assertIn("Extra Subject Words", outbox[1]["Subject"])
         self.assertIn("WG Last Call", outbox[1]["Subject"])
         body = get_payload_text(outbox[1])
         self.assertIn("disclosure obligations", body)
-        self.assertIn("starts a 10-week", body)
-        draft = WgDraftFactory(group=role.group)
-        url = urlreverse(
-            "ietf.doc.views_draft.change_stream_state",
-            kwargs=dict(name=draft.name, state_type="draft-stream-ietf"),
+        self.assertIn("FGgqbQ$UNeXs", body)
+
+    def test_issue_wg_call_for_adoption_form(self):
+        end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1)
+        post = dict(
+            end_date=end_date,
+            to="foo@example.net, bar@example.com",
+            # Intentionally not passing cc
+            subject=f"garbage {end_date.isoformat()}",
+            body=f"garbage {end_date.isoformat()}",
         )
-        old_state = draft.get_state("draft-stream-%s" % draft.stream_id)
-        new_state = State.objects.get(
-            used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-lc"
+        form = IssueCallForAdoptionForm(post)
+        self.assertTrue(form.is_valid())
+        post["end_date"] = date_today(DEADLINE_TZINFO)
+        form = IssueCallForAdoptionForm(post)
+        self.assertFalse(form.is_valid())
+        self.assertIn(
+            "End date must be later than today",
+            form.errors["end_date"],
+            "Form accepted a too-early date",
         )
-        self.assertNotEqual(old_state, new_state)
-        empty_outbox()
-        r = self.client.post(
-            url,
-            dict(
-                new_state=new_state.pk,
-                comment="some comment",
-                tags=[
-                    t.pk
-                    for t in draft.tags.filter(
-                        slug__in=get_tags_for_stream_id(draft.stream_id)
-                    )
-                ],
-            ),
+        post["end_date"] = end_date + datetime.timedelta(days=2)
+        form = IssueCallForAdoptionForm(post)
+        self.assertFalse(form.is_valid())
+        self.assertIn(
+            f"Call for adoption end date ({post['end_date'].isoformat()}) not found in subject",
+            form.errors["subject"],
+            "form allowed subject without end_date",
+        )
+        self.assertIn(
+            f"Call for adoption end date ({post['end_date'].isoformat()}) not found in body",
+            form.errors["body"],
+            "form allowed body without end_date",
+        )
+
+    def test_issue_wg_call_for_adoption(self):
+        def _assert_rejected(testcase, doc, person, group=None):
+            target_acronym = group.acronym if group is not None else doc.group.acronym
+            url = urlreverse(
+                "ietf.doc.views_draft.issue_wg_call_for_adoption",
+                kwargs=dict(name=doc.name, acronym=target_acronym),
+            )
+            login_testing_unauthorized(testcase, person.user.username, url)
+            r = testcase.client.get(url)
+            testcase.assertEqual(r.status_code, 403)
+            testcase.client.logout()
+
+        def _verify_call_issued(testcase, doc, chair_role):
+            url = urlreverse(
+                "ietf.doc.views_draft.issue_wg_call_for_adoption",
+                kwargs=dict(name=doc.name, acronym=chair_role.group.acronym),
+            )
+            login_testing_unauthorized(testcase, chair_role.person.user.username, url)
+            r = testcase.client.get(url)
+            testcase.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            postdict = dict()
+            postdict["end_date"] = q("input#id_end_date").attr("value")
+            postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.com"
+            self.assertIn(chair_role.group.list_email, postdict["to"])
+            cc = q("input#id_cc").attr("value")
+            if cc is not None:
+                postdict["cc"] = cc + ", extracc@example.com"
+            else:
+                postdict["cc"] = "extracc@example.com"
+            postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words"
+            postdict["body"] = q("textarea#id_body").text() + "FGgqbQ$UNeXs"
+            empty_outbox()
+            r = testcase.client.post(
+                url,
+                postdict,
+            )
+            testcase.assertEqual(r.status_code, 302)
+            doc.refresh_from_db()
+            self.assertEqual(doc.group, chair_role.group)
+            self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "c-adopt")
+            self.assertEqual(len(outbox), 2)
+            self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"])
+            self.assertIn("extrato@example.com", outbox[1]["To"])
+            self.assertIn("extracc@example.com", outbox[1]["Cc"])
+            self.assertIn("Call for adoption", outbox[1]["Subject"])
+            self.assertIn("Extra Subject Words", outbox[1]["Subject"])
+            body = get_payload_text(outbox[1])
+            self.assertIn("disclosure obligations", body)
+            self.assertIn("FGgqbQ$UNeXs", body)
+            self.client.logout()
+            return doc
+
+        already_rfc = WgDraftFactory(states=[("draft", "rfc")])
+        rfc = WgRfcFactory(group=already_rfc.group)
+        already_rfc.relateddocument_set.create(relationship_id="became_rfc",target=rfc)
+        rfc_chair = RoleFactory(name_id="chair", group=already_rfc.group).person
+        _assert_rejected(self, already_rfc, rfc_chair)
+        rg_doc = RgDraftFactory()
+        rg_chair = RoleFactory(name_id="chair", group=rg_doc.group).person
+        _assert_rejected(self, rg_doc, rg_chair)
+        inwglc_doc = WgDraftFactory(states=[("draft-stream-ietf", "wg-lc")])
+        inwglc_chair = RoleFactory(name_id="chair", group=inwglc_doc.group).person
+        _assert_rejected(self, inwglc_doc, inwglc_chair)
+        ind_doc = IndividualDraftFactory()
+        _assert_rejected(self, ind_doc, rg_chair, rg_doc.group)
+
+        # Successful call issued for doc already in WG
+        doc = WgDraftFactory(states=[("draft-stream-ietf","wg-cand")])
+        chair_role = RoleFactory(name_id="chair",group=doc.group)
+        _ = _verify_call_issued(self, doc, chair_role)
+
+        # Successful call issued for doc not yet in WG
+        doc = IndividualDraftFactory()
+        chair_role = RoleFactory(name_id="chair",group__type_id="wg")
+        doc = _verify_call_issued(self, doc, chair_role)
+        self.assertEqual(doc.group, chair_role.group)
+        self.assertEqual(doc.stream_id, "ietf")
+        self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "c-adopt")
+        self.assertCountEqual(
+            doc.docevent_set.values_list("type", flat=True),
+            ["changed_state", "changed_group", "changed_stream", "new_revision"]
         )
-        self.assertEqual(r.status_code, 302)
-        self.assertEqual(len(outbox), 2)
-        self.assertIn("mars-wg@ietf.org", outbox[1]["To"])
-        self.assertIn("WG Last Call", outbox[1]["Subject"])
-        body = get_payload_text(outbox[1])
-        self.assertIn("disclosure obligations", body)
-        self.assertIn("starts a 2-week", body)
 
     def test_pubreq_validation(self):
         role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man')
@@ -2393,6 +2534,188 @@ def test_editorial_metadata(self):
         self.assertNotIn("IESG", top_level_metadata_headings)
         self.assertNotIn("IANA", top_level_metadata_headings)
 
+class IetfGroupActionHelperTests(TestCase):
+    def test_manage_adoption_routing(self):
+        draft = IndividualDraftFactory()
+        nobody = PersonFactory()
+        rgchair = RoleFactory(group__type_id="rg", name_id="chair").person
+        wgchair = RoleFactory(group__type_id="wg", name_id="chair").person
+        multichair = RoleFactory(group__type_id="rg", name_id="chair").person
+        RoleFactory(group__type_id="wg", person=multichair, name_id="chair")
+        ad = RoleFactory(group__type_id="area", name_id="ad").person
+        secretary = Role.objects.filter(
+            name_id="secr", group__acronym="secretariat"
+        ).first()
+        self.assertIsNotNone(secretary)
+        secretary = secretary.person
+        self.assertFalse(
+            has_role(rgchair.user, ["Secretariat", "Area Director", "WG Chair"])
+        )
+        url = urlreverse(
+            "ietf.doc.views_doc.document_main", kwargs={"name": draft.name}
+        )
+        ask_about_ietf_link = urlreverse(
+            "ietf.doc.views_draft.ask_about_ietf_adoption_call",
+            kwargs={"name": draft.name},
+        )
+        non_ietf_adoption_link = urlreverse(
+            "ietf.doc.views_draft.adopt_draft", kwargs={"name": draft.name}
+        )
+        for person in (None, nobody, rgchair, wgchair, multichair, ad, secretary):
+            if person is not None:
+                self.client.login(
+                    username=person.user.username,
+                    password=f"{person.user.username}+password",
+                )
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            has_ask_about_ietf_link = len(q(f'a[href="{ask_about_ietf_link}"]')) != 0
+            has_non_ietf_adoption_link = (
+                len(q(f'a[href="{non_ietf_adoption_link}"]')) != 0
+            )
+            ask_about_r = self.client.get(ask_about_ietf_link)
+            ask_about_link_return_code = ask_about_r.status_code
+            if person == rgchair:
+                self.assertFalse(has_ask_about_ietf_link)
+                self.assertTrue(has_non_ietf_adoption_link)
+                self.assertEqual(ask_about_link_return_code, 403)
+            elif person in (ad, nobody, None):
+                self.assertFalse(has_ask_about_ietf_link)
+                self.assertFalse(has_non_ietf_adoption_link)
+                self.assertEqual(
+                    ask_about_link_return_code, 302 if person is None else 403
+                )
+            else:
+                self.assertTrue(has_ask_about_ietf_link)
+                self.assertFalse(has_non_ietf_adoption_link)
+                self.assertEqual(ask_about_link_return_code, 200)
+            self.client.logout()
+
+    def test_ask_about_ietf_adoption_call(self):
+        # Basic permission tests above
+        doc = IndividualDraftFactory()
+        self.assertEqual(doc.docevent_set.count(), 1)
+        chair_role = RoleFactory(group__type_id="wg", name_id="chair")
+        chair = chair_role.person
+        group = chair_role.group
+        othergroup = GroupFactory(type_id="wg")
+        url = urlreverse(
+            "ietf.doc.views_draft.ask_about_ietf_adoption_call",
+            kwargs={"name": doc.name},
+        )
+        login_testing_unauthorized(self, chair.user.username, url)
+        r = self.client.post(url, {"group": othergroup.pk})
+        self.assertEqual(r.status_code, 200)
+        r = self.client.post(url, {"group": group.pk})
+        self.assertEqual(r.status_code, 302)
+
+    def test_offer_wg_action_helpers(self):
+        def _assert_view_presents_buttons(testcase, response, expected):
+            q = PyQuery(response.content)
+            for id, expect in expected:
+                button = q(f"#{id}")
+                testcase.assertEqual(
+                    len(button) != 0,
+                    expect
+                )
+
+        # View rejects access
+        came_from_draft = WgDraftFactory(states=[("draft","rfc")])
+        rfc = WgRfcFactory(group=came_from_draft.group)
+        came_from_draft.relateddocument_set.create(relationship_id="became_rfc",target=rfc)
+        rfc_chair = RoleFactory(name_id="chair", group=rfc.group).person
+        url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=came_from_draft.name))
+        login_testing_unauthorized(self, rfc_chair.user.username, url)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
+        self.client.logout()
+        rg_draft = RgDraftFactory()
+        rg_chair = RoleFactory(group=rg_draft.group, name_id="chair").person
+        url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=rg_draft.name))
+        login_testing_unauthorized(self, rg_chair.user.username, url)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,404)
+        self.client.logout()
+
+        # View offers access
+        draft = WgDraftFactory()
+        chair = RoleFactory(group=draft.group, name_id="chair").person
+        url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=draft.name))
+        login_testing_unauthorized(self, chair.user.username, url)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+        _assert_view_presents_buttons(
+            self,
+            r,
+            [
+                ("id_wgadopt_button", False),
+                ("id_wglc_button", True),
+                ("id_pubreq_button", True),
+            ],
+        )
+        draft.set_state(State.objects.get(type_id="draft-stream-ietf", slug="wg-cand"))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+        _assert_view_presents_buttons(
+            self,
+            r,
+            [
+                ("id_wgadopt_button", True),
+                ("id_wglc_button", False),
+                ("id_pubreq_button", False),
+            ],
+        ) 
+        draft.set_state(State.objects.get(type_id="draft-stream-ietf", slug="wg-lc"))
+        StateDocEventFactory(
+            doc=draft,
+            state_type_id="draft-stream-ietf",
+            state=("draft-stream-ietf", "wg-lc"),
+        )
+        self.assertEqual(draft.docevent_set.count(), 2)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+        _assert_view_presents_buttons(
+            self,
+            r,
+            [
+                ("id_wgadopt_button", False),
+                ("id_wglc_button", False),
+                ("id_pubreq_button", True),
+            ],
+        )
+        draft.set_state(State.objects.get(type_id="draft-stream-ietf",slug="chair-w"))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        _assert_view_presents_buttons(
+            self,
+            r,
+            [
+                ("id_wgadopt_button", False),
+                ("id_wglc_button", True),
+                ("id_pubreq_button", True),
+            ],
+        )
+        self.assertContains(response=r,text="Issue Another Working Group Last Call", status_code=200)
+        other_draft = WgDraftFactory()
+        self.client.logout()
+        url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=other_draft.name))
+        login_testing_unauthorized(self, "secretary", url)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        _assert_view_presents_buttons(
+            self,
+            r,
+            [
+                ("id_wgadopt_button", False),
+                ("id_wglc_button", True),
+                ("id_pubreq_button", True),
+            ],
+        )
+        self.assertContains(
+            response=r, text="Issue Working Group Last Call", status_code=200
+        )
+
 class BallotEmailAjaxTests(TestCase):
     def test_ajax_build_position_email(self):
         def _post_json(self, url, json_to_post):
diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py
index 8e9c0569e2..61e94b2231 100644
--- a/ietf/doc/urls.py
+++ b/ietf/doc/urls.py
@@ -125,6 +125,7 @@
     url(r'^%(name)s/edit/info/$' % settings.URL_REGEXPS, views_draft.edit_info),
     url(r'^%(name)s/edit/requestresurrect/$' % settings.URL_REGEXPS, views_draft.request_resurrect),
     url(r'^%(name)s/edit/submit-to-iesg/$' % settings.URL_REGEXPS, views_draft.to_iesg),
+    url(r'^%(name)s/edit/issue-wg-lc/$' % settings.URL_REGEXPS, views_draft.issue_wg_lc),
     url(r'^%(name)s/edit/resurrect/$' % settings.URL_REGEXPS, views_draft.resurrect),
     url(r'^%(name)s/edit/addcomment/$' % settings.URL_REGEXPS, views_doc.add_comment),
 
@@ -143,9 +144,13 @@
     url(r'^%(name)s/edit/shepherdemail/$' % settings.URL_REGEXPS, views_draft.change_shepherd_email),
     url(r'^%(name)s/edit/shepherdwriteup/$' % settings.URL_REGEXPS, views_draft.edit_shepherd_writeup),
     url(r'^%(name)s/edit/requestpublication/$' % settings.URL_REGEXPS, views_draft.request_publication),
+    url(r'^%(name)s/edit/ask-about-ietf-adoption/$' % settings.URL_REGEXPS, views_draft.ask_about_ietf_adoption_call),
     url(r'^%(name)s/edit/adopt/$' % settings.URL_REGEXPS, views_draft.adopt_draft),
+    url(r'^%(name)s/edit/issue-wg-call-for-adoption/%(acronym)s/$' % settings.URL_REGEXPS, views_draft.issue_wg_call_for_adoption),
+
     url(r'^%(name)s/edit/release/$' % settings.URL_REGEXPS, views_draft.release_draft),
     url(r'^%(name)s/edit/state/(?P<state_type>draft-stream-[a-z]+)/$' % settings.URL_REGEXPS, views_draft.change_stream_state),
+    url(r'^%(name)s/edit/wg-action-helpers/$' % settings.URL_REGEXPS, views_draft.offer_wg_action_helpers),
     url(r'^%(name)s/edit/state/statement/$' % settings.URL_REGEXPS, views_statement.change_statement_state),
 
     url(r'^%(name)s/edit/clearballot/(?P<ballot_type_slug>[\w-]+)/$' % settings.URL_REGEXPS, views_ballot.clear_ballot),
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 5210317325..5564904504 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -515,13 +515,17 @@ def document_main(request, name, rev=None, document_html=False):
         # remaining actions
         actions = []
 
-        if can_adopt_draft(request.user, doc) and not doc.get_state_slug() in ["rfc"] and not snapshot:
+        if can_adopt_draft(request.user, doc) and doc.get_state_slug() not in ["rfc"] and not snapshot:
+            target = urlreverse("ietf.doc.views_draft.adopt_draft", kwargs=dict(name=doc.name))
             if doc.group and doc.group.acronym != 'none': # individual submission
                 # already adopted in one group
                 button_text = "Switch adoption"
             else:
                 button_text = "Manage adoption"
-            actions.append((button_text, urlreverse('ietf.doc.views_draft.adopt_draft', kwargs=dict(name=doc.name))))
+                # can_adopt_draft currently returns False for Area Directors
+                if has_role(request.user, ["Secretariat", "WG Chair"]):
+                    target = urlreverse("ietf.doc.views_draft.ask_about_ietf_adoption_call", kwargs=dict(name=doc.name))
+            actions.append((button_text, target))
 
         if can_unadopt_draft(request.user, doc) and not doc.get_state_slug() in ["rfc"] and not snapshot:
             if doc.get_state_slug('draft-iesg') == 'idexists':
diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py
index 16d04ee66a..c5faf1140b 100644
--- a/ietf/doc/views_draft.py
+++ b/ietf/doc/views_draft.py
@@ -28,12 +28,12 @@
     IanaExpertDocEvent, IESG_SUBSTATE_TAGS)
 from ietf.doc.mails import ( email_pulled_from_rfc_queue, email_resurrect_requested,
     email_resurrection_completed, email_state_changed, email_stream_changed,
-    email_wg_call_for_adoption_issued, email_wg_last_call_issued,
     email_stream_state_changed, email_stream_tags_changed, extra_automation_headers,
     generate_publication_request, email_adopted, email_intended_status_changed,
     email_iesg_processing_document, email_ad_approved_doc,
     email_iana_expert_review_state_changed )
 from ietf.doc.storage_utils import retrieve_bytes, store_bytes
+from ietf.doc.templatetags.ietf_filters import is_doc_ietf_adoptable
 from ietf.doc.utils import ( add_state_change_event, can_adopt_draft, can_unadopt_draft,
     get_tags_for_stream_id, nice_consensus, update_action_holders,
     update_reminder, update_telechat, make_notify_changed_event, get_initial_notify,
@@ -51,12 +51,12 @@
 from ietf.name.models import IntendedStdLevelName, DocTagName, StreamName
 from ietf.person.fields import SearchableEmailField
 from ietf.person.models import Person, Email
-from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of
+from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of, send_mail_text
 from ietf.utils.textupload import get_cleaned_text_file_content
 from ietf.utils import log
-from ietf.utils.fields import ModelMultipleChoiceField
+from ietf.utils.fields import DatepickerDateField, ModelMultipleChoiceField, MultiEmailField
 from ietf.utils.response import permission_denied
-from ietf.utils.timezone import datetime_today, DEADLINE_TZINFO
+from ietf.utils.timezone import date_today, datetime_from_date, datetime_today, DEADLINE_TZINFO
 
 
 class ChangeStateForm(forms.Form):
@@ -1564,7 +1564,7 @@ def adopt_draft(request, name):
                 events.append(e)
 
                 due_date = None
-                if form.cleaned_data["weeks"] != None:
+                if form.cleaned_data["weeks"] is not None:
                     due_date = datetime_today(DEADLINE_TZINFO) + datetime.timedelta(weeks=form.cleaned_data["weeks"])
 
                 update_reminder(doc, "stream-s", e, due_date)
@@ -1573,11 +1573,6 @@ def adopt_draft(request, name):
                 # setting states that are _not_ the adopted state.
                 email_adopted(request, doc, prev_state, new_state, by, comment)
 
-                # Currently only the IETF stream uses the c-adopt state - guard against other
-                # streams starting to use it asthe IPR rules for those streams will be different.
-                if doc.stream_id == "ietf" and new_state.slug == "c-adopt":
-                    email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=form.cleaned_data["weeks"])
-
             # comment
             if comment:
                 e = DocEvent(type="added_comment", doc=doc, rev=doc.rev, by=by)
@@ -1689,11 +1684,14 @@ def __init__(self, *args, **kwargs):
             f.queryset = f.queryset.exclude(pk__in=unused_states)
         f.label = state_type.label
         if self.stream.slug == 'ietf':
+            help_text_items = []
             if self.can_set_sub_pub:
-                f.help_text = "Only select 'Submitted to IESG for Publication' to correct errors. Use the document's main page to request publication."
+                help_text_items.append("Only select 'Submitted to IESG for Publication' to correct errors. This is not how to submit a document to the IESG.")
             else:
                 f.queryset = f.queryset.exclude(slug='sub-pub')
-                f.help_text = "You may not set the 'Submitted to IESG for Publication' using this form - Use the document's main page to request publication."
+                help_text_items.append("You may not set the 'Submitted to IESG for Publication' using this form - Use the button above or the document's main page to request publication.")
+            help_text_items.append("Only use this form in unusual circumstances when issuing call for adoption or working group last call.")
+            f.help_text = " ".join(help_text_items)
 
         f = self.fields['tags']
         f.queryset = f.queryset.filter(slug__in=get_tags_for_stream_id(doc.stream_id))
@@ -1704,7 +1702,7 @@ def __init__(self, *args, **kwargs):
     def clean_new_state(self):
         new_state = self.cleaned_data.get('new_state')
         if new_state.slug=='sub-pub' and not self.can_set_sub_pub:
-            raise forms.ValidationError('You may not set the %s state using this form. Use the "Submit to IESG for publication" button on the document\'s main page instead. If that button does not appear, the document may already have IESG state. Ask your Area Director or the Secretariat for help.'%new_state.name)
+            raise forms.ValidationError('You may not set the %s state using this form. Use the "Submit to IESG for Publication" button on the document\'s main page instead. If that button does not appear, the document may already have IESG state. Ask your Area Director or the Secretariat for help.'%new_state.name)
         return new_state
          
 
@@ -1730,6 +1728,19 @@ def next_states_for_stream_state(doc, state_type, current_state):
 
     return next_states
 
+@login_required
+def offer_wg_action_helpers(request, name):
+    doc = get_object_or_404(Document, type="draft", name=name)
+    if doc.stream is None or doc.stream_id != "ietf" or doc.became_rfc() is not None:
+        raise Http404
+
+    if not is_authorized_in_doc_stream(request.user, doc):
+        permission_denied(request, "You don't have permission to access this page.")
+    
+    return render(request, "doc/draft/wg_action_helpers.html",
+                              {"doc": doc,
+                              })
+
 @login_required
 def change_stream_state(request, name, state_type):
     doc = get_object_or_404(Document, type="draft", name=name)
@@ -1744,10 +1755,17 @@ def change_stream_state(request, name, state_type):
     prev_state = doc.get_state(state_type.slug)
     next_states = next_states_for_stream_state(doc, state_type, prev_state)
 
+    # These tell the form to allow directly setting the state to fix up errors.
     can_set_sub_pub = has_role(request.user,('Secretariat','Area Director')) or (prev_state and prev_state.slug=='sub-pub')
 
     if request.method == 'POST':
-        form = ChangeStreamStateForm(request.POST, doc=doc, state_type=state_type,can_set_sub_pub=can_set_sub_pub,stream=doc.stream)
+        form = ChangeStreamStateForm(
+            request.POST,
+            doc=doc,
+            state_type=state_type,
+            can_set_sub_pub=can_set_sub_pub,
+            stream=doc.stream,
+        )
         if form.is_valid():
             by = request.user.person
             events = []
@@ -1768,14 +1786,7 @@ def change_stream_state(request, name, state_type):
                 update_reminder(doc, "stream-s", e, due_date)
 
                 email_stream_state_changed(request, doc, prev_state, new_state, by, comment)
-
-                if doc.stream_id == "ietf":
-                    if new_state.slug == "c-adopt":
-                        email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=form.cleaned_data["weeks"])
                     
-                    if new_state.slug == "wg-lc":
-                        email_wg_last_call_issued(request, doc, wglc_duration_weeks=form.cleaned_data["weeks"])
-
             # tags
             existing_tags = set(doc.tags.all())
             new_tags = set(form.cleaned_data["tags"])
@@ -1811,8 +1822,15 @@ def change_stream_state(request, name, state_type):
             else:
                 form.add_error(None, "No change in state or tags found, and no comment provided -- nothing to do.")
     else:
-        form = ChangeStreamStateForm(initial=dict(new_state=prev_state.pk if prev_state else None, tags= doc.tags.all()),
-                                     doc=doc, state_type=state_type, can_set_sub_pub = can_set_sub_pub,stream = doc.stream)
+        form = ChangeStreamStateForm(
+            initial=dict(
+                new_state=prev_state.pk if prev_state else None, tags=doc.tags.all()
+            ),
+            doc=doc,
+            state_type=state_type,
+            can_set_sub_pub=can_set_sub_pub,
+            stream=doc.stream,
+        )
 
     milestones = doc.groupmilestone_set.all()
 
@@ -1857,3 +1875,325 @@ def set_intended_status_level(request, doc, new_level, old_level, comment):
         msg = "\n".join(e.desc for e in events)
 
         email_intended_status_changed(request, doc, msg)
+
+class IssueWorkingGroupLastCallForm(forms.Form):
+    end_date = DatepickerDateField(
+        required=True,
+        date_format="yyyy-mm-dd",
+        picker_settings={
+            "autoclose": "1",
+        },
+        help_text="The date the Last Call closes. If you change this, review the subject and body carefully to ensure the change is captured correctly.",
+    )
+
+    to = MultiEmailField(
+        required=True,
+        help_text="Comma separated list of address to use in the To: header",
+    )
+    cc = MultiEmailField(
+        required=False, help_text="Comma separated list of addresses to copy"
+    )
+    subject = forms.CharField(
+        required=True,
+        help_text="Subject for Last Call message. If you change the date here, be sure to make a matching change in the body.",
+    )
+    body = forms.CharField(
+        widget=forms.Textarea, required=True, help_text="Body for Last Call message"
+    )
+
+    def clean_end_date(self):
+        end_date = self.cleaned_data["end_date"]
+        if end_date <= date_today(DEADLINE_TZINFO):
+            raise forms.ValidationError("End date must be later than today")
+        return end_date
+
+    def clean(self):
+        cleaned_data = super().clean()
+        end_date = cleaned_data.get("end_date")
+        if end_date is not None:
+            body = cleaned_data.get("body")
+            subject = cleaned_data.get("subject")
+            if end_date.isoformat() not in body:
+                self.add_error(
+                    "body",
+                    forms.ValidationError(
+                        f"Last call end date ({end_date.isoformat()}) not found in body"
+                    ),
+                )
+            if end_date.isoformat() not in subject:
+                self.add_error(
+                    "subject",
+                    forms.ValidationError(
+                        f"Last call end date ({end_date.isoformat()}) not found in subject"
+                    ),
+                )
+        return cleaned_data
+
+
+@login_required
+def issue_wg_lc(request, name):
+    doc = get_object_or_404(Document, name=name)
+
+    if doc.stream_id != "ietf":
+        raise Http404
+    if doc.type_id != "draft" or doc.group.type_id != "wg":
+        raise Http404
+    if doc.get_state_slug("draft-stream-ietf") == "wg-lc":
+        raise Http404
+    if doc.get_state_slug("draft") == "rfc":
+        raise Http404
+
+    if not is_authorized_in_doc_stream(request.user, doc):
+        permission_denied(request, "You don't have permission to access this page.")
+
+    if request.method == "POST":
+        form = IssueWorkingGroupLastCallForm(request.POST)
+        if form.is_valid():
+            # Intentionally not changing tags or adding a comment
+            # those things can be done with other workflows
+            by = request.user.person
+            prev_state = doc.get_state("draft-stream-ietf")
+            events = []
+            wglc_state = State.objects.get(type="draft-stream-ietf", slug="wg-lc")
+            doc.set_state(wglc_state)
+            e = add_state_change_event(doc, by, prev_state, wglc_state)
+            events.append(e)
+            end_date = form.cleaned_data["end_date"]
+            update_reminder(
+                doc, "stream-s", e, datetime_from_date(end_date, DEADLINE_TZINFO)
+            )
+            doc.save_with_history(events)
+            email_stream_state_changed(request, doc, prev_state, wglc_state, by)
+            send_mail_text(
+                request, 
+                to = form.cleaned_data["to"], 
+                frm = request.user.person.formatted_email(),
+                subject = form.cleaned_data["subject"], 
+                txt = form.cleaned_data["body"],
+                cc = form.cleaned_data["cc"],
+            )
+            return redirect("ietf.doc.views_doc.document_main", name=doc.name)
+    else:
+        end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=14)
+        subject = f"WG Last Call: {doc.name}-{doc.rev} (Ends {end_date})"
+        body = render_to_string(
+            "doc/mail/wg_last_call_issued.txt",
+            dict(
+                doc=doc,
+                end_date=end_date,
+                wg_list=doc.group.list_email,
+                settings=settings,
+            ),
+        )
+        (to, cc) = gather_address_lists("doc_wg_last_call_issued", doc=doc)
+
+        form = IssueWorkingGroupLastCallForm(
+            initial=dict(
+                end_date=end_date,
+                to=", ".join(to),
+                cc=", ".join(cc),
+                subject=subject,
+                body=body,
+            )
+        )
+
+    return render(
+        request,
+        "doc/draft/issue_working_group_last_call.html",
+        dict(
+            doc=doc,
+            form=form,
+        ),
+    )
+
+class IssueCallForAdoptionForm(forms.Form):
+    end_date = DatepickerDateField(
+        required=True,
+        date_format="yyyy-mm-dd",
+        picker_settings={
+            "autoclose": "1",
+        },
+        help_text="The date the Call for Adoption closes. If you change this, review the subject and body carefully to ensure the change is captured correctly.",
+    )
+
+    to = MultiEmailField(
+        required=True,
+        help_text="Comma separated list of address to use in the To: header",
+    )
+    cc = MultiEmailField(
+        required=False, help_text="Comma separated list of addresses to copy"
+    )
+    subject = forms.CharField(
+        required=True,
+        help_text="Subject for Call for Adoption message. If you change the date here, be sure to make a matching change in the body.",
+    )
+    body = forms.CharField(
+        widget=forms.Textarea, required=True, help_text="Body for Call for Adoption message"
+    )
+
+    def clean_end_date(self):
+        end_date = self.cleaned_data["end_date"]
+        if end_date <= date_today(DEADLINE_TZINFO):
+            raise forms.ValidationError("End date must be later than today")
+        return end_date
+
+    def clean(self):
+        cleaned_data = super().clean()
+        end_date = cleaned_data.get("end_date")
+        if end_date is not None:
+            body = cleaned_data.get("body")
+            subject = cleaned_data.get("subject")
+            if end_date.isoformat() not in body:
+                self.add_error(
+                    "body",
+                    forms.ValidationError(
+                        f"Call for adoption end date ({end_date.isoformat()}) not found in body"
+                    ),
+                )
+            if end_date.isoformat() not in subject:
+                self.add_error(
+                    "subject",
+                    forms.ValidationError(
+                        f"Call for adoption end date ({end_date.isoformat()}) not found in subject"
+                    ),
+                )
+        return cleaned_data
+
+@login_required
+def issue_wg_call_for_adoption(request, name, acronym):
+    doc = get_object_or_404(Document, name=name)
+    group = Group.objects.filter(acronym=acronym, type_id="wg").first()
+    reject = False
+    if group is None or doc.type_id != "draft" or not is_doc_ietf_adoptable(doc):
+        reject = True
+    if doc.stream is None:
+        if not can_adopt_draft(request.user, doc):
+            reject = True
+    elif doc.stream_id != "ietf":
+        reject = True
+    else: # doc.stream_id == "ietf"
+        if not is_authorized_in_doc_stream(request.user, doc):
+            reject = True
+    if reject:
+        raise permission_denied(request, f"You can't issue a {acronym} wg call for adoption for this document.")
+
+    if request.method == "POST":
+        form = IssueCallForAdoptionForm(request.POST)
+        if form.is_valid():
+            # Intentionally not changing tags or adding a comment
+            # those things can be done with other workflows
+            by = request.user.person
+
+            events = []
+            if doc.stream_id != "ietf":
+                stream = StreamName.objects.get(slug="ietf")
+                doc.stream = stream
+                e = DocEvent(type="changed_stream", doc=doc, rev=doc.rev, by=by)
+                e.desc = f"Changed stream to <b>{stream.name}</b>"  # Propogates embedding html in DocEvent.desc for consistency
+                e.save()
+                events.append(e)
+            if doc.group != group:
+                doc.group = group
+                e = DocEvent(type="changed_group", doc=doc, rev=doc.rev, by=by)
+                e.desc = f"Changed group to <b>{group.name} ({group.acronym.upper()})</b>"  # Even if it makes the cats cry
+                e.save()
+                events.append(e)
+            prev_state = doc.get_state("draft-stream-ietf")
+            c_adopt_state = State.objects.get(type="draft-stream-ietf", slug="c-adopt")
+            doc.set_state(c_adopt_state)
+            e = add_state_change_event(doc, by, prev_state, c_adopt_state)
+            events.append(e)
+            end_date = form.cleaned_data["end_date"]
+            update_reminder(
+                doc, "stream-s", e, datetime_from_date(end_date, DEADLINE_TZINFO)
+            )
+            doc.save_with_history(events)
+            email_stream_state_changed(request, doc, prev_state, c_adopt_state, by)
+            send_mail_text(
+                request, 
+                to = form.cleaned_data["to"], 
+                frm = request.user.person.formatted_email(),
+                subject = form.cleaned_data["subject"], 
+                txt = form.cleaned_data["body"],
+                cc = form.cleaned_data["cc"],
+            )
+            return redirect("ietf.doc.views_doc.document_main", name=doc.name)
+    else:
+        end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=14)
+        subject = f"Call for adoption: {doc.name}-{doc.rev}  (Ends {end_date})"
+        body = render_to_string(
+            "doc/mail/wg_call_for_adoption_issued.txt",
+            dict(
+                doc=doc,
+                group=group,
+                end_date=end_date,
+                wg_list=doc.group.list_email,
+                settings=settings,
+            ),
+        )
+        (to, cc) = gather_address_lists("doc_wg_call_for_adoption_issued", doc=doc)
+        if doc.group.acronym == "none":
+            to.insert(0, f"{group.acronym}-chairs@ietf.org")
+            to.insert(0, group.list_email)
+        form = IssueCallForAdoptionForm(
+            initial=dict(
+                end_date=end_date,
+                to=", ".join(to),
+                cc=", ".join(cc),
+                subject=subject,
+                body=body,
+            )
+        )
+
+    return render(
+        request,
+        "doc/draft/issue_working_group_call_for_adoption.html",
+        dict(
+            doc=doc,
+            form=form,
+        ),
+    )
+
+class GroupModelChoiceField(forms.ModelChoiceField):
+    def label_from_instance(self, obj):
+        return f"{obj.acronym} - {obj.name}"
+
+
+class WgForm(forms.Form):
+    group = GroupModelChoiceField(
+        queryset=Group.objects.filter(type_id="wg", state="active")
+        .order_by("acronym")
+        .distinct(),
+        required=True,
+        empty_label="Select IETF Working Group",
+    )
+
+    def __init__(self, *args, **kwargs):
+        user = kwargs.pop("user")
+        super(WgForm, self).__init__(*args, **kwargs)
+        if not has_role(user, ["Secretariat", "Area Director"]):
+            self.fields["group"].queryset = self.fields["group"].queryset.filter(
+                role__name_id="chair", role__person=user.person
+            )
+
+
+@role_required("Secretariat", "WG Chair")
+def ask_about_ietf_adoption_call(request, name):
+    doc = get_object_or_404(Document, name=name)
+    if doc.stream is not None or doc.group.acronym != "none":
+        raise Http404
+    if request.method == "POST":
+        form = WgForm(request.POST, user=request.user)
+        if form.is_valid():
+            group = form.cleaned_data["group"]
+            return redirect(issue_wg_call_for_adoption, name=doc.name, acronym=group.acronym)
+    else:
+        form = WgForm(initial={"group": None}, user=request.user)
+    return render(
+        request,
+        "doc/draft/ask_about_ietf_adoption.html",
+        dict(
+            doc=doc,
+            form=form,
+        ),
+    )
diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py
index e2893a90f7..1f634278be 100644
--- a/ietf/ietfauth/utils.py
+++ b/ietf/ietfauth/utils.py
@@ -211,9 +211,9 @@ def role_required(*role_names):
 
 # specific permissions
 
+
 def is_authorized_in_doc_stream(user, doc):
-    """Return whether user is authorized to perform stream duties on
-    document."""
+    """Is user authorized to perform stream duties on doc?"""
     if has_role(user, ["Secretariat"]):
         return True
 
diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html
index 6414538283..eab1d779fb 100644
--- a/ietf/templates/doc/document_draft.html
+++ b/ietf/templates/doc/document_draft.html
@@ -63,7 +63,12 @@
                     <td class="edit">
                         {% if doc.stream and can_edit_stream_info and doc.stream.slug != "legacy" and not snapshot %}
                             <a class="btn btn-primary btn-sm"
-                               href="{% url 'ietf.doc.views_draft.change_stream_state' name=doc.name state_type=stream_state_type_slug %}">
+                               {% if doc|is_doc_ietf_adoptable or doc|can_issue_ietf_wg_lc or doc|can_submit_to_iesg %}
+                                 href="{% url 'ietf.doc.views_draft.offer_wg_action_helpers' name=doc.name %}"
+                               {% else %}
+                                 href="{% url 'ietf.doc.views_draft.change_stream_state' name=doc.name state_type=stream_state_type_slug %}"
+                               {% endif %}
+                            >
                                 Edit
                             </a>
                         {% endif %}
diff --git a/ietf/templates/doc/draft/ask_about_ietf_adoption.html b/ietf/templates/doc/draft/ask_about_ietf_adoption.html
new file mode 100644
index 0000000000..d19e4572b7
--- /dev/null
+++ b/ietf/templates/doc/draft/ask_about_ietf_adoption.html
@@ -0,0 +1,22 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2015-2025, All Rights Reserved #}
+{% load django_bootstrap5 ietf_filters origin %}
+{% block title %}Manage Adoption of {{ doc }}{% endblock %}
+{% block content %}
+    {% origin %}
+    <h1>
+        Manage Adoption
+        <br>
+        <small class="text-body-secondary">{{ doc }}</small>
+    </h1>
+    <div>
+        Do you wish to issue an IETF Working Group call for adoption to one of these working groups?
+    </div>
+    <form method="post">
+        {% csrf_token %}
+        {% bootstrap_form form %}
+        <button type="submit" class="btn btn-primary">Issue IETF WG Call For Adoption</button>
+        <a class="btn btn-secondary" href="{% url 'ietf.doc.views_draft.adopt_draft' name=doc.name %}">No, I wish to manage adoption directly, perhaps with non-IETF-stream groups</a>
+        <a class="btn btn-secondary float-end" href="{{ doc.get_absolute_url }}">Back</a>
+    </form>
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/doc/draft/change_stream_state.html b/ietf/templates/doc/draft/change_stream_state.html
index 0b13e02fdf..7f3132b3c8 100644
--- a/ietf/templates/doc/draft/change_stream_state.html
+++ b/ietf/templates/doc/draft/change_stream_state.html
@@ -1,7 +1,6 @@
 {% extends "base.html" %}
-{# Copyright The IETF Trust 2015, All Rights Reserved #}
-{% load origin %}
-{% load django_bootstrap5 %}
+{# Copyright The IETF Trust 2015-2025, All Rights Reserved #}
+{% load django_bootstrap5 ietf_filters origin %}
 {% block title %}Change {{ state_type.label }} for {{ doc }}{% endblock %}
 {% block content %}
     {% origin %}
@@ -14,12 +13,10 @@ <h1>
         <a class="my-3 btn btn-info"
            href="{% url 'ietf.doc.views_help.state_help' type=state_type.slug %}">Help on states</a>
         <p>
-            <b>Move document to {{ next_states|pluralize:"to one of" }} the recommended next state{{ next_states|pluralize }}:</b>
+            <b>Move document to {{ next_states|pluralize:"one of" }} the recommended next state{{ next_states|pluralize }}:</b>
         </p>
         {% for state in next_states %}
-            {% if state.slug == 'sub-pub' %}
-                <a type="button" class="btn btn-primary next-state" href="{% url 'ietf.doc.views_draft.to_iesg' name=doc.name %}">{{ state.name }}</a>
-            {% else %}
+            {% if state.slug != 'sub-pub' and state.slug != "wg-lc" %}
                 <button type="button" class="btn btn-primary next-state" data-state="{{ state.pk }}">{{ state.name }}</button>
             {% endif %}
         {% endfor %}
@@ -28,7 +25,13 @@ <h1>
         {% csrf_token %}
         {% bootstrap_form form %}
         <button type="submit" class="btn btn-primary">Submit</button>
-        <a class="btn btn-secondary float-end" href="{{ doc.get_absolute_url }}">Back</a>
+        <a class="btn btn-secondary float-end"
+          {% if doc|can_issue_ietf_wg_lc or doc|can_submit_to_iesg %}
+            href="{% url 'ietf.doc.views_draft.offer_wg_action_helpers' name=doc.name %}"
+          {% else %} 
+            href="{{ doc.get_absolute_url }}"
+          {% endif %}
+        >Back</a>
     </form>
 {% endblock %}
 {% block js %}
diff --git a/ietf/templates/doc/draft/issue_working_group_call_for_adoption.html b/ietf/templates/doc/draft/issue_working_group_call_for_adoption.html
new file mode 100644
index 0000000000..61094b053a
--- /dev/null
+++ b/ietf/templates/doc/draft/issue_working_group_call_for_adoption.html
@@ -0,0 +1,61 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% load origin django_bootstrap5 static %}
+={% block title %}Issue Working Group Call for Adoption of {{ doc }}{% endblock %}
+{% block pagehead %}
+    <link rel="stylesheet" href="{% static 'ietf/css/datepicker.css' %}">
+{% endblock %}
+{% block content %}
+    {% origin %}
+    <h1>
+        Issue Working Group Call for Adoption
+        <br>
+        <small class="text-body-secondary">{{ doc }}</small>
+    </h1>
+    {% if form.errors %}
+        <p class="alert alert-danger my-3">
+            Please correct the following:
+        </p>
+    {% endif %}
+    {% bootstrap_form_errors form %}
+    <form class="mt-5" method="post">
+        {% csrf_token %}
+        {% bootstrap_form form %}
+        <div id="end_date_warn" class="alert alert-warning d-none" role="alert">
+            All dates have been updated automatically in the Subject and Body fields with the End date value. Since you previously edited these fields manually, review both to ensure they match your expectations.
+        </div>
+        <button type="submit" class="btn btn-primary">Submit</button>
+        <a class="btn btn-secondary float-end" href="{{ doc.get_absolute_url }}">Back</a>
+    </form>
+{% endblock %}
+{% block js %}
+    <script src="{% static 'ietf/js/datepicker.js' %}"></script>
+    <script>
+        (function () {
+            const datepickerElm = document.querySelector('#id_end_date')
+            const subjectElm = document.querySelector('#id_subject')
+            const contentElm = document.querySelector('#id_body')
+            const warnElm = document.querySelector('#end_date_warn')
+            let isSubjectOrContentDirty = false
+
+            // Handle date auto-replace in subject + content
+            datepickerElm.addEventListener('changeDate', ev => {
+                subjectElm.value = subjectElm.value.replaceAll(/2[0-9]{3}-[0-9]{2}-[0-9]{2}/g, datepickerElm.value)
+                contentElm.value = contentElm.value.replaceAll(/2[0-9]{3}-[0-9]{2}-[0-9]{2}/g, datepickerElm.value)
+
+                // Warn about auto-replace on dirtied fields
+                if (isSubjectOrContentDirty) {
+                    warnElm.classList.remove('d-none')
+                }
+            })
+
+            // Detect dirtied fields
+            subjectElm.addEventListener('input', () => {
+                isSubjectOrContentDirty = true
+            })
+            contentElm.addEventListener('input', () => {
+                isSubjectOrContentDirty = true
+            })
+        })()
+    </script>
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/doc/draft/issue_working_group_last_call.html b/ietf/templates/doc/draft/issue_working_group_last_call.html
new file mode 100644
index 0000000000..d6f35a0e82
--- /dev/null
+++ b/ietf/templates/doc/draft/issue_working_group_last_call.html
@@ -0,0 +1,61 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% load origin django_bootstrap5 static %}
+={% block title %}Issue Working Group Last Call for {{ doc }}{% endblock %}
+{% block pagehead %}
+    <link rel="stylesheet" href="{% static 'ietf/css/datepicker.css' %}">
+{% endblock %}
+{% block content %}
+    {% origin %}
+    <h1>
+        Issue Working Group Last Call
+        <br>
+        <small class="text-body-secondary">{{ doc }}</small>
+    </h1>
+    {% if form.errors %}
+        <p class="alert alert-danger my-3">
+            Please correct the following:
+        </p>
+    {% endif %}
+    {% bootstrap_form_errors form %}
+    <form class="mt-5" method="post">
+        {% csrf_token %}
+        {% bootstrap_form form %}
+        <div id="end_date_warn" class="alert alert-warning d-none" role="alert">
+            All dates have been updated automatically in the Subject and Body fields with the End date value. Since you previously edited these fields manually, review both to ensure they match your expectations.
+        </div>
+        <button type="submit" class="btn btn-primary">Submit</button>
+        <a class="btn btn-secondary float-end" href="{{ doc.get_absolute_url }}">Back</a>
+    </form>
+{% endblock %}
+{% block js %}
+    <script src="{% static 'ietf/js/datepicker.js' %}"></script>
+    <script>
+        (function () {
+            const datepickerElm = document.querySelector('#id_end_date')
+            const subjectElm = document.querySelector('#id_subject')
+            const contentElm = document.querySelector('#id_body')
+            const warnElm = document.querySelector('#end_date_warn')
+            let isSubjectOrContentDirty = false
+
+            // Handle date auto-replace in subject + content
+            datepickerElm.addEventListener('changeDate', ev => {
+                subjectElm.value = subjectElm.value.replaceAll(/2[0-9]{3}-[0-9]{2}-[0-9]{2}/g, datepickerElm.value)
+                contentElm.value = contentElm.value.replaceAll(/2[0-9]{3}-[0-9]{2}-[0-9]{2}/g, datepickerElm.value)
+
+                // Warn about auto-replace on dirtied fields
+                if (isSubjectOrContentDirty) {
+                    warnElm.classList.remove('d-none')
+                }
+            })
+
+            // Detect dirtied fields
+            subjectElm.addEventListener('input', () => {
+                isSubjectOrContentDirty = true
+            })
+            contentElm.addEventListener('input', () => {
+                isSubjectOrContentDirty = true
+            })
+        })()
+    </script>
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/doc/draft/wg_action_helpers.html b/ietf/templates/doc/draft/wg_action_helpers.html
new file mode 100644
index 0000000000..d21f3c0926
--- /dev/null
+++ b/ietf/templates/doc/draft/wg_action_helpers.html
@@ -0,0 +1,25 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2015-2025, All Rights Reserved #}
+{% load django_bootstrap5 ietf_filters origin %}
+{% block title %}Change IETF WG state for {{ doc }}{% endblock %}
+{% block content %}
+    {% origin %}
+    <h1>
+        Change IETF WG state
+        <br>
+        <small class="text-body-secondary">{{ doc }}</small>
+    </h1>
+    <div>
+        {% if doc|is_doc_ietf_adoptable %}
+          <a class="btn btn-primary" href="{% url 'ietf.doc.views_draft.issue_wg_call_for_adoption' name=doc.name acronym=doc.group.acronym %}" id="id_wgadopt_button">Issue WG Call for Adoption</a>
+        {% endif %}
+        {% if doc|can_issue_ietf_wg_lc %}
+          <a class="btn btn-primary" href="{% url 'ietf.doc.views_draft.issue_wg_lc' name=doc.name %}" id="id_wglc_button">Issue{% if doc|has_had_ietf_wg_lc %} Another{% endif %} Working Group Last Call</a>
+        {% endif %}
+        {% if doc|can_submit_to_iesg %}
+          <a class="btn btn-primary" href="{% url 'ietf.doc.views_draft.to_iesg' name=doc.name %}" id="id_pubreq_button">Submit to IESG for Publication</a>
+        {% endif %}
+        <a class="btn btn-primary" href="{% url 'ietf.doc.views_draft.change_stream_state' name=doc.name state_type='draft-stream-ietf' %}">Set any WG state directly</a>
+        <a class="btn btn-secondary float-end" href="{{ doc.get_absolute_url }}">Back</a>
+    </div>
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt b/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt
index c4a2401bc2..15ace9495b 100644
--- a/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt
+++ b/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt
@@ -1,15 +1,13 @@
-{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %}
-Subject: {{ subject }}
+{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %}This message starts a {{group.acronym}} WG Call for Adoption of:
+{{ doc.name }}-{{ doc.rev }}
 
-This message starts a {{ cfa_duration_weeks }}-week Call for Adoption for this document. 
+
+This Working Group Call for Adoption ends on {{ end_date }}
 
 Abstract: 
 {{ doc.abstract }}
 
-File can be retrieved from:
-{{ url }}
-
-Please reply to this message keeping {{ wg_list }} in copy by indicating whether you support or not the adoption of this draft as a WG document. Comments to motivate your preference are highly appreciated. 
+Please reply to this message and indicate whether or not you support adoption of this Internet-Draft by the {{group.acronym}} WG. Comments to explain your preference are greatly appreciated. Please reply to all recipients of this message and include this message in your response.
 
 Authors, and WG participants in general, are reminded of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [2]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3].
 
@@ -17,5 +15,17 @@ Thank you.
 [1] https://datatracker.ietf.org/doc/bcp78/
 [2] https://datatracker.ietf.org/doc/bcp79/
 [3] https://datatracker.ietf.org/doc/rfc6701/
+
+The IETF datatracker status page for this Internet-Draft is:
+{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name=doc.name %}
+{% if doc.submission.xml_version == "3" %}
+There is also an HTML version available at:
+{{ settings.IETF_ID_ARCHIVE_URL }}{{ doc.name }}-{{ doc.rev }}.html{% else %}
+There is also an HTMLized version available at:
+{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_html' name=doc.name rev=doc.rev %}{% endif %}
+{% if doc.rev != "00" %}
+A diff from the previous version is available at:
+{{settings.RFCDIFF_BASE_URL}}?url2={{ doc.name }}-{{ doc.rev }}
+{% endif %}
 {% endfilter %}
 {% endautoescape %}
diff --git a/ietf/templates/doc/mail/wg_last_call_issued.txt b/ietf/templates/doc/mail/wg_last_call_issued.txt
index 35b1e149d7..114f8bc5e2 100644
--- a/ietf/templates/doc/mail/wg_last_call_issued.txt
+++ b/ietf/templates/doc/mail/wg_last_call_issued.txt
@@ -1,7 +1,7 @@
-{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %}
-Subject: {{ subject }}
+{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %}This message starts a WG Last Call for:
+{{ doc.name }}-{{ doc.rev }}
 
-This message starts a {{ wglc_duration_weeks }}-week WG Last Call for this document. 
+This Working Group Last Call ends on {{ end_date }}
 
 Abstract: 
 {{ doc.abstract }}
@@ -9,14 +9,26 @@ Abstract:
 File can be retrieved from:
 {{ url }}
 
-Please review and indicate your support or objection to proceed with the publication of this document by replying to this email keeping {{ wg_list }} in copy. Objections should be motivated and suggestions to resolve them are highly appreciated.
+Please review and indicate your support or objection to proceed with the publication of this document by replying to this email keeping {{ wg_list }} in copy. Objections should be explained and suggestions to resolve them are highly appreciated.
 
-Authors, and WG participants in general, are reminded again of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [1]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3].
+Authors, and WG participants in general, are reminded of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [1]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3].
 
 Thank you.
 
 [1] https://datatracker.ietf.org/doc/bcp78/
 [2] https://datatracker.ietf.org/doc/bcp79/
 [3] https://datatracker.ietf.org/doc/rfc6701/
+
+The IETF datatracker status page for this Internet-Draft is:
+{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name=doc.name %}
+{% if doc.submission.xml_version == "3" %}
+There is also an HTML version available at:
+{{ settings.IETF_ID_ARCHIVE_URL }}{{ doc.name }}-{{ doc.rev }}.html{% else %}
+There is also an HTMLized version available at:
+{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_html' name=doc.name rev=doc.rev %}{% endif %}
+{% if doc.rev != "00" %}
+A diff from the previous version is available at:
+{{settings.RFCDIFF_BASE_URL}}?url2={{ doc.name }}-{{ doc.rev }}
+{% endif %}
 {% endfilter %}
 {% endautoescape %}

From fb50ac0d90f7a8d5b5ebe7f88ce08be2e9ea2acf Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 16 Dec 2025 11:58:05 -0400
Subject: [PATCH 527/656] chore: optional opentelemetry (#10112)

* chore: optional opentelemetry

* chore: add comment
---
 dev/build/gunicorn.conf.py | 45 +++++++++++++++++++++++---------------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py
index c54b24a054..9af4478685 100644
--- a/dev/build/gunicorn.conf.py
+++ b/dev/build/gunicorn.conf.py
@@ -135,21 +135,30 @@ def post_request(worker, req, environ, resp):
 def post_fork(server, worker):
     server.log.info("Worker spawned (pid: %s)", worker.pid)
 
-    resource = Resource.create(attributes={
-        "service.name": "datatracker",
-        "service.version": ietf.__version__,
-        "service.instance.id": worker.pid,
-        "service.namespace": "datatracker",
-        "deployment.environment.name": os.environ.get("DATATRACKER_SERVICE_ENV", "dev")
-    })
-
-    trace.set_tracer_provider(TracerProvider(resource=resource))
-    otlp_exporter = OTLPSpanExporter(endpoint="https://heimdall-otlp.ietf.org/v1/traces")
-
-    trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter))
-
-    # Instrumentations
-    DjangoInstrumentor().instrument()
-    Psycopg2Instrumentor().instrument()
-    PymemcacheInstrumentor().instrument()
-    RequestsInstrumentor().instrument()
+    # Setting DATATRACKER_OPENTELEMETRY_ENABLE=all in the environment will enable all
+    # opentelemetry instrumentations. Individual instrumentations can be selected by
+    # using a space-separated list. See the code below for available instrumentations.
+    telemetry_env = os.environ.get("DATATRACKER_OPENTELEMETRY_ENABLE", "").strip()
+    if telemetry_env != "":
+        enabled_telemetry = [tok.strip().lower() for tok in telemetry_env.split()]
+        resource = Resource.create(attributes={
+            "service.name": "datatracker",
+            "service.version": ietf.__version__,
+            "service.instance.id": worker.pid,
+            "service.namespace": "datatracker",
+            "deployment.environment.name": os.environ.get("DATATRACKER_SERVICE_ENV", "dev")
+        })
+        trace.set_tracer_provider(TracerProvider(resource=resource))
+        otlp_exporter = OTLPSpanExporter(endpoint="https://heimdall-otlp.ietf.org/v1/traces")
+    
+        trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter))
+    
+        # Instrumentations
+        if "all" in enabled_telemetry or "django" in enabled_telemetry: 
+            DjangoInstrumentor().instrument()
+        if "all" in enabled_telemetry or "psycopg2" in enabled_telemetry: 
+            Psycopg2Instrumentor().instrument()
+        if "all" in enabled_telemetry or "pymemcache" in enabled_telemetry: 
+            PymemcacheInstrumentor().instrument()
+        if "all" in enabled_telemetry or "requests" in enabled_telemetry: 
+            RequestsInstrumentor().instrument()

From 58430dbdfdf306c107fe62cae97e23050688e56a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 16 Dec 2025 13:44:41 -0400
Subject: [PATCH 528/656] refactor: no delete from inline admins (#10113)

* refactor: no delete from TabularInlines

* refactor: no delete from StackedInlines
---
 ietf/doc/admin.py      | 11 +++++++----
 ietf/group/admin.py    |  7 ++++---
 ietf/ipr/admin.py      |  5 +++--
 ietf/liaisons/admin.py |  5 +++--
 ietf/meeting/admin.py  |  9 +++++----
 ietf/name/admin.py     |  3 ++-
 ietf/person/admin.py   |  5 +++--
 ietf/utils/admin.py    | 13 +++++++++++++
 8 files changed, 40 insertions(+), 18 deletions(-)

diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py
index 745536f9a1..920e389a9a 100644
--- a/ietf/doc/admin.py
+++ b/ietf/doc/admin.py
@@ -15,6 +15,7 @@
     ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder,
     BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject )
 
+from ietf.utils.admin import SaferTabularInline
 from ietf.utils.validators import validate_external_resource_value
 
 class StateTypeAdmin(admin.ModelAdmin):
@@ -28,17 +29,19 @@ class StateAdmin(admin.ModelAdmin):
     filter_horizontal = ["next_states"]
 admin.site.register(State, StateAdmin)
 
-class DocAuthorInline(admin.TabularInline):
+class DocAuthorInline(SaferTabularInline):
     model = DocumentAuthor
     raw_id_fields = ['person', 'email']
     extra = 1
+    can_delete = False
+    show_change_link = True
 
-class DocActionHolderInline(admin.TabularInline):
+class DocActionHolderInline(SaferTabularInline):
     model = DocumentActionHolder
     raw_id_fields = ['person']
     extra = 1
 
-class RelatedDocumentInline(admin.TabularInline):
+class RelatedDocumentInline(SaferTabularInline):
     model = RelatedDocument
     fk_name= 'source'
     def this(self, instance):
@@ -48,7 +51,7 @@ def this(self, instance):
     raw_id_fields = ['target']
     extra = 1
 
-class AdditionalUrlInLine(admin.TabularInline):
+class AdditionalUrlInLine(SaferTabularInline):
     model = DocumentURL
     fields = ['tag','desc','url',]
     extra = 1
diff --git a/ietf/group/admin.py b/ietf/group/admin.py
index fedec49d85..685c10aeea 100644
--- a/ietf/group/admin.py
+++ b/ietf/group/admin.py
@@ -26,14 +26,15 @@
     MilestoneGroupEvent, GroupExtResource, Appeal, AppealArtifact )
 from ietf.name.models import GroupTypeName
 
-from ietf.utils.validators import validate_external_resource_value
+from ietf.utils.admin import SaferTabularInline
 from ietf.utils.response import permission_denied
+from ietf.utils.validators import validate_external_resource_value
 
-class RoleInline(admin.TabularInline):
+class RoleInline(SaferTabularInline):
     model = Role
     raw_id_fields = ["person", "email"]
 
-class GroupURLInline(admin.TabularInline):
+class GroupURLInline(SaferTabularInline):
     model = GroupURL
 
 class GroupForm(forms.ModelForm):
diff --git a/ietf/ipr/admin.py b/ietf/ipr/admin.py
index 1a8a908dcd..d6a320203b 100644
--- a/ietf/ipr/admin.py
+++ b/ietf/ipr/admin.py
@@ -17,6 +17,7 @@
     NonDocSpecificIprDisclosure,
     LegacyMigrationIprEvent,
 )
+from ietf.utils.admin import SaferTabularInline
 
 # ------------------------------------------------------
 # ModelAdmins
@@ -29,13 +30,13 @@ class Meta:
           'sections':forms.TextInput,
         }
 
-class IprDocRelInline(admin.TabularInline):
+class IprDocRelInline(SaferTabularInline):
     model = IprDocRel
     form = IprDocRelAdminForm
     raw_id_fields = ['document']
     extra = 1
 
-class RelatedIprInline(admin.TabularInline):
+class RelatedIprInline(SaferTabularInline):
     model = RelatedIpr
     raw_id_fields = ['target']
     fk_name = 'source'
diff --git a/ietf/liaisons/admin.py b/ietf/liaisons/admin.py
index 21515ed1a3..d873cce536 100644
--- a/ietf/liaisons/admin.py
+++ b/ietf/liaisons/admin.py
@@ -7,15 +7,16 @@
 
 from ietf.liaisons.models import  ( LiaisonStatement, LiaisonStatementEvent,
     RelatedLiaisonStatement, LiaisonStatementAttachment )
+from ietf.utils.admin import SaferTabularInline
 
 
-class RelatedLiaisonStatementInline(admin.TabularInline):
+class RelatedLiaisonStatementInline(SaferTabularInline):
     model = RelatedLiaisonStatement
     fk_name = 'source'
     raw_id_fields = ['target']
     extra = 1
 
-class LiaisonStatementAttachmentInline(admin.TabularInline):
+class LiaisonStatementAttachmentInline(SaferTabularInline):
     model = LiaisonStatementAttachment
     raw_id_fields = ['document']
     extra = 1
diff --git a/ietf/meeting/admin.py b/ietf/meeting/admin.py
index d886a9a4b6..03abf5c029 100644
--- a/ietf/meeting/admin.py
+++ b/ietf/meeting/admin.py
@@ -10,6 +10,7 @@
     SessionPresentation, ImportantDate, SlideSubmission, SchedulingEvent, BusinessConstraint,
     ProceedingsMaterial, MeetingHost, Registration, RegistrationTicket,
     AttendanceTypeName)
+from ietf.utils.admin import SaferTabularInline
 
 
 class UrlResourceAdmin(admin.ModelAdmin):
@@ -18,7 +19,7 @@ class UrlResourceAdmin(admin.ModelAdmin):
     raw_id_fields = ['room', ]
 admin.site.register(UrlResource, UrlResourceAdmin)
 
-class UrlResourceInline(admin.TabularInline):
+class UrlResourceInline(SaferTabularInline):
     model = UrlResource
 
 class RoomAdmin(admin.ModelAdmin):
@@ -28,7 +29,7 @@ class RoomAdmin(admin.ModelAdmin):
 
 admin.site.register(Room, RoomAdmin)
 
-class RoomInline(admin.TabularInline):
+class RoomInline(SaferTabularInline):
     model = Room
 
 class MeetingAdmin(admin.ModelAdmin):
@@ -93,7 +94,7 @@ def name_lower(self, instance):
 
 admin.site.register(Constraint, ConstraintAdmin)
 
-class SchedulingEventInline(admin.TabularInline):
+class SchedulingEventInline(SaferTabularInline):
     model = SchedulingEvent
     raw_id_fields = ["by"]
 
@@ -244,7 +245,7 @@ def queryset(self, request, queryset):
             return queryset.filter(tickets__attendance_type__slug=self.value()).distinct()
         return queryset
 
-class RegistrationTicketInline(admin.TabularInline):
+class RegistrationTicketInline(SaferTabularInline):
     model = RegistrationTicket
 
 class RegistrationAdmin(admin.ModelAdmin):
diff --git a/ietf/name/admin.py b/ietf/name/admin.py
index 4336e0569c..b89d6d141c 100644
--- a/ietf/name/admin.py
+++ b/ietf/name/admin.py
@@ -57,6 +57,7 @@
 
 
 from ietf.stats.models import CountryAlias
+from ietf.utils.admin import SaferTabularInline
 
 
 class NameAdmin(admin.ModelAdmin):
@@ -86,7 +87,7 @@ class GroupTypeNameAdmin(NameAdmin):
 admin.site.register(GroupTypeName, GroupTypeNameAdmin)
 
 
-class CountryAliasInline(admin.TabularInline):
+class CountryAliasInline(SaferTabularInline):
     model = CountryAlias
     extra = 1
 
diff --git a/ietf/person/admin.py b/ietf/person/admin.py
index cd8ca2abf1..f46edcf8ae 100644
--- a/ietf/person/admin.py
+++ b/ietf/person/admin.py
@@ -7,6 +7,7 @@
 from ietf.person.models import Email, Alias, Person, PersonalApiKey, PersonEvent, PersonApiKeyEvent, PersonExtResource
 from ietf.person.name import name_parts
 
+from ietf.utils.admin import SaferStackedInline, SaferTabularInline
 from ietf.utils.validators import validate_external_resource_value
 
 
@@ -16,7 +17,7 @@ class EmailAdmin(simple_history.admin.SimpleHistoryAdmin):
     search_fields = ["address", "person__name", ]
 admin.site.register(Email, EmailAdmin)
     
-class EmailInline(admin.TabularInline):
+class EmailInline(SaferTabularInline):
     model = Email
 
 class AliasAdmin(admin.ModelAdmin):
@@ -25,7 +26,7 @@ class AliasAdmin(admin.ModelAdmin):
     raw_id_fields = ["person"]
 admin.site.register(Alias, AliasAdmin)
 
-class AliasInline(admin.StackedInline):
+class AliasInline(SaferStackedInline):
     model = Alias
 
 class PersonAdmin(simple_history.admin.SimpleHistoryAdmin):
diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py
index 6c1c8726e1..e6324ad7cd 100644
--- a/ietf/utils/admin.py
+++ b/ietf/utils/admin.py
@@ -51,6 +51,19 @@ def _link(self):
     _link.admin_order_field = ordering
     return _link
 
+
+class SaferStackedInline(admin.StackedInline):
+    """StackedInline without delete by default"""
+    can_delete = False  # no delete button
+    show_change_link = True  # show a link to the resource (where it can be deleted)
+
+
+class SaferTabularInline(admin.TabularInline):
+    """TabularInline without delete by default"""
+    can_delete = False  # no delete button
+    show_change_link = True  # show a link to the resource (where it can be deleted)
+
+
 from .models import DumpInfo
 class DumpInfoAdmin(admin.ModelAdmin):
     list_display = ['date', 'host', 'tz']

From 4f0102d4f48e59ab65e0545959315f060a9c3a11 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 16 Dec 2025 14:28:02 -0400
Subject: [PATCH 529/656] fix: remove redundant options (#10117)

---
 ietf/doc/admin.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py
index 920e389a9a..8f26b222e1 100644
--- a/ietf/doc/admin.py
+++ b/ietf/doc/admin.py
@@ -33,8 +33,6 @@ class DocAuthorInline(SaferTabularInline):
     model = DocumentAuthor
     raw_id_fields = ['person', 'email']
     extra = 1
-    can_delete = False
-    show_change_link = True
 
 class DocActionHolderInline(SaferTabularInline):
     model = DocumentActionHolder

From 4f1d8000b47aa3a3deadabfa4a863cb741227281 Mon Sep 17 00:00:00 2001
From: Orie Steele <orie@or13.io>
Date: Fri, 19 Dec 2025 08:38:07 -0600
Subject: [PATCH 530/656] Merge pull request #10135 from OR13/patch-1

fix: change title from 'IAB news we can use' to 'IESG Liaison News'
---
 ietf/iesg/agenda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/iesg/agenda.py b/ietf/iesg/agenda.py
index 587713089f..ace4c9ec40 100644
--- a/ietf/iesg/agenda.py
+++ b/ietf/iesg/agenda.py
@@ -133,7 +133,7 @@ def agenda_sections():
         ('4.2', {'title':"WG rechartering"}),
         ('4.2.1', {'title':"Under evaluation for IETF review", 'docs':[]}),
         ('4.2.2', {'title':"Proposed for approval", 'docs':[]}),
-        ('5', {'title':"IAB news we can use"}),
+        ('5', {'title':"IESG Liaison News"}),
         ('6', {'title':"Management issues"}),
         ('7', {'title':"Any Other Business (WG News, New Proposals, etc.)"}),
         ])

From 7f566673abef2df32c8bf477fda1f6bf63fc0cf9 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 19 Dec 2025 12:15:11 -0600
Subject: [PATCH 531/656] feat: move base containers to trixie (#9535) (#10127)

* Use latest release of yanglint, e.g. libyang3-tools.

* Remove apt-utils, since apt-key is removed.

Fixes: #8701

Co-authored-by: avtobiff <avtobiff@gmail.com>
Co-authored-by: Per Andersson <per@ionio.se>
---
 docker/base.Dockerfile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile
index 2501636049..1b1f5264b8 100644
--- a/docker/base.Dockerfile
+++ b/docker/base.Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.12-bookworm
+FROM python:3.12-trixie
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -7,7 +7,7 @@ ENV NODE_MAJOR=16
 # Update system packages
 RUN apt-get update \
     && apt-get -qy upgrade \
-    && apt-get -y install --no-install-recommends apt-utils dialog 2>&1
+    && apt-get -y install --no-install-recommends dialog 2>&1
 
 # Add Node.js Source
 RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \
@@ -51,7 +51,6 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends
 	libgtk2.0-0 \
 	libgtk-3-0 \
 	libnotify-dev \
-	libgconf-2-4 \
 	libgbm-dev \
 	libnss3 \
 	libxss1 \
@@ -60,7 +59,7 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends
 	libmagic-dev \
 	libmariadb-dev \
 	libmemcached-tools \
-	libyang2-tools \
+	libyang3-tools \
 	locales \
 	make \
 	mariadb-client \

From e510c8e4a9016c267a1b02557ebaa3656f7f13c4 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Fri, 19 Dec 2025 18:29:28 +0000
Subject: [PATCH 532/656] ci: update base image target version to 20251219T1815

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index ae59ba1440..d40a734f89 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20251201T1548
+FROM ghcr.io/ietf-tools/datatracker-app-base:20251219T1815
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 726f080c67..8b107939b6 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20251201T1548
+20251219T1815

From 6ee56b52c1b412c0ba8cacd6421a3c8b529f5a4e Mon Sep 17 00:00:00 2001
From: Absit Iniuria <absniuria@nours.page>
Date: Fri, 19 Dec 2025 19:02:42 +0000
Subject: [PATCH 533/656] fix: adjust rendering of agenda sesh pop-up on mobile
 (#10134)

* fix: adjust popup display on mobile

* fix: adjust ui spacing
---
 client/agenda/AgendaDetailsModal.vue | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/client/agenda/AgendaDetailsModal.vue b/client/agenda/AgendaDetailsModal.vue
index 2582bf2159..69c8ef8b53 100644
--- a/client/agenda/AgendaDetailsModal.vue
+++ b/client/agenda/AgendaDetailsModal.vue
@@ -274,6 +274,7 @@ async function fetchSessionMaterials () {
 <style lang="scss">
 @import "bootstrap/scss/functions";
 @import "bootstrap/scss/variables";
+@import "../shared/breakpoints";
 
 .agenda-eventdetails {
   width: 90vw;
@@ -403,5 +404,29 @@ async function fetchSessionMaterials () {
       font-size: 16px;
     }
   }
+
+  .n-card-header {
+    @media screen and (max-width: $bs5-break-sm) {
+      display: block;
+      padding: 1em;
+    }
+  }
+
+  .n-card__content {
+    @media screen and (max-width: $bs5-break-sm) {
+      padding: 1em;
+
+      .detail-title {
+        display: block;
+      }
+
+      .nav-link,
+      .nav-link.active {
+        display: grid;
+        width: 1em;
+        border-width: 1em;
+      }
+    }
+  }
 }
 </style>

From 3565d8427efdbefb2b36cd2fe0776ad67c55d130 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 19 Dec 2025 15:51:47 -0600
Subject: [PATCH 534/656] fix: revert "feat: move base containers to trixie
 (#9535) (#10127)" (#10140)

This reverts commit 7f566673abef2df32c8bf477fda1f6bf63fc0cf9.
---
 docker/base.Dockerfile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile
index 1b1f5264b8..2501636049 100644
--- a/docker/base.Dockerfile
+++ b/docker/base.Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.12-trixie
+FROM python:3.12-bookworm
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -7,7 +7,7 @@ ENV NODE_MAJOR=16
 # Update system packages
 RUN apt-get update \
     && apt-get -qy upgrade \
-    && apt-get -y install --no-install-recommends dialog 2>&1
+    && apt-get -y install --no-install-recommends apt-utils dialog 2>&1
 
 # Add Node.js Source
 RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \
@@ -51,6 +51,7 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends
 	libgtk2.0-0 \
 	libgtk-3-0 \
 	libnotify-dev \
+	libgconf-2-4 \
 	libgbm-dev \
 	libnss3 \
 	libxss1 \
@@ -59,7 +60,7 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends
 	libmagic-dev \
 	libmariadb-dev \
 	libmemcached-tools \
-	libyang3-tools \
+	libyang2-tools \
 	locales \
 	make \
 	mariadb-client \

From c85ebbf326eb5dc6fa23e315e462ae4fefffb709 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Fri, 19 Dec 2025 22:04:33 +0000
Subject: [PATCH 535/656] ci: update base image target version to 20251219T2152

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index d40a734f89..e05ad51d1b 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20251219T1815
+FROM ghcr.io/ietf-tools/datatracker-app-base:20251219T2152
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 8b107939b6..f8afdadf36 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20251219T1815
+20251219T2152

From 3b07c70435697625d5c20432f619a9f91c5c10e3 Mon Sep 17 00:00:00 2001
From: Absit Iniuria <absniuria@nours.page>
Date: Fri, 9 Jan 2026 15:59:30 +0000
Subject: [PATCH 536/656] feat: provide link to detailed submission status page
 for submission api (#10233)

* feat: include link to detailed submission status page for submission api

* chore: rename endpoint to submission_url

---------

Co-authored-by: nouralmaa <absnuiria@nours.page>
---
 ietf/submit/tests.py | 9 ++++++++-
 ietf/submit/views.py | 4 ++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index ede63d2752..216fc7de6b 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -2404,7 +2404,7 @@ def test_upload_draft(self):
         response = r.json()
         self.assertCountEqual(
             response.keys(),
-            ['id', 'name', 'rev', 'status_url'],
+            ['id', 'name', 'rev', 'status_url', 'submission_url'],
         )
         submission_id = int(response['id'])
         self.assertEqual(response['name'], 'draft-somebody-test')
@@ -2416,6 +2416,13 @@ def test_upload_draft(self):
                 kwargs={'submission_id': submission_id},
             ),
         )
+        self.assertEqual(
+            response['submission_url'],
+            'https://datatracker.example.com' + urlreverse(
+                'ietf.submit.views.submission_status',
+                kwargs={'submission_id': submission_id},
+            )
+        )
         self.assertEqual(mock_task.delay.call_count, 1)
         self.assertEqual(mock_task.delay.call_args.args, (submission_id,))
         submission = Submission.objects.get(pk=submission_id)
diff --git a/ietf/submit/views.py b/ietf/submit/views.py
index 8329a312bb..2db3f51098 100644
--- a/ietf/submit/views.py
+++ b/ietf/submit/views.py
@@ -182,6 +182,10 @@ def err(code, error, messages=None):
                             settings.IDTRACKER_BASE_URL,
                             urlreverse(api_submission_status, kwargs={'submission_id': submission.pk}),
                         ),
+                        'submission_url': urljoin(
+                            settings.IDTRACKER_BASE_URL,
+                            urlreverse("ietf.submit.views.submission_status", kwargs={'submission_id': submission.pk}),
+                        ),
                     }
                 )
             else:

From 967fffaa61a5bb6ba69e2e6766f043dee86799ee Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 9 Jan 2026 12:25:00 -0400
Subject: [PATCH 537/656] fix: f-strings for replicator log msgs (#10234)

---
 ietf/blobdb/replication.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ietf/blobdb/replication.py b/ietf/blobdb/replication.py
index b9d55c9498..d251d3b95c 100644
--- a/ietf/blobdb/replication.py
+++ b/ietf/blobdb/replication.py
@@ -146,11 +146,11 @@ def replicate_blob(bucket, name):
     blob = fetch_blob_via_sql(bucket, name)
     if blob is None:
         if verbose_logging_enabled():
-            log.log("Deleting {bucket}:{name} from replica")
+            log.log(f"Deleting {bucket}:{name} from replica")
         try:
             destination_storage.delete(name)
         except Exception as e:
-            log.log("Failed to delete {bucket}:{name} from replica: {e}")
+            log.log(f"Failed to delete {bucket}:{name} from replica: {e}")
             raise ReplicationError from e
     else:
         # Add metadata expected by the MetadataS3Storage
@@ -170,7 +170,7 @@ def replicate_blob(bucket, name):
         try:
             destination_storage.save(name, file_with_metadata)
         except Exception as e:
-            log.log("Failed to save {bucket}:{name} to replica: {e}")
+            log.log(f"Failed to save {bucket}:{name} to replica: {e}")
             raise ReplicationError from e
 
 

From 79cb013a190080c3cf3cf0032253da9193d36491 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 9 Jan 2026 19:18:59 -0400
Subject: [PATCH 538/656] chore: squelch pyparsing deprecation warnings
 (#10239)

---
 ietf/settings.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ietf/settings.py b/ietf/settings.py
index f8d8a28d65..05eab0f12f 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -35,6 +35,8 @@
 warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="oic.utils.time_util")
 warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="pytz.tzinfo")
 warnings.filterwarnings("ignore", message="'instantiateVariableFont' is deprecated", module="weasyprint")
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="bibtexparser")  # https://github.com/sciunto-org/python-bibtexparser/issues/502
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="pyparsing")  # https://github.com/sciunto-org/python-bibtexparser/issues/502
 
 
 base_path = pathlib.Path(__file__).resolve().parent

From d06001f640315502988ec6eb99ae3d6f08b3fc6c Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 14 Jan 2026 13:55:12 -0400
Subject: [PATCH 539/656] feat: RPC modernization APIs (#9631)

* feat: Add rpc_person API call

* chore: Add OpenAPI yaml for RPC API

* feat: api endpoint for docs submitted to the rpc

* chore: remove some debug

* feat: api for created demo people

* feat: optimize fetching persons

* feat: api for creating demo drafts (#6402)

* fix: Typo in rpcapi.yaml

* refactor: Allow existing Person in create_demo_person (#6398)

* fix: include pubreq docevents in demo drafts (#6404)

* fix: Minor API fixes (#6405)

* chore: Document 404 from rpcapi get_person_by_id

* feat: adding rev to demo doc creation (#6425)

* feat: adding rev to demo doc creation

* fix: remove attempt to control required

* refactor: Replace api token checks with decorator (#6434)

* feat: Add @requires_api_token decorator

* refactor: Use @requires_api_token

* refactor: Tweak api token endpoints

This might be drifting from the design intent, but at least
uses the defined endpoint values. Further cleanup may well
be needed.

* refactor: Improve usability of @requires_api_token

* feat: get_draft_by_id api call (#6446)

* fix: construct the cdn photo url correctly (#6491)

* chore: restructure the rpc api (#6505)

* fix: authenticate person api correctly

* chore: Fix plain_name lookup

* feat: subject_id -> Person api call (#6566)

* feat: subject_id -> Person api call

* doc: Add error responses to openapi spec

* feat: rpc api drafts by names (#6853)

* feat: get stream rfcs were first published into (#7814)

* feat: get stream rfcs were first published into

* chore: black

* fix: deal with the case of no DocHistory having a stream

* fix: return "none" from drafts_by_names if a draft has a stream of None (#7863)

* feat: API changes needed for "real" draft imports (#7877)

* style: black

* fix: submitted as DateTime, not Date

* fix: pk -> id in API

* feat: rpc_draft source_format

* feat: add shepherd to rpc_draft() api

* fix: "unknown" src fmt instead of error

* feat: add intended_std_level to api

* refactor: blank, not None, for shepherd / std_level

* style: black

* fix: typo in drafts_by_names() (#7912)

* feat: support for building rfc authors (#7983)

* feat: support for building rfc authors

* chore: copyrights

* feat: api to gather draft authors (#8126)

* feat: api to gather draft authors

* chore: black

* ci: tag feature branch release images

* chore: fix git nonsense

* feat: API for RFC metadata fetch/filtering (#8291)

* feat: RFC API for rfceditor website (WIP)

* feat: pagination + ordered RFC index

* feat: filter by publication date

* feat: stream + stream filtering

* feat: DOI

* feat: group/area for RFCs

* feat: group/area filtering

* feat: result sorting

* refactor: send rfc number, not name

* feat: search rfc title/abstract

* style: Black

* feat: add 'status' field

* feat: filter by 'status'

* style: remove redundant parentheses

* feat: add updated_by/obsoleted_by fields

* feat: add 'abstract' to rpc api

* chore: fix unused/duplicate imports

* chore: fix mypy lint

* chore: unused import

* feat: retrieve single rfc, including text (#8346)

* feat: retrieve single rfc

Use RFC number instead of doc PK as id

* feat: include text in single-rfc response

* chore: drop doc id from api response

* fix: many=True for identifiers (#8425)

* feat: add more rfc api fields (many stubs)

* chore: adding postscript (ps) to rfc meta serializer (#8560)

* fix: acknowledge not-issued in RfcStatusSlugT

* feat: Add API call to get references

* fix: Filter drafts

* chore: Optimize the data query

* test: Test for norminative references API call

* chore: Fix typos in tests

* chore: Fix typos

* refactor: Separate demo logic (#8937)

* refactor: Separate demo logic

* chore: Skip tests

* fix: trailing slashes for all rpc api endpoints (#8940)

* chore: Add RPC references API call to OpenAPI spec (#8941)

* chore: Remove line noise

* chore: Add RPC references API call to OpenAPI spec

* chore: Update rpcapi.yaml

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>

* fix: Fix OpenAPI spec errors (#8943)

* chore: Remove more line noise

* fix: Fix OpenAPI spec errors

* feat: include picture URL in rpc_person API (#9009)

* feat: person search endpoint (#9062)

* feat: person search endpoint

* refactor: address review comments

  * improved naming of operation/components in API schema
  * reused Person schema component
  * added serializers_rpc.py

* chore: RpcPersonSerializer -> PersonSerializer

Better matches the hand-written schema.

* fix: search for entire term, not word-by-word

* fix: only look at name/plain in search

Including ascii / ascii_short might be useful eventually, but since we only show plain_name in the response it can cause confusing results.

By the same reasoning we could remove email__address as well, but that's useful and I expect we'll include email addresses in our response soon anyway.

* refactor: reimplement purple API in django-rest-framework (#9097)

* refactor: rpc_person -> PersonViewSet

* refactor: rpc_subject_person -> SubjectPersonView

* refactor: rpc_persons -> RpcPersonsView

* refactor: move get_persons into PersonViewSet

Changes the interface to return a list of Persons instead of a map from ID to name.

* refactor: rpc_draft -> DraftViewSet

* refactor: drafts_by_names -> DraftsByNameView

* refactor: submitted_to_rpc -> DraftViewSet

* refactor: rfc_original_stream -> RfcViewSet

* refactor: rpc demo APIs -> viewset

* refactor: get_draft_refs -> DraftViewSet

* refactor: persons_by_email -> PersonViewSet

* refactor: rfc_authors -> RfcViewSet

* refactor: draft_authors -> DraftViewSet

* refactor: avoid \x00 in regex validator

Gets turned into a literal nul somewhere in the process of generating a schema and building a Python client for purple. This has the same effect but avoids the nul.

* fix: missing arg on references() action

* style: ruff, remove unused imports

* style: ruff ruff

* chore: remove rpcapi.yaml

* refactor: move API to /api/purple

Side effect is that the purple API client is named PurpleApi instead of RpcApi.

* fix: get_draft_authors returns DraftWithAuthors

* fix: distinguish CharField flavors

* fix: no serializer validators for draft name/title

This prevents at least one existing draft from being looked up.

* fix: get_draft_authors works with str, not int

* Revert "refactor: avoid \x00 in regex validator"

This reverts commit 63f40cf2

* Revert "Revert "refactor: avoid \x00 in regex validator"" (#9111)

This reverts commit d8656f470045c21542824d7b9b9be41bdcb8866d.

* ci: only migrate blobdb if it is configured

* feat: add email/url to purple person API (#9127)

* feat: expose consensus in submission api

* feat: subseries api for red (#9556)

* refactor: central def of subseries doc types

* feat: subseries doc API

* refactor: optimize queries via prefetch

Reduced 4500 to 18 queries

* chore: remove debug

* fix: fix serialization of draft field

* refactor: clean up prefetch a bit

* feat: filter by subseries type

* fix: restore max_limit for RFC pagination

* feat: add subseries+stub titlepage_name to rfc serializer (#9569)

* feat: add subseries to RfcMetadataSerializer

* feat: titlepage_name for RfcAuthorSerializer

Always blank for now

* chore: update copyrights

* refactor: use py3.12 typing syntax

* fix: renumber migrations

* feat: add consensus on FullDraftSerializer

* feat: add type field in serializer

* feat: tag subseries API endpoints for purple (#9763)

* feat: tag subseries API endpoints for purple

* ruff

* feat: change slugs/names (#9778)

* change slugs/names

* change slug names

* fix: update RfcStatusSlugT

* fix: remove double tag (#9787)

This was meant to include the API call in both purple and red API clients.
It seems this does not work, at least with some generators.
Need to investigate further, but we should be able to work around it.

* feat: rfc authors (#9937)

* feat: rfc authors

* fix: distinct rfc search results

* fix: include titlepage_name in author name searches

* fix: add is_editor to rfcauthor model. Adjust FK on_delete. Tweak admin.

* fix: renumber migration

* refactor: realistic titlepage_name in factory

* refactor: comment + rename iteration var

* chore: bump copyright year

* chore: remove country from RfcAuthor

Not planning to track this

* refactor: make blank=False explicit

* chore: remove country from admin

* fix: author_list() for type=rfc

* fix: blankable RfcAuthor.person

* feat: RfcAuthor-aware document_json()

* feat: limit docs to rfcs for RfcAuthor admin

* test: document_json authors

* fix: use author_names() for get_document_emails()

* feat: suggest affiliation based on RfcAuthor

* chore: revert "remove country from RfcAuthor"

This reverts commit 3044d10439bed2137e28356f27250600ce7a2529.

* chore: revert "remove country from admin"

This reverts commit 208879368fa5be5ecf79ae67b0be636db8a7e81f.

* feat: use rfcauthors for nomcom eligibility (#9629)

* refactor: author-based eligibility cleanup

* feat: use rfcauthor recs for nomcom eligbility

* chore: remove commented code

* refactor: factor out helper for testing

* test: test_get_qualified_author_queryset

* fix: restore a necessary import

* test: fix test_elig_by_author

* test: fix test_decorate_volunteers_with_qualifications

* test: add comment

* fix: drop test for draft-rfceditor state

Attempted to limit to drafts literally in the queue, but was not a
valid check when looking back in time. As a practical matter, the test
is not necessary.

* fix: exclude double counting, not rfc pub state

* test: update test

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>

* fix: renumber migrations

* feat: basic RfcAuthor API (#9940)

* refactor: move get_rfc_authors API URL

* refactor: drop format_suffixes from router

Creates a bunch of API endpoints we have no intention of ever using

* feat: RfcAuthor API (WIP)

* fix: remove debug code

* style: remove stray whitespace

* fix: authors API for xfer (#9961)

* fix: partial implementation of rfc authors()

Does not handle RfcAuthor instances where person is None yet.

* refactor: authors -> bulk_authors for URL consistency

* fix: add OpenApi param definition doc_id (#9962)

* fix: add OpenApi param definition doc_id

* add schema def for person_id

* feat: RFC publication API (#9975)

* feat: API to publish RFC (WIP)

Incomplete and in need of refactoring, but publishes an RFC.

* feat: group / formal_languages from draft

* feat: allow optional formal_languages via API

Could do the same with group, but not clear it would ever be used.

* feat: fill in overrides/updates

* feat: subseries membership

* fix: tolerate race to create related docs

* fix: wrap pub in a transaction

* feat: prevent re-publishing draft as RFC

* chore: remove stale code

* chore: remove debug

* feat: RFC file upload API (WIP)

Checkpointing progress before going further.

* feat: specify RFC, validate file exts

* feat: move uploaded files into place

* feat: add replace option

* fix: add rest of replace option

* feat: handle ad/group more consistently

* chore: remove inadvertent change

* chore: drop external_url, get note from draft

* refactor: clarify default value logic

* refactor: ID obsoletes/updates by number

* fix: handle draft-stream-editorial

* feat: split unknown vs already published draft error (#10011)

* feat: use RfcAuthor for red API (#10014)

* refactor: combine redundant serializers

* feat: edit authors via RFC update API

* fix: remove all None authors, not just first (#10044)

* refactor: extract update_rfcauthors() method

* feat: EditedRfcAuthorsDocEvent

* refactor: reduce RfcAuthor instance churn

* feat: create RfcAuthor edit DocEvents

* feat: handle DocumentAuthor->RfcAuthor updates

* refactor: reduce code duplication

* fix: remove leftover import

* chore: relabel titlepage_name in DocEvent desc

* feat: transaction

* chore: make RfcAuthorViewset read-only

This can perhaps go away entirely

* style: undo accidental whitespace change

* fix: actual RFC file extensions/locations + tests (#10131)

* chore: update list of rfc file exts

* refactor: _destination() helper

* fix: .notprepped.xml -> prerelease/ subdir

* refactor: better prefixed DRF Routers

* test: fix references API test

* test: test notify_rfc_published

* test: test upload_rfc_files

* chore: remove unused imports

* chore: remove unused imports

* chore: add a todo

* fix: find pubreq event for editorial stream

* chore: remove obsolete rpc demo API

* chore: clean up outdated comments

* feat: api key for red api (#10232)

* fix: avoid over-return (#10231)

* fix: avoid over-return

* chore: undo accidental commit

This is a separate bug fix on main; let it come in from there.

* refactor: authors() -> author_persons() (#10237)

* refactor: authors() -> author_persons()

* refactor: select_related() a couple more places

* refactor: update uses of Document.authors()

* chore: remove debug

* fix: typo

* fix: mypy lint and minor bugs; mypy->1.11.2 (#10249)

* fix: doc property authors needs refactoring (#10250)

* fix: doc property needs refactoring

* fix: set source for author fields

* chore: comment

* chore: fix random typo

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>

* chore: remove completed todo, add comment

* chore(dev): accept devtoken for red api

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
Co-authored-by: Matthew Holloway <matthew@staff.ietf.org>
Co-authored-by: Kesara Rathnayake <kesara@fq.nz>
Co-authored-by: Rudi Matz <rudi@staff.ietf.org>
---
 dev/build/migration-start.sh                  |   8 +-
 dev/deploy-to-container/settings_local.py     |   6 +-
 docker/configs/settings_local.py              |   5 +
 ietf/api/routers.py                           |  25 +-
 ietf/api/serializers_rpc.py                   | 609 ++++++++++++++++++
 ietf/api/tests_views_rpc.py                   | 299 +++++++++
 ietf/api/urls.py                              |  19 +-
 ietf/api/urls_rpc.py                          |  42 ++
 ietf/api/views.py                             |   2 +-
 ietf/api/views_rpc.py                         | 434 +++++++++++++
 ietf/community/utils.py                       |  13 +-
 ietf/doc/admin.py                             |  10 +-
 ietf/doc/api.py                               | 194 ++++++
 ietf/doc/factories.py                         |  15 +-
 .../management/commands/reset_rfc_authors.py  |  69 --
 ietf/doc/management/commands/tests.py         |  72 ---
 ...r_dochistory_title_alter_document_title.py |  41 ++
 ietf/doc/migrations/0028_rfcauthor.py         |  84 +++
 .../0029_editedrfcauthorsdocevent.py          |  30 +
 ...r_dochistory_title_alter_document_title.py |  41 ++
 ietf/doc/models.py                            | 134 +++-
 ietf/doc/resources.py                         |  55 +-
 ietf/doc/serializers.py                       | 316 +++++++++
 ietf/doc/tests.py                             |  88 ++-
 ietf/doc/tests_draft.py                       |   8 +-
 ietf/doc/tests_review.py                      |   2 +-
 ietf/doc/utils.py                             | 175 ++++-
 ietf/doc/views_doc.py                         |  19 +-
 ietf/doc/views_search.py                      |  13 +-
 ietf/group/models.py                          |   3 +
 ietf/group/serializers.py                     |  11 +
 ietf/ietfauth/utils.py                        |   2 +-
 ietf/ipr/views.py                             |   3 +-
 ietf/name/serializers.py                      |  11 +
 ietf/nomcom/tests.py                          | 194 +++++-
 ietf/nomcom/utils.py                          | 123 +++-
 ietf/person/models.py                         |  12 +-
 ietf/secr/telechat/tests.py                   |   4 +-
 ietf/settings.py                              |   2 +
 ietf/submit/tests.py                          |   2 +-
 ietf/submit/utils.py                          |   2 +-
 ietf/sync/rfceditor.py                        |  16 +-
 ietf/sync/tests.py                            |   2 +-
 ietf/templates/doc/document_info.html         |   6 +-
 ietf/templates/doc/index_active_drafts.html   |   2 +-
 ietf/templates/doc/opengraph.html             |   4 +-
 ietf/templates/doc/review/request_info.html   |   6 +-
 .../group/manage_review_requests.html         |   6 +-
 ietf/utils/test_utils.py                      |   9 +
 ietf/utils/validators.py                      |   5 +-
 mypy.ini                                      |   3 +
 requirements.txt                              |   4 +-
 52 files changed, 2945 insertions(+), 315 deletions(-)
 create mode 100644 ietf/api/serializers_rpc.py
 create mode 100644 ietf/api/tests_views_rpc.py
 create mode 100644 ietf/api/urls_rpc.py
 create mode 100644 ietf/api/views_rpc.py
 create mode 100644 ietf/doc/api.py
 delete mode 100644 ietf/doc/management/commands/reset_rfc_authors.py
 delete mode 100644 ietf/doc/management/commands/tests.py
 create mode 100644 ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py
 create mode 100644 ietf/doc/migrations/0028_rfcauthor.py
 create mode 100644 ietf/doc/migrations/0029_editedrfcauthorsdocevent.py
 create mode 100644 ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py
 create mode 100644 ietf/doc/serializers.py
 create mode 100644 ietf/group/serializers.py
 create mode 100644 ietf/name/serializers.py

diff --git a/dev/build/migration-start.sh b/dev/build/migration-start.sh
index 901026e53b..578daf5cef 100644
--- a/dev/build/migration-start.sh
+++ b/dev/build/migration-start.sh
@@ -3,7 +3,11 @@
 echo "Running Datatracker migrations..."
 ./ietf/manage.py migrate --settings=settings_local
 
-echo "Running Blobdb migrations ..."
-./ietf/manage.py migrate --settings=settings_local --database=blobdb
+# Check whether the blobdb database exists - inspectdb will return a false
+# status if not.
+if ./ietf/manage.py inspectdb --database blobdb > /dev/null 2>&1; then
+    echo "Running Blobdb migrations ..."
+    ./ietf/manage.py migrate --settings=settings_local --database=blobdb
+fi
 
 echo "Done!"
diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py
index aacf000093..055b48d0f5 100644
--- a/dev/deploy-to-container/settings_local.py
+++ b/dev/deploy-to-container/settings_local.py
@@ -71,11 +71,11 @@
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
 
-# No real secrets here, these are public testing values _only_
 APP_API_TOKENS = {
-  "ietf.api.views.ingest_email_test": ["ingestion-test-token"]
+  "ietf.api.red_api" : ["devtoken", "redtoken"],  # Not a real secret
+  "ietf.api.views.ingest_email_test": ["ingestion-test-token"],  # Not a real secret
+  "ietf.api.views_rpc" : ["devtoken"],  # Not a real secret
 }
 
-
 # OIDC configuration
 SITE_URL = 'https://__HOSTNAME__'
diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py
index 3ee7a4295d..e357ce3f73 100644
--- a/docker/configs/settings_local.py
+++ b/docker/configs/settings_local.py
@@ -100,3 +100,8 @@
             bucket_name=f"{storagename}",
         ),
     }
+
+APP_API_TOKENS = {
+    "ietf.api.red_api" : ["devtoken", "redtoken"],  # Not a real secret
+    "ietf.api.views_rpc" : ["devtoken"],  # Not a real secret
+}
diff --git a/ietf/api/routers.py b/ietf/api/routers.py
index 745ddaa811..99afdb242a 100644
--- a/ietf/api/routers.py
+++ b/ietf/api/routers.py
@@ -3,14 +3,29 @@
 from django.core.exceptions import ImproperlyConfigured
 from rest_framework import routers
 
-class PrefixedSimpleRouter(routers.SimpleRouter):
-    """SimpleRouter that adds a dot-separated prefix to its basename"""
+
+class PrefixedBasenameMixin:
+    """Mixin to add a prefix to the basename of a rest_framework BaseRouter"""
     def __init__(self, name_prefix="", *args, **kwargs):
         self.name_prefix = name_prefix
         if len(self.name_prefix) == 0 or self.name_prefix[-1] == ".":
             raise ImproperlyConfigured("Cannot use a name_prefix that is empty or ends with '.'")
         super().__init__(*args, **kwargs)
 
-    def get_default_basename(self, viewset):
-        basename = super().get_default_basename(viewset)
-        return f"{self.name_prefix}.{basename}"
+    def register(self, prefix, viewset, basename=None):
+        # Get the superclass "register" method from the class this is mixed-in with.
+        # This avoids typing issues with calling super().register() directly in a
+        # mixin class.
+        super_register = getattr(super(), "register")
+        if not super_register or not callable(super_register):
+            raise TypeError("Must mixin with superclass that has register() method")
+        super_register(prefix, viewset, basename=f"{self.name_prefix}.{basename}")
+
+
+class PrefixedSimpleRouter(PrefixedBasenameMixin, routers.SimpleRouter):
+    """SimpleRouter that adds a dot-separated prefix to its basename"""
+
+
+class PrefixedDefaultRouter(PrefixedBasenameMixin, routers.DefaultRouter):
+    """DefaultRouter that adds a dot-separated prefix to its basename"""
+
diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py
new file mode 100644
index 0000000000..2223f04aeb
--- /dev/null
+++ b/ietf/api/serializers_rpc.py
@@ -0,0 +1,609 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+import datetime
+from pathlib import Path
+from typing import Literal, Optional
+
+from django.db import transaction
+from django.urls import reverse as urlreverse
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import extend_schema_field
+from rest_framework import serializers
+
+from ietf.doc.expire import move_draft_files_to_archive
+from ietf.doc.models import (
+    DocumentAuthor,
+    Document,
+    RelatedDocument,
+    State,
+    DocEvent,
+    RfcAuthor,
+)
+from ietf.doc.serializers import RfcAuthorSerializer
+from ietf.doc.utils import (
+    default_consensus,
+    prettify_std_name,
+    update_action_holders,
+    update_rfcauthors,
+)
+from ietf.group.models import Group
+from ietf.name.models import StreamName, StdLevelName, FormalLanguageName
+from ietf.person.models import Person
+from ietf.utils import log
+
+
+class PersonSerializer(serializers.ModelSerializer):
+    email = serializers.EmailField(read_only=True)
+    picture = serializers.URLField(source="cdn_photo_url", read_only=True)
+    url = serializers.SerializerMethodField(
+        help_text="relative URL for datatracker person page"
+    )
+
+    class Meta:
+        model = Person
+        fields = ["id", "plain_name", "email", "picture", "url"]
+        read_only_fields = ["id", "plain_name", "email", "picture", "url"]
+
+    @extend_schema_field(OpenApiTypes.URI)
+    def get_url(self, object: Person):
+        return urlreverse(
+            "ietf.person.views.profile",
+            kwargs={"email_or_name": object.email_address() or object.name},
+        )
+
+
+class EmailPersonSerializer(serializers.Serializer):
+    email = serializers.EmailField(source="address")
+    person_pk = serializers.IntegerField(source="person.pk")
+    name = serializers.CharField(source="person.name")
+    last_name = serializers.CharField(source="person.last_name")
+    initials = serializers.CharField(source="person.initials")
+
+
+class LowerCaseEmailField(serializers.EmailField):
+    def to_representation(self, value):
+        return super().to_representation(value).lower()
+
+
+class AuthorPersonSerializer(serializers.ModelSerializer):
+    person_pk = serializers.IntegerField(source="pk", read_only=True)
+    last_name = serializers.CharField()
+    initials = serializers.CharField()
+    email_addresses = serializers.ListField(
+        source="email_set.all", child=LowerCaseEmailField()
+    )
+
+    class Meta:
+        model = Person
+        fields = ["person_pk", "name", "last_name", "initials", "email_addresses"]
+
+
+class RfcWithAuthorsSerializer(serializers.ModelSerializer):
+    authors = AuthorPersonSerializer(many=True, source="author_persons")
+
+    class Meta:
+        model = Document
+        fields = ["rfc_number", "authors"]
+
+
+class DraftWithAuthorsSerializer(serializers.ModelSerializer):
+    draft_name = serializers.CharField(source="name")
+    authors = AuthorPersonSerializer(many=True, source="author_persons")
+
+    class Meta:
+        model = Document
+        fields = ["draft_name", "authors"]
+
+
+class DocumentAuthorSerializer(serializers.ModelSerializer):
+    """Serializer for a Person in a response"""
+
+    plain_name = serializers.SerializerMethodField()
+
+    class Meta:
+        model = DocumentAuthor
+        fields = ["person", "plain_name"]
+
+    def get_plain_name(self, document_author: DocumentAuthor) -> str:
+        return document_author.person.plain_name()
+
+
+class FullDraftSerializer(serializers.ModelSerializer):
+    # Redefine these fields so they don't pick up the regex validator patterns.
+    # There seem to be some non-compliant drafts in the system! If this serializer
+    # is used for a writeable view, the validation will need to be added back.
+    name = serializers.CharField(max_length=255)
+    title = serializers.CharField(max_length=255)
+
+    # Other fields we need to add / adjust
+    source_format = serializers.SerializerMethodField()
+    authors = DocumentAuthorSerializer(many=True, source="documentauthor_set")
+    shepherd = serializers.SerializerMethodField()
+    consensus = serializers.SerializerMethodField()
+
+    class Meta:
+        model = Document
+        fields = [
+            "id",
+            "name",
+            "rev",
+            "stream",
+            "title",
+            "pages",
+            "source_format",
+            "authors",
+            "shepherd",
+            "intended_std_level",
+            "consensus",
+        ]
+
+    def get_consensus(self, doc: Document) -> Optional[bool]:
+        return default_consensus(doc)
+
+    def get_source_format(
+        self, doc: Document
+    ) -> Literal["unknown", "xml-v2", "xml-v3", "txt"]:
+        submission = doc.submission()
+        if submission is None:
+            return "unknown"
+        if ".xml" in submission.file_types:
+            if submission.xml_version == "3":
+                return "xml-v3"
+            else:
+                return "xml-v2"
+        elif ".txt" in submission.file_types:
+            return "txt"
+        return "unknown"
+
+    @extend_schema_field(OpenApiTypes.EMAIL)
+    def get_shepherd(self, doc: Document) -> str:
+        if doc.shepherd:
+            return doc.shepherd.formatted_ascii_email()
+        return ""
+
+
+class DraftSerializer(FullDraftSerializer):
+    class Meta:
+        model = Document
+        fields = [
+            "id",
+            "name",
+            "rev",
+            "stream",
+            "title",
+            "pages",
+            "source_format",
+            "authors",
+        ]
+
+
+class SubmittedToQueueSerializer(FullDraftSerializer):
+    submitted = serializers.SerializerMethodField()
+    consensus = serializers.SerializerMethodField()
+
+    class Meta:
+        model = Document
+        fields = [
+            "id",
+            "name",
+            "stream",
+            "submitted",
+            "consensus",
+        ]
+
+    def get_submitted(self, doc) -> Optional[datetime.datetime]:
+        event = doc.sent_to_rfc_editor_event()
+        return None if event is None else event.time
+    
+    def get_consensus(self, doc) -> Optional[bool]:
+        return default_consensus(doc)
+
+
+class OriginalStreamSerializer(serializers.ModelSerializer):
+    stream = serializers.CharField(read_only=True, source="orig_stream_id")
+
+    class Meta:
+        model = Document
+        fields = ["rfc_number", "stream"]
+
+
+class ReferenceSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Document
+        fields = ["id", "name"]
+        read_only_fields = ["id", "name"]
+
+
+class EditableRfcSerializer(serializers.ModelSerializer):
+    # Would be nice to reconcile this with ietf.doc.serializers.RfcSerializer.
+    # The purposes of that serializer (representing data for Red) and this one
+    # (accepting updates from Purple) are different enough that separate formats
+    # may be needed, but if not it'd be nice to have a single RfcSerializer that
+    # can serve both.
+    #
+    # For now, only handles authors
+    authors = RfcAuthorSerializer(many=True, min_length=1, source="rfcauthor_set")
+
+    class Meta:
+        model = Document
+        fields = ["id", "authors"]
+
+    def update(self, instance, validated_data):
+        assert isinstance(instance, Document)
+        authors_data = validated_data.pop("rfcauthor_set", None)
+        if authors_data is not None:
+            # Construct unsaved instances from validated author data
+            new_authors = [RfcAuthor(**ad) for ad in authors_data]
+            # Update the RFC with the new author set
+            with transaction.atomic():
+                change_events = update_rfcauthors(instance, new_authors)
+                for event in change_events:
+                    event.save()
+        return instance
+
+
+class RfcPubSerializer(serializers.ModelSerializer):
+    """Write-only serializer for RFC publication"""
+    # publication-related fields
+    published = serializers.DateTimeField(default_timezone=datetime.timezone.utc)
+    draft_name = serializers.RegexField(
+        required=False, regex=r"^draft-[a-zA-Z0-9-]+$"
+    )
+    draft_rev = serializers.RegexField(
+        required=False, regex=r"^[0-9][0-9]$"
+    )
+
+    # fields on the RFC Document that need tweaking from ModelSerializer defaults
+    rfc_number = serializers.IntegerField(min_value=1, required=True)
+    group = serializers.SlugRelatedField(
+        slug_field="acronym", queryset=Group.objects.all(), required=False
+    )
+    stream = serializers.PrimaryKeyRelatedField(
+        queryset=StreamName.objects.filter(used=True)
+    )
+    formal_languages = serializers.PrimaryKeyRelatedField(
+        many=True,
+        required=False,
+        queryset=FormalLanguageName.objects.filter(used=True),
+        help_text=(
+            "formal languages used in RFC (defaults to those from draft, send empty"
+            "list to override)"
+        )
+    )
+    std_level = serializers.PrimaryKeyRelatedField(
+        queryset=StdLevelName.objects.filter(used=True),
+    )
+    ad = serializers.PrimaryKeyRelatedField(
+        queryset=Person.objects.all(),
+        allow_null=True,
+        required=False,
+    )
+    obsoletes = serializers.SlugRelatedField(
+        many=True,
+        required=False,
+        slug_field="rfc_number",
+        queryset=Document.objects.filter(type_id="rfc"),
+    )
+    updates = serializers.SlugRelatedField(
+        many=True,
+        required=False,
+        slug_field="rfc_number",
+        queryset=Document.objects.filter(type_id="rfc"),
+    )
+    subseries = serializers.ListField(
+        child=serializers.RegexField(
+            required=False,
+            # pattern: no leading 0, finite length (arbitrarily set to 5 digits)
+            regex=r"^(bcp|std|fyi)[1-9][0-9]{0,4}$", 
+        )
+    )
+    # N.b., authors is _not_ a field on Document!
+    authors = RfcAuthorSerializer(many=True)
+
+    class Meta:
+        model = Document
+        fields = [
+            "published",
+            "draft_name",
+            "draft_rev",
+            "rfc_number",
+            "title",
+            "authors",
+            "group",
+            "stream",
+            "abstract",
+            "pages",
+            "words",
+            "formal_languages",
+            "std_level",
+            "ad",
+            "note",
+            "obsoletes",
+            "updates",
+            "subseries",
+        ]
+
+    def validate(self, data):
+        if "draft_name" in data or "draft_rev" in data:
+            if "draft_name" not in data:
+                raise serializers.ValidationError(
+                    {"draft_name": "Missing draft_name"},
+                    code="invalid-draft-spec",
+                ) 
+            if "draft_rev" not in data:
+                raise serializers.ValidationError(
+                    {"draft_rev": "Missing draft_rev"},
+                    code="invalid-draft-spec",
+                )
+        return data
+
+    def create(self, validated_data):
+        """Publish an RFC"""
+        published = validated_data.pop("published")
+        draft_name = validated_data.pop("draft_name", None)
+        draft_rev = validated_data.pop("draft_rev", None)
+        obsoletes = validated_data.pop("obsoletes", [])
+        updates = validated_data.pop("updates", [])
+        subseries = validated_data.pop("subseries", [])
+
+        system_person = Person.objects.get(name="(System)")
+
+        # If specified, retrieve draft and extract RFC default values from it
+        if draft_name is None:
+            draft = None
+            defaults_from_draft = {
+                "group": Group.objects.get(acronym="none", type_id="individ"),
+            }
+        else:
+            # validation enforces that draft_name and draft_rev are both present
+            draft = Document.objects.filter(
+                type_id="draft",
+                name=draft_name,
+                rev=draft_rev,
+            ).first()
+            if draft is None:
+                raise serializers.ValidationError(
+                    {
+                        "draft_name": "No such draft",
+                        "draft_rev": "No such draft",
+                    },
+                    code="invalid-draft"
+                )
+            elif draft.get_state_slug() == "rfc":
+                raise serializers.ValidationError(
+                    {
+                        "draft_name": "Draft already published as RFC",
+                    },
+                    code="already-published-draft",
+                )
+            defaults_from_draft = {
+                "ad": draft.ad,
+                "formal_languages": draft.formal_languages.all(),
+                "group": draft.group,
+                "note": draft.note,
+            }
+
+        # Transaction to clean up if something fails
+        with transaction.atomic():
+            # create rfc, letting validated request data override draft defaults
+            rfc = self._create_rfc(defaults_from_draft | validated_data)
+            DocEvent.objects.create(
+                doc=rfc,
+                rev=rfc.rev,
+                type="published_rfc",
+                time=published,
+                by=system_person,
+                desc="RFC published",
+            )
+            rfc.set_state(State.objects.get(used=True, type_id="rfc", slug="published"))
+    
+            # create updates / obsoletes relations
+            for obsoleted_rfc_pk in obsoletes:
+                RelatedDocument.objects.get_or_create(
+                    source=rfc, target=obsoleted_rfc_pk, relationship_id="obs"
+                )
+            for updated_rfc_pk in updates:
+                RelatedDocument.objects.get_or_create(
+                    source=rfc, target=updated_rfc_pk, relationship_id="updates"
+                )
+        
+            # create subseries relations
+            for subseries_doc_name in subseries:
+                ss_slug = subseries_doc_name[:3]
+                subseries_doc, ss_doc_created = Document.objects.get_or_create(
+                    type_id=ss_slug, name=subseries_doc_name
+                )
+                if ss_doc_created:
+                    subseries_doc.docevent_set.create(
+                        type=f"{ss_slug}_doc_created",
+                        by=system_person,
+                        desc=f"Created {subseries_doc_name} via publication of {rfc.name}",
+                    )
+                _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create(
+                    relationship_id="contains", target=rfc
+                )
+                if ss_rel_created:
+                    subseries_doc.docevent_set.create(
+                        type="sync_from_rfc_editor",
+                        by=system_person,
+                        desc=f"Added {rfc.name} to {subseries_doc.name}",
+                    )
+                    rfc.docevent_set.create(
+                        type="sync_from_rfc_editor",
+                        by=system_person,
+                        desc=f"Added {rfc.name} to {subseries_doc.name}",
+                    )
+    
+    
+            # create relation with draft and update draft state
+            if draft is not None:
+                draft_changes = []
+                draft_events = []
+                if draft.get_state_slug() != "rfc":
+                    draft.set_state(
+                        State.objects.get(used=True, type="draft", slug="rfc")
+                    )
+                    move_draft_files_to_archive(draft, draft.rev)
+                    draft_changes.append(f"changed state to {draft.get_state()}")
+    
+                r, created_relateddoc = RelatedDocument.objects.get_or_create(
+                    source=draft, target=rfc, relationship_id="became_rfc",
+                )
+                if created_relateddoc:
+                    change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format(
+                        rel_name=r.relationship.name.lower(),
+                        pretty_draft_name=prettify_std_name(draft_name),
+                        pretty_rfc_name=prettify_std_name(rfc.name),
+                    )
+                    draft_changes.append(change)
+    
+                # Always set the "draft-iesg" state. This state should be set for all drafts, so
+                # log a warning if it is not set. What should happen here is that ietf stream
+                # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index.
+                # Other stream documents should normally be "idexists" and be left that way. The
+                # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub",
+                # and changes any other state to "pub". If unset, it changes it to "idexists".
+                # This reflects historical behavior and should probably be updated, but a migration
+                # of existing drafts (and validation of the change) is needed before we change the
+                # handling.
+                prev_iesg_state = draft.get_state("draft-iesg")
+                if prev_iesg_state is None:
+                    log.log(f'Warning while processing {rfc.name}: {draft.name} has no "draft-iesg" state')
+                    new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists")
+                elif prev_iesg_state.slug not in ("pub", "idexists"):
+                    if prev_iesg_state.slug != "rfcqueue":
+                        log.log(
+                            'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format(
+                                rfc.name, draft.name, prev_iesg_state.slug
+                            )
+                        )
+                    new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub")
+                else:
+                    new_iesg_state = prev_iesg_state
+        
+                if new_iesg_state != prev_iesg_state:
+                    draft.set_state(new_iesg_state)
+                    draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}")
+                    e = update_action_holders(draft, prev_iesg_state, new_iesg_state)
+                    if e:
+                        draft_events.append(e)
+    
+                # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain.
+                if draft.stream != rfc.stream:
+                    log.log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format(
+                        rfc.name, draft.name, draft.stream, rfc.stream
+                    ))
+                elif draft.stream.slug in ["iab", "irtf", "ise", "editorial"]:
+                    stream_slug = f"draft-stream-{draft.stream.slug}"
+                    prev_state = draft.get_state(stream_slug)
+                    if prev_state is not None and prev_state.slug != "pub":
+                        new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub")
+                        draft.set_state(new_state)
+                        draft_changes.append(
+                            f"changed {new_state.type.label} to {new_state}"
+                        )
+                        e = update_action_holders(draft, prev_state, new_state)
+                        if e:
+                            draft_events.append(e)
+                if draft_changes:
+                    draft_events.append(
+                        DocEvent.objects.create(
+                            doc=draft,
+                            rev=draft.rev,
+                            by=system_person,
+                            type="sync_from_rfc_editor",
+                            desc=f"Updated while publishing {rfc.name} ({', '.join(draft_changes)})",
+                        )
+                    )
+                    draft.save_with_history(draft_events)
+
+        return rfc
+
+    def _create_rfc(self, validated_data):
+        authors_data = validated_data.pop("authors")
+        formal_languages = validated_data.pop("formal_languages", [])
+        # todo ad field
+        rfc = Document.objects.create(
+            type_id="rfc",
+            name=f"rfc{validated_data['rfc_number']}",
+            **validated_data,
+        )
+        rfc.formal_languages.set(formal_languages)  # list of PKs is ok
+        for order, author_data in enumerate(authors_data):
+            rfc.rfcauthor_set.create(
+                order=order,
+                **author_data,
+            )
+        return rfc
+
+
+class RfcFileSerializer(serializers.Serializer):
+    # The structure of this serializer is constrained by what openapi-generator-cli's
+    # python generator can correctly serialize as multipart/form-data. It does not
+    # handle nested serializers well (or perhaps at all). ListFields with child
+    # ChoiceField or RegexField do not serialize correctly. DictFields don't seem
+    # to work.
+    #
+    # It does seem to correctly send filenames along with FileFields, even as a child
+    # in a ListField, so we use that to convey the file format of each item. There
+    # are other options we could consider (e.g., a structured CharField) but this
+    # works.
+    allowed_extensions = (
+        ".html",
+        ".json",
+        ".notprepped.xml",
+        ".pdf",
+        ".txt",
+        ".xml",
+    )
+
+    rfc = serializers.SlugRelatedField(
+        slug_field="rfc_number",
+        queryset=Document.objects.filter(type_id="rfc"),
+        help_text="RFC number to which the contents belong",
+    )
+    contents = serializers.ListField(
+        child=serializers.FileField(
+            allow_empty_file=False,
+            use_url=False,
+        ),
+        help_text=(
+            "List of content files. Filename extensions are used to identify "
+            "file types, but filenames are otherwise ignored."
+        ),
+    )
+    replace = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=(
+            "Replace existing files for this RFC. Defaults to false. When false, "
+            "if _any_ files already exist for the specified RFC the upload will be "
+            "rejected regardless of which files are being uploaded. When true,"
+            "existing files will be removed and new ones will be put in place. BE"
+            "VERY CAREFUL WITH THIS OPTION IN PRODUCTION."
+        ),
+    )
+
+    def validate_contents(self, data):
+        found_extensions = []
+        for uploaded_file in data:
+            if not hasattr(uploaded_file, "name"):
+                raise serializers.ValidationError(
+                    "filename not specified for uploaded file",
+                    code="missing-filename",
+                )
+            ext = "".join(Path(uploaded_file.name).suffixes)
+            if ext not in self.allowed_extensions:
+                raise serializers.ValidationError(
+                    f"File uploaded with invalid extension '{ext}'",
+                    code="invalid-filename-ext",
+                )
+            if ext in found_extensions:
+                raise serializers.ValidationError(
+                    f"More than one file uploaded with extension '{ext}'",
+                    code="duplicate-filename-ext",
+                )
+        return data
+
+
+class NotificationAckSerializer(serializers.Serializer):
+    message = serializers.CharField(default="ack")
diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py
new file mode 100644
index 0000000000..032b4b9495
--- /dev/null
+++ b/ietf/api/tests_views_rpc.py
@@ -0,0 +1,299 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+from io import StringIO
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from django.conf import settings
+from django.core.files.base import ContentFile
+from django.db.models import Max
+from django.test.utils import override_settings
+from django.urls import reverse as urlreverse
+
+from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory
+from ietf.doc.models import RelatedDocument, Document
+from ietf.group.factories import RoleFactory, GroupFactory
+from ietf.person.factories import PersonFactory
+from ietf.utils.test_utils import APITestCase, reload_db_objects
+
+
+class RpcApiTests(APITestCase):
+    @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
+    def test_draftviewset_references(self):
+        viewname = "ietf.api.purple_api.draft-references"
+
+        # non-existent draft
+        bad_id = Document.objects.aggregate(unused_id=Max("id") + 100)["unused_id"]
+        url = urlreverse(viewname, kwargs={"doc_id": bad_id})
+        # Without credentials
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 403)
+        # Add credentials
+        r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 404)
+
+        # draft without any normative references
+        draft = IndividualDraftFactory()
+        draft = reload_db_objects(draft)
+        url = urlreverse(viewname, kwargs={"doc_id": draft.id})
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 403)
+        r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 200)
+        refs = r.json()
+        self.assertEqual(refs, [])
+
+        # draft without any normative references but with an informative reference
+        draft_foo = IndividualDraftFactory()
+        draft_foo = reload_db_objects(draft_foo)
+        RelatedDocument.objects.create(
+            source=draft, target=draft_foo, relationship_id="refinfo"
+        )
+        url = urlreverse(viewname, kwargs={"doc_id": draft.id})
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 403)
+        r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 200)
+        refs = r.json()
+        self.assertEqual(refs, [])
+
+        # draft with a normative reference
+        draft_bar = IndividualDraftFactory()
+        draft_bar = reload_db_objects(draft_bar)
+        RelatedDocument.objects.create(
+            source=draft, target=draft_bar, relationship_id="refnorm"
+        )
+        url = urlreverse(viewname, kwargs={"doc_id": draft.id})
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 403)
+        r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(r.status_code, 200)
+        refs = r.json()
+        self.assertEqual(len(refs), 1)
+        self.assertEqual(refs[0]["id"], draft_bar.id)
+        self.assertEqual(refs[0]["name"], draft_bar.name)
+
+    @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
+    def test_notify_rfc_published(self):
+        url = urlreverse("ietf.api.purple_api.notify_rfc_published")
+        area = GroupFactory(type_id="area")
+        draft_ad = RoleFactory(group=area, name_id="ad").person
+        authors = PersonFactory.create_batch(2)
+        draft = WgDraftFactory(group__parent=area, authors=authors)
+        assert isinstance(draft, Document), "WgDraftFactory should generate a Document"
+        unused_rfc_number = (
+            Document.objects.filter(rfc_number__isnull=False).aggregate(
+                unused_rfc_number=Max("rfc_number") + 1
+            )["unused_rfc_number"]
+            or 10000
+        )
+
+        post_data = {
+            "published": "2025-12-17T20:29:00Z",
+            "draft_name": draft.name,
+            "draft_rev": draft.rev,
+            "rfc_number": unused_rfc_number,
+            "title": draft.title,
+            "authors": [
+                {
+                    "titlepage_name": f"titlepage {author.name}",
+                    "is_editor": False,
+                    "person": author.pk,
+                    "email": author.email_address(),
+                    "affiliation": "Some Affiliation",
+                    "country": "CA",
+                }
+                for author in authors
+            ],
+            "group": draft.group.acronym,
+            "stream": draft.stream_id,
+            "abstract": draft.abstract,
+            "pages": draft.pages,
+            "words": draft.pages * 250,
+            "formal_languages": [],
+            "std_level": "ps",
+            "ad": draft_ad.pk,
+            "note": "noted",
+            "obsoletes": [],
+            "updates": [],
+            "subseries": [],
+        }
+        r = self.client.post(url, data=post_data, format="json")
+        self.assertEqual(r.status_code, 403)
+
+        r = self.client.post(
+            url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"}
+        )
+        self.assertEqual(r.status_code, 200)
+        rfc = Document.objects.filter(rfc_number=unused_rfc_number).first()
+        self.assertIsNotNone(rfc)
+        self.assertEqual(rfc.came_from_draft(), draft)
+        self.assertEqual(
+            rfc.docevent_set.filter(
+                type="published_rfc", time="2025-12-17T20:29:00Z"
+            ).count(),
+            1,
+        )
+        self.assertEqual(rfc.title, draft.title)
+        self.assertEqual(rfc.documentauthor_set.count(), 0)
+        self.assertEqual(
+            list(
+                rfc.rfcauthor_set.values(
+                    "titlepage_name",
+                    "is_editor",
+                    "person",
+                    "email",
+                    "affiliation",
+                    "country",
+                )
+            ),
+            [
+                {
+                    "titlepage_name": f"titlepage {author.name}",
+                    "is_editor": False,
+                    "person": author.pk,
+                    "email": author.email_address(),
+                    "affiliation": "Some Affiliation",
+                    "country": "CA",
+                }
+                for author in authors
+            ],
+        )
+        self.assertEqual(rfc.group, draft.group)
+        self.assertEqual(rfc.stream, draft.stream)
+        self.assertEqual(rfc.abstract, draft.abstract)
+        self.assertEqual(rfc.pages, draft.pages)
+        self.assertEqual(rfc.words, draft.pages * 250)
+        self.assertEqual(rfc.formal_languages.count(), 0)
+        self.assertEqual(rfc.std_level_id, "ps")
+        self.assertEqual(rfc.ad, draft_ad)
+        self.assertEqual(rfc.note, "noted")
+        self.assertEqual(rfc.related_that_doc("obs"), [])
+        self.assertEqual(rfc.related_that_doc("updates"), [])
+        self.assertEqual(rfc.part_of(), [])
+        self.assertEqual(draft.get_state().slug, "rfc")
+        # todo test non-empty relationships
+        # todo test references (when updating that is part of the handling)
+
+    @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
+    def test_upload_rfc_files(self):
+        def _valid_post_data():
+            """Generate a valid post data dict
+
+            Each API call needs a fresh set of files, so don't reuse the return
+            value from this for multiple calls!
+            """
+            return {
+                "rfc": rfc.rfc_number,
+                "contents": [
+                    ContentFile(b"This is .xml", "myfile.xml"),
+                    ContentFile(b"This is .txt", "myfile.txt"),
+                    ContentFile(b"This is .html", "myfile.html"),
+                    ContentFile(b"This is .pdf", "myfile.pdf"),
+                    ContentFile(b"This is .json", "myfile.json"),
+                    ContentFile(b"This is .notprepped.xml", "myfile.notprepped.xml"),
+                ],
+                "replace": False,
+            }
+
+        url = urlreverse("ietf.api.purple_api.upload_rfc_files")
+        unused_rfc_number = (
+            Document.objects.filter(rfc_number__isnull=False).aggregate(
+                unused_rfc_number=Max("rfc_number") + 1
+            )["unused_rfc_number"]
+            or 10000
+        )
+
+        rfc = WgRfcFactory(rfc_number=unused_rfc_number)
+        assert isinstance(rfc, Document), "WgRfcFactory should generate a Document"
+        with TemporaryDirectory() as rfc_dir:
+            settings.RFC_PATH = rfc_dir  # affects overridden settings
+            rfc_path = Path(rfc_dir)
+            (rfc_path / "prerelease").mkdir()
+            content = StringIO("XML content\n")
+            content.name = "myrfc.xml"
+
+            # no api key
+            r = self.client.post(url, _valid_post_data(), format="multipart")
+            self.assertEqual(r.status_code, 403)
+
+            # invalid RFC
+            r = self.client.post(
+                url,
+                _valid_post_data() | {"rfc": unused_rfc_number + 1},
+                format="multipart",
+                headers={"X-Api-Key": "valid-token"},
+            )
+            self.assertEqual(r.status_code, 400)
+
+            # empty files
+            r = self.client.post(
+                url,
+                _valid_post_data() | {
+                    "contents": [
+                        ContentFile(b"", "myfile.xml"),
+                        ContentFile(b"", "myfile.txt"),
+                        ContentFile(b"", "myfile.html"),
+                        ContentFile(b"", "myfile.pdf"),
+                        ContentFile(b"", "myfile.json"),
+                        ContentFile(b"", "myfile.notprepped.xml"),
+                    ]
+                },
+                format="multipart",
+                headers={"X-Api-Key": "valid-token"},
+            )
+            self.assertEqual(r.status_code, 400)
+
+            # bad file type
+            r = self.client.post(
+                url,
+                _valid_post_data() | {
+                    "contents": [
+                        ContentFile(b"Some content", "myfile.jpg"),
+                    ]
+                },
+                format="multipart",
+                headers={"X-Api-Key": "valid-token"},
+            )
+            self.assertEqual(r.status_code, 400)
+
+            # valid post
+            r = self.client.post(
+                url,
+                _valid_post_data(),
+                format="multipart",
+                headers={"X-Api-Key": "valid-token"},
+            )
+            self.assertEqual(r.status_code, 200)
+            for suffix in [".xml", ".txt", ".html", ".pdf", ".json"]:
+                self.assertEqual(
+                    (rfc_path / f"rfc{unused_rfc_number}")
+                    .with_suffix(suffix)
+                    .read_text(),
+                    f"This is {suffix}",
+                    f"{suffix} file should contain the expected content",
+                )
+            self.assertEqual(
+                (
+                    rfc_path / "prerelease" / f"rfc{unused_rfc_number}.notprepped.xml"
+                ).read_text(),
+                "This is .notprepped.xml",
+                ".notprepped.xml file should contain the expected content",
+            )
+
+            # re-post with replace = False should now fail
+            r = self.client.post(
+                url,
+                _valid_post_data(),
+                format="multipart",
+                headers={"X-Api-Key": "valid-token"},
+            )
+            self.assertEqual(r.status_code, 409)  # conflict
+            
+            # re-post with replace = True should succeed
+            r = self.client.post(
+                url,
+                _valid_post_data() | {"replace": True},
+                format="multipart",
+                headers={"X-Api-Key": "valid-token"},
+            )
+            self.assertEqual(r.status_code, 200)  # conflict
diff --git a/ietf/api/urls.py b/ietf/api/urls.py
index 04575b34cb..7a082567b8 100644
--- a/ietf/api/urls.py
+++ b/ietf/api/urls.py
@@ -1,26 +1,31 @@
 # Copyright The IETF Trust 2017-2024, All Rights Reserved
 
+from drf_spectacular.views import SpectacularAPIView
+
 from django.conf import settings
-from django.urls import include
+from django.urls import include, path
 from django.views.generic import TemplateView
 
 from ietf import api
-from ietf.doc import views_ballot
+from ietf.doc import views_ballot, api as doc_api
 from ietf.meeting import views as meeting_views
 from ietf.submit import views as submit_views
 from ietf.utils.urls import url
 
 from . import views as api_views
+from .routers import PrefixedSimpleRouter 
 
 # DRF API routing - disabled until we plan to use it
-# from drf_spectacular.views import SpectacularAPIView
-# from django.urls import path
 # from ietf.person import api as person_api
-# from .routers import PrefixedSimpleRouter 
 # core_router = PrefixedSimpleRouter(name_prefix="ietf.api.core_api")  # core api router
 # core_router.register("email", person_api.EmailViewSet)
 # core_router.register("person", person_api.PersonViewSet)
 
+# todo more general name for this API?
+red_router = PrefixedSimpleRouter(name_prefix="ietf.api.red_api")  # red api router
+red_router.register("doc", doc_api.RfcViewSet)
+red_router.register("subseries", doc_api.SubseriesViewSet, basename="subseries")
+
 api.autodiscover()
 
 urlpatterns = [
@@ -32,7 +37,9 @@
     url(r'^v2/person/person', api_views.ApiV2PersonExportView.as_view()),
     # --- DRF API ---
     # path("core/", include(core_router.urls)),
-    # path("schema/", SpectacularAPIView.as_view()),
+    path("purple/", include("ietf.api.urls_rpc")),
+    path("red/", include(red_router.urls)),
+    path("schema/", SpectacularAPIView.as_view()),
     #
     # --- Custom API endpoints, sorted alphabetically ---
     # Email alias information for drafts
diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py
new file mode 100644
index 0000000000..9d41ac137f
--- /dev/null
+++ b/ietf/api/urls_rpc.py
@@ -0,0 +1,42 @@
+# Copyright The IETF Trust 2023-2026, All Rights Reserved
+from django.urls import include, path
+
+from ietf.api import views_rpc
+from ietf.api.routers import PrefixedDefaultRouter
+from ietf.utils.urls import url
+
+router = PrefixedDefaultRouter(use_regex_path=False, name_prefix="ietf.api.purple_api")
+router.include_format_suffixes = False
+router.register(r"draft", views_rpc.DraftViewSet, basename="draft")
+router.register(r"person", views_rpc.PersonViewSet)
+router.register(r"rfc", views_rpc.RfcViewSet, basename="rfc")
+
+router.register(
+    r"rfc/<int:rfc_number>/authors",
+    views_rpc.RfcAuthorViewSet,
+    basename="rfc-authors",
+)
+
+urlpatterns = [
+    url(r"^doc/drafts_by_names/", views_rpc.DraftsByNamesView.as_view()),
+    url(r"^persons/search/", views_rpc.RpcPersonSearch.as_view()),
+    path(
+        r"rfc/publish/",
+        views_rpc.RfcPubNotificationView.as_view(),
+        name="ietf.api.purple_api.notify_rfc_published",
+    ),
+    path(
+        r"rfc/publish/files/",
+        views_rpc.RfcPubFilesView.as_view(),
+        name="ietf.api.purple_api.upload_rfc_files",
+    ),
+    path(r"subject/<str:subject_id>/person/", views_rpc.SubjectPersonView.as_view()),
+]
+
+# add routers at the end so individual routes can steal parts of their address
+# space (e.g., ^rfc/publish/ superseding the ^rfc/ routes of RfcViewSet)
+urlpatterns.extend(
+    [
+        path("", include(router.urls)),
+    ]
+)
diff --git a/ietf/api/views.py b/ietf/api/views.py
index 22523b2f17..420bc39693 100644
--- a/ietf/api/views.py
+++ b/ietf/api/views.py
@@ -97,7 +97,7 @@ class PersonalInformationExportView(DetailView, JsonExportMixin):
 
     def get(self, request):
         person = get_object_or_404(self.model, user=request.user)
-        expand = ['searchrule', 'documentauthor', 'ad_document_set', 'ad_dochistory_set', 'docevent',
+        expand = ['searchrule', 'documentauthor', 'rfcauthor', 'ad_document_set', 'ad_dochistory_set', 'docevent',
             'ballotpositiondocevent', 'deletedevent', 'email_set', 'groupevent', 'role', 'rolehistory', 'iprdisclosurebase',
             'iprevent', 'liaisonstatementevent', 'allowlisted', 'schedule', 'constraint', 'schedulingevent', 'message',
             'sendqueue', 'nominee', 'topicfeedbacklastseen', 'alias', 'email', 'apikeys', 'personevent',
diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
new file mode 100644
index 0000000000..fce174ab72
--- /dev/null
+++ b/ietf/api/views_rpc.py
@@ -0,0 +1,434 @@
+# Copyright The IETF Trust 2023-2026, All Rights Reserved
+import shutil
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from django.conf import settings
+from drf_spectacular.utils import OpenApiParameter
+from rest_framework import mixins, parsers, serializers, viewsets, status
+from rest_framework.decorators import action
+from rest_framework.exceptions import APIException
+from rest_framework.views import APIView
+from rest_framework.response import Response
+
+from django.db.models import CharField as ModelCharField, OuterRef, Subquery, Q
+from django.db.models.functions import Coalesce
+from django.http import Http404
+from drf_spectacular.utils import extend_schema_view, extend_schema
+from rest_framework import generics
+from rest_framework.fields import CharField as DrfCharField
+from rest_framework.filters import SearchFilter
+from rest_framework.pagination import LimitOffsetPagination
+
+from ietf.api.serializers_rpc import (
+    PersonSerializer,
+    FullDraftSerializer,
+    DraftSerializer,
+    SubmittedToQueueSerializer,
+    OriginalStreamSerializer,
+    ReferenceSerializer,
+    EmailPersonSerializer,
+    RfcWithAuthorsSerializer,
+    DraftWithAuthorsSerializer,
+    NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer,
+    EditableRfcSerializer,
+)
+from ietf.doc.models import Document, DocHistory, RfcAuthor
+from ietf.doc.serializers import RfcAuthorSerializer
+from ietf.person.models import Email, Person
+
+
+class Conflict(APIException):
+    status_code = status.HTTP_409_CONFLICT
+    default_detail = "Conflict."
+    default_code = "conflict"
+
+
+@extend_schema_view(
+    retrieve=extend_schema(
+        operation_id="get_person_by_id",
+        summary="Find person by ID",
+        description="Returns a single person",
+        parameters=[
+            OpenApiParameter(
+                name="person_id",
+                type=int,
+                location="path",
+                description="Person ID identifying this person.",
+            ),
+        ],
+    ),
+)
+class PersonViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+    queryset = Person.objects.all()
+    serializer_class = PersonSerializer
+    api_key_endpoint = "ietf.api.views_rpc"
+    lookup_url_kwarg = "person_id"
+
+    @extend_schema(
+        operation_id="get_persons",
+        summary="Get a batch of persons",
+        description="Returns a list of persons matching requested ids. Omits any that are missing.",
+        request=list[int],
+        responses=PersonSerializer(many=True),
+    )
+    @action(detail=False, methods=["post"])
+    def batch(self, request):
+        """Get a batch of rpc person names"""
+        pks = request.data
+        return Response(
+            self.get_serializer(Person.objects.filter(pk__in=pks), many=True).data
+        )
+
+    @extend_schema(
+        operation_id="persons_by_email",
+        summary="Get a batch of persons by email addresses",
+        description=(
+            "Returns a list of persons matching requested ids. "
+            "Omits any that are missing."
+        ),
+        request=list[str],
+        responses=EmailPersonSerializer(many=True),
+    )
+    @action(detail=False, methods=["post"], serializer_class=EmailPersonSerializer)
+    def batch_by_email(self, request):
+        emails = Email.objects.filter(address__in=request.data, person__isnull=False)
+        serializer = self.get_serializer(emails, many=True)
+        return Response(serializer.data)
+
+
+class SubjectPersonView(APIView):
+    api_key_endpoint = "ietf.api.views_rpc"
+
+    @extend_schema(
+        operation_id="get_subject_person_by_id",
+        summary="Find person for OIDC subject by ID",
+        description="Returns a single person",
+        responses=PersonSerializer,
+        parameters=[
+            OpenApiParameter(
+                name="subject_id",
+                type=str,
+                description="subject ID of person to return",
+                location="path",
+            ),
+        ],
+    )
+    def get(self, request, subject_id: str):
+        try:
+            user_id = int(subject_id)
+        except ValueError:
+            raise serializers.ValidationError(
+                {"subject_id": "This field must be an integer value."}
+            )
+        person = Person.objects.filter(user__pk=user_id).first()
+        if person:
+            return Response(PersonSerializer(person).data)
+        raise Http404
+
+
+class RpcLimitOffsetPagination(LimitOffsetPagination):
+    default_limit = 10
+    max_limit = 100
+
+
+class SingleTermSearchFilter(SearchFilter):
+    """SearchFilter backend that does not split terms
+
+    The default SearchFilter treats comma or whitespace-separated terms as individual
+    search terms. This backend instead searches for the exact term.
+    """
+
+    def get_search_terms(self, request):
+        value = request.query_params.get(self.search_param, "")
+        field = DrfCharField(trim_whitespace=False, allow_blank=True)
+        cleaned_value = field.run_validation(value)
+        return [cleaned_value]
+
+
+@extend_schema_view(
+    get=extend_schema(
+        operation_id="search_person",
+        description="Get a list of persons, matching by partial name or email",
+    ),
+)
+class RpcPersonSearch(generics.ListAPIView):
+    # n.b. the OpenAPI schema for this can be generated by running
+    # ietf/manage.py spectacular --file spectacular.yaml
+    # and extracting / touching up the rpc_person_search_list operation
+    api_key_endpoint = "ietf.api.views_rpc"
+    queryset = Person.objects.all()
+    serializer_class = PersonSerializer
+    pagination_class = RpcLimitOffsetPagination
+
+    # Searchable on all name-like fields or email addresses
+    filter_backends = [SingleTermSearchFilter]
+    search_fields = ["name", "plain", "email__address"]
+
+
+@extend_schema_view(
+    retrieve=extend_schema(
+        operation_id="get_draft_by_id",
+        summary="Get a draft",
+        description="Returns the draft for the requested ID",
+        parameters=[
+            OpenApiParameter(
+                name="doc_id",
+                type=int,
+                location="path",
+                description="Doc ID identifying this draft.",
+            ),
+        ],
+    ),
+    submitted_to_rpc=extend_schema(
+        operation_id="submitted_to_rpc",
+        summary="List documents ready to enter the RFC Editor Queue",
+        description="List documents ready to enter the RFC Editor Queue",
+        responses=SubmittedToQueueSerializer(many=True),
+    ),
+)
+class DraftViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet):
+    queryset = Document.objects.filter(type_id="draft")
+    serializer_class = FullDraftSerializer
+    api_key_endpoint = "ietf.api.views_rpc"
+    lookup_url_kwarg = "doc_id"
+
+    @action(detail=False, serializer_class=SubmittedToQueueSerializer)
+    def submitted_to_rpc(self, request):
+        """Return documents in datatracker that have been submitted to the RPC but are not yet in the queue
+
+        Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue.
+        """
+        ietf_docs = Q(states__type_id="draft-iesg", states__slug__in=["ann"])
+        irtf_iab_ise_docs = Q(
+            states__type_id__in=[
+                "draft-stream-iab",
+                "draft-stream-irtf",
+                "draft-stream-ise",
+            ],
+            states__slug__in=["rfc-edit"],
+        )
+        # TODO: Need a way to talk about editorial stream docs
+        docs = (
+            self.get_queryset()
+            .filter(type_id="draft")
+            .filter(ietf_docs | irtf_iab_ise_docs)
+        )
+        serializer = self.get_serializer(docs, many=True)
+        return Response(serializer.data)
+
+    @extend_schema(
+        operation_id="get_draft_references",
+        summary="Get normative references to I-Ds",
+        description=(
+            "Returns the id and name of each normatively "
+            "referenced Internet-Draft for the given docId"
+        ),
+        parameters=[
+            OpenApiParameter(
+                name="doc_id",
+                type=int,
+                location="path",
+                description="Doc ID identifying this draft.",
+            ),
+        ],
+        responses=ReferenceSerializer(many=True),
+    )
+    @action(detail=True, serializer_class=ReferenceSerializer)
+    def references(self, request, doc_id=None):
+        doc = self.get_object()
+        serializer = self.get_serializer(
+            [
+                reference
+                for reference in doc.related_that_doc("refnorm")
+                if reference.type_id == "draft"
+            ],
+            many=True,
+        )
+        return Response(serializer.data)
+
+    @extend_schema(
+        operation_id="get_draft_authors",
+        summary="Gather authors of the drafts with the given names",
+        description="returns a list mapping draft names to objects describing authors",
+        request=list[str],
+        responses=DraftWithAuthorsSerializer(many=True),
+    )
+    @action(detail=False, methods=["post"], serializer_class=DraftWithAuthorsSerializer)
+    def bulk_authors(self, request):
+        drafts = self.get_queryset().filter(name__in=request.data)
+        serializer = self.get_serializer(drafts, many=True)
+        return Response(serializer.data)
+
+
+@extend_schema_view(
+    rfc_original_stream=extend_schema(
+        operation_id="get_rfc_original_streams",
+        summary="Get the streams RFCs were originally published into",
+        description="returns a list of dicts associating an RFC with its originally published stream",
+        responses=OriginalStreamSerializer(many=True),
+    )
+)
+class RfcViewSet(mixins.UpdateModelMixin, viewsets.GenericViewSet):
+    queryset = Document.objects.filter(type_id="rfc")
+    api_key_endpoint = "ietf.api.views_rpc"
+    lookup_field = "rfc_number"
+    serializer_class = EditableRfcSerializer
+
+    @action(detail=False, serializer_class=OriginalStreamSerializer)
+    def rfc_original_stream(self, request):
+        rfcs = self.get_queryset().annotate(
+            orig_stream_id=Coalesce(
+                Subquery(
+                    DocHistory.objects.filter(doc=OuterRef("pk"))
+                    .exclude(stream__isnull=True)
+                    .order_by("time")
+                    .values_list("stream_id", flat=True)[:1]
+                ),
+                "stream_id",
+                output_field=ModelCharField(),
+            ),
+        )
+        serializer = self.get_serializer(rfcs, many=True)
+        return Response(serializer.data)
+
+    @extend_schema(
+        operation_id="get_rfc_authors",
+        summary="Gather authors of the RFCs with the given numbers",
+        description="returns a list mapping rfc numbers to objects describing authors",
+        request=list[int],
+        responses=RfcWithAuthorsSerializer(many=True),
+    )
+    @action(detail=False, methods=["post"], serializer_class=RfcWithAuthorsSerializer)
+    def bulk_authors(self, request):
+        rfcs = self.get_queryset().filter(rfc_number__in=request.data)
+        serializer = self.get_serializer(rfcs, many=True)
+        return Response(serializer.data)
+
+
+class DraftsByNamesView(APIView):
+    api_key_endpoint = "ietf.api.views_rpc"
+
+    @extend_schema(
+        operation_id="get_drafts_by_names",
+        summary="Get a batch of drafts by draft names",
+        description="returns a list of drafts with matching names",
+        request=list[str],
+        responses=DraftSerializer(many=True),
+    )
+    def post(self, request):
+        names = request.data
+        docs = Document.objects.filter(type_id="draft", name__in=names)
+        return Response(DraftSerializer(docs, many=True).data)
+
+
+class RfcAuthorViewSet(viewsets.ReadOnlyModelViewSet):
+    """ViewSet for RfcAuthor model
+    
+    Router needs to provide rfc_number as a kwarg
+    """
+    api_key_endpoint = "ietf.api.views_rpc"
+
+    queryset = RfcAuthor.objects.all()
+    serializer_class = RfcAuthorSerializer
+    lookup_url_kwarg = "author_id"
+    rfc_number_param = "rfc_number"
+
+    def get_queryset(self):
+        return (
+            super()
+            .get_queryset()
+            .filter(
+                document__type_id="rfc",
+                document__rfc_number=self.kwargs[self.rfc_number_param],
+            )
+        )
+
+
+class RfcPubNotificationView(APIView):
+    api_key_endpoint = "ietf.api.views_rpc"
+
+    @extend_schema(
+        operation_id="notify_rfc_published",
+        summary="Notify datatracker of RFC publication",
+        request=RfcPubSerializer,
+        responses=NotificationAckSerializer,
+    )
+    def post(self, request):
+        serializer = RfcPubSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        # Create RFC
+        serializer.save()
+        return Response(NotificationAckSerializer().data)
+
+
+class RfcPubFilesView(APIView):
+    api_key_endpoint = "ietf.api.views_rpc"
+    parser_classes = [parsers.MultiPartParser]
+
+    def _destination(self, filename: str | Path) -> Path:
+        """Destination for an uploaded RFC file
+        
+        Strips any path components in filename and returns an absolute Path.
+        """
+        rfc_path = Path(settings.RFC_PATH)
+        filename = Path(filename)  # could potentially have directory components
+        extension = "".join(filename.suffixes)
+        if extension == ".notprepped.xml":
+            return rfc_path / "prerelease" / filename.name
+        return rfc_path / filename.name
+
+    @extend_schema(
+        operation_id="upload_rfc_files",
+        summary="Upload files for a published RFC",
+        request=RfcFileSerializer,
+        responses=NotificationAckSerializer,
+    )
+    def post(self, request):
+        serializer = RfcFileSerializer(
+            # many=True,
+            data=request.data,
+        )
+        serializer.is_valid(raise_exception=True)
+        rfc = serializer.validated_data["rfc"]
+        uploaded_files = serializer.validated_data["contents"]  # list[UploadedFile]
+        replace = serializer.validated_data["replace"]
+        dest_stem = f"rfc{rfc.rfc_number}"
+
+        # List of files that might exist for an RFC
+        possible_rfc_files = [
+            self._destination(dest_stem + ext)
+            for ext in serializer.allowed_extensions
+        ]
+        if not replace:
+            # this is the default: refuse to overwrite anything if not replacing
+            for possible_existing_file in possible_rfc_files:
+                if possible_existing_file.exists():
+                    raise Conflict(
+                        "File(s) already exist for this RFC",
+                        code="files-exist",
+                    )
+
+        with TemporaryDirectory() as tempdir:
+            # Save files in a temporary directory. Use the uploaded filename
+            # extensions to identify files, but ignore the stems and generate our own.
+            files_to_move = []  # list[Path]
+            tmpfile_stem = Path(tempdir) / dest_stem
+            for upfile in uploaded_files:
+                uploaded_filename = Path(upfile.name)  # name supplied by request
+                uploaded_ext = "".join(uploaded_filename.suffixes)
+                tempfile_path = tmpfile_stem.with_suffix(uploaded_ext)
+                with tempfile_path.open("wb") as dest:
+                    for chunk in upfile.chunks():
+                        dest.write(chunk)
+                files_to_move.append(tempfile_path)
+            # copy files to final location, removing any existing ones first if the
+            # remove flag was set
+            if replace:
+                for possible_existing_file in possible_rfc_files:
+                    possible_existing_file.unlink(missing_ok=True)
+            for ftm in files_to_move:
+                shutil.move(ftm, self._destination(ftm))
+                # todo store in blob storage as well (need a bucket for RFCs)
+
+        return Response(NotificationAckSerializer().data)
diff --git a/ietf/community/utils.py b/ietf/community/utils.py
index f23e8d26ab..b6137095ef 100644
--- a/ietf/community/utils.py
+++ b/ietf/community/utils.py
@@ -72,8 +72,10 @@ def docs_matching_community_list_rule(rule):
         return docs.filter(group=rule.group_id)
     elif rule.rule_type.startswith("state_"):
         return docs
-    elif rule.rule_type in ["author", "author_rfc"]:
+    elif rule.rule_type == "author": 
         return docs.filter(documentauthor__person=rule.person)
+    elif rule.rule_type == "author_rfc":
+        return docs.filter(Q(rfcauthor__person=rule.person)|Q(rfcauthor__isnull=True,documentauthor__person=rule.person))
     elif rule.rule_type == "ad":
         return docs.filter(ad=rule.person)
     elif rule.rule_type == "shepherd":
@@ -122,9 +124,16 @@ def community_list_rules_matching_doc(doc):
 
     # author rules
     if doc.type_id == "rfc":
+        has_rfcauthors = doc.rfcauthor_set.exists()
         rules |= SearchRule.objects.filter(
             rule_type="author_rfc",
-            person__in=list(Person.objects.filter(documentauthor__document=doc)),
+            person__in=list(
+                Person.objects.filter(
+                    Q(rfcauthor__document=doc)
+                    if has_rfcauthors
+                    else Q(documentauthor__document=doc)
+                )
+            ),
         )
     else:
         rules |= SearchRule.objects.filter(
diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py
index 8f26b222e1..f082418935 100644
--- a/ietf/doc/admin.py
+++ b/ietf/doc/admin.py
@@ -13,7 +13,8 @@
     TelechatDocEvent, BallotPositionDocEvent, ReviewRequestDocEvent, InitialReviewDocEvent,
     AddedMessageEvent, SubmissionDocEvent, DeletedEvent, EditedAuthorsDocEvent, DocumentURL,
     ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder,
-    BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject )
+    BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject, RfcAuthor,
+    EditedRfcAuthorsDocEvent)
 
 from ietf.utils.admin import SaferTabularInline
 from ietf.utils.validators import validate_external_resource_value
@@ -174,6 +175,7 @@ def short_desc(self, obj):
 admin.site.register(TelechatDocEvent, DocEventAdmin)
 admin.site.register(InitialReviewDocEvent, DocEventAdmin)
 admin.site.register(EditedAuthorsDocEvent, DocEventAdmin)
+admin.site.register(EditedRfcAuthorsDocEvent, DocEventAdmin)
 admin.site.register(IanaExpertDocEvent, DocEventAdmin)
 
 class BallotPositionDocEventAdmin(DocEventAdmin):
@@ -237,3 +239,9 @@ def is_deleted(self, instance):
     
 
 admin.site.register(StoredObject, StoredObjectAdmin)
+
+class RfcAuthorAdmin(admin.ModelAdmin):
+    list_display = ['id', 'document', 'titlepage_name', 'person', 'email', 'affiliation', 'country', 'order']
+    search_fields = ['document__name', 'titlepage_name', 'person__name', 'email__address', 'affiliation', 'country']
+    raw_id_fields = ["document", "person", "email"]
+admin.site.register(RfcAuthor, RfcAuthorAdmin)
diff --git a/ietf/doc/api.py b/ietf/doc/api.py
new file mode 100644
index 0000000000..47e7e6fffd
--- /dev/null
+++ b/ietf/doc/api.py
@@ -0,0 +1,194 @@
+# Copyright The IETF Trust 2024-2026, All Rights Reserved
+"""Doc API implementations"""
+
+from django.db.models import OuterRef, Subquery, Prefetch, Value, JSONField, QuerySet
+from django.db.models.functions import TruncDate
+from django_filters import rest_framework as filters
+from rest_framework import filters as drf_filters
+from rest_framework.mixins import ListModelMixin, RetrieveModelMixin
+from rest_framework.pagination import LimitOffsetPagination
+from rest_framework.viewsets import GenericViewSet
+
+from ietf.group.models import Group
+from ietf.name.models import StreamName, DocTypeName
+from ietf.utils.timezone import RPC_TZINFO
+from .models import (
+    Document,
+    DocEvent,
+    RelatedDocument,
+    DocumentAuthor,
+    SUBSERIES_DOC_TYPE_IDS,
+)
+from .serializers import (
+    RfcMetadataSerializer,
+    RfcStatus,
+    RfcSerializer,
+    SubseriesDocSerializer,
+)
+
+
+class RfcLimitOffsetPagination(LimitOffsetPagination):
+    default_limit = 10
+    max_limit = 500
+
+
+class RfcFilter(filters.FilterSet):
+    published = filters.DateFromToRangeFilter()
+    stream = filters.ModelMultipleChoiceFilter(
+        queryset=StreamName.objects.filter(used=True)
+    )
+    group = filters.ModelMultipleChoiceFilter(
+        queryset=Group.objects.wgs(),
+        field_name="group__acronym",
+        to_field_name="acronym",
+    )
+    area = filters.ModelMultipleChoiceFilter(
+        queryset=Group.objects.areas(),
+        field_name="group__parent__acronym",
+        to_field_name="acronym",
+    )
+    status = filters.MultipleChoiceFilter(
+        choices=[(slug, slug) for slug in RfcStatus.status_slugs],
+        method=RfcStatus.filter,
+    )
+    sort = filters.OrderingFilter(
+        fields=(
+            ("rfc_number", "number"),  # ?sort=number / ?sort=-number
+            ("published", "published"),  # ?sort=published / ?sort=-published
+        ),
+    )
+
+
+class PrefetchRelatedDocument(Prefetch):
+    """Prefetch via a RelatedDocument
+
+    Prefetches following RelatedDocument relationships to other docs. By default, includes
+    those for which the current RFC is the `source`. If `reverse` is True, includes those
+    for which it is the `target` instead. Defaults to only "rfc" documents.
+    """
+
+    @staticmethod
+    def _get_queryset(relationship_id, reverse, doc_type_ids):
+        """Get queryset to use for the prefetch"""
+        if isinstance(doc_type_ids, str):
+            doc_type_ids = (doc_type_ids,)
+
+        return RelatedDocument.objects.filter(
+            **{
+                "relationship_id": relationship_id,
+                f"{'source' if reverse else 'target'}__type_id__in": doc_type_ids,
+            }
+        ).select_related("source" if reverse else "target")
+
+    def __init__(self, to_attr, relationship_id, reverse=False, doc_type_ids="rfc"):
+        super().__init__(
+            lookup="targets_related" if reverse else "relateddocument_set",
+            queryset=self._get_queryset(relationship_id, reverse, doc_type_ids),
+            to_attr=to_attr,
+        )
+
+
+def augment_rfc_queryset(queryset: QuerySet[Document]):
+    return (
+        queryset.select_related("std_level", "stream")
+        .prefetch_related(
+            Prefetch(
+                "group",
+                Group.objects.select_related("parent"),
+            ),
+            Prefetch(
+                "documentauthor_set",
+                DocumentAuthor.objects.select_related("email", "person"),
+            ),
+            PrefetchRelatedDocument(
+                to_attr="drafts",
+                relationship_id="became_rfc",
+                doc_type_ids="draft",
+                reverse=True,
+            ),
+            PrefetchRelatedDocument(to_attr="obsoletes", relationship_id="obs"),
+            PrefetchRelatedDocument(
+                to_attr="obsoleted_by", relationship_id="obs", reverse=True
+            ),
+            PrefetchRelatedDocument(to_attr="updates", relationship_id="updates"),
+            PrefetchRelatedDocument(
+                to_attr="updated_by", relationship_id="updates", reverse=True
+            ),
+            PrefetchRelatedDocument(
+                to_attr="subseries",
+                relationship_id="contains",
+                reverse=True,
+                doc_type_ids=SUBSERIES_DOC_TYPE_IDS,
+            ),
+        )
+        .annotate(
+            published_datetime=Subquery(
+                DocEvent.objects.filter(
+                    doc_id=OuterRef("pk"),
+                    type="published_rfc",
+                )
+                .order_by("-time")
+                .values("time")[:1]
+            ),
+        )
+        .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO))
+        .annotate(
+            # TODO implement these fake fields for real
+            see_also=Value([], output_field=JSONField()),
+            formats=Value(["txt", "xml"], output_field=JSONField()),
+            keywords=Value(["keyword"], output_field=JSONField()),
+            errata=Value([], output_field=JSONField()),
+        )
+    )
+
+
+class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet):
+    api_key_endpoint = "ietf.api.red_api"  # matches prefix in ietf/api/urls.py
+    lookup_field = "rfc_number"
+    queryset = augment_rfc_queryset(
+        Document.objects.filter(type_id="rfc", rfc_number__isnull=False)
+    ).order_by("-rfc_number")
+
+    pagination_class = RfcLimitOffsetPagination
+    filter_backends = [filters.DjangoFilterBackend, drf_filters.SearchFilter]
+    filterset_class = RfcFilter
+    search_fields = ["title", "abstract"]
+
+    def get_serializer_class(self):
+        if self.action == "retrieve":
+            return RfcSerializer
+        return RfcMetadataSerializer
+
+
+class PrefetchSubseriesContents(Prefetch):
+    def __init__(self, to_attr):
+        super().__init__(
+            lookup="relateddocument_set",
+            queryset=RelatedDocument.objects.filter(
+                relationship_id="contains",
+                target__type_id="rfc",
+            ).prefetch_related(
+                Prefetch(
+                    "target",
+                    queryset=augment_rfc_queryset(Document.objects.all()),
+                )
+            ),
+            to_attr=to_attr,
+        )
+
+
+class SubseriesFilter(filters.FilterSet):
+    type = filters.ModelMultipleChoiceFilter(
+        queryset=DocTypeName.objects.filter(pk__in=SUBSERIES_DOC_TYPE_IDS)
+    )
+
+
+class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet):
+    api_key_endpoint = "ietf.api.red_api"  # matches prefix in ietf/api/urls.py
+    lookup_field = "name"
+    serializer_class = SubseriesDocSerializer
+    queryset = Document.objects.subseries_docs().prefetch_related(
+        PrefetchSubseriesContents(to_attr="contents")
+    )
+    filter_backends = [filters.DjangoFilterBackend]
+    filterset_class = SubseriesFilter
diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py
index 19aa9ecc9c..aad01be04f 100644
--- a/ietf/doc/factories.py
+++ b/ietf/doc/factories.py
@@ -14,7 +14,7 @@
 
 from ietf.doc.models import ( Document, DocEvent, NewRevisionDocEvent, State, DocumentAuthor,
     StateDocEvent, BallotPositionDocEvent, BallotDocEvent, BallotType, IRSGBallotDocEvent, TelechatDocEvent,
-    DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, DocExtResource )
+    DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, DocExtResource, RfcAuthor )
 from ietf.group.models import Group
 from ietf.person.factories import PersonFactory
 from ietf.group.factories import RoleFactory
@@ -382,6 +382,19 @@ class Meta:
     country = factory.Faker('country')
     order = factory.LazyAttribute(lambda o: o.document.documentauthor_set.count() + 1)
 
+class RfcAuthorFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = RfcAuthor
+
+    document = factory.SubFactory(DocumentFactory)
+    titlepage_name = factory.LazyAttribute(
+        lambda obj: " ".join([obj.person.initials(), obj.person.last_name()])
+    )
+    person = factory.SubFactory('ietf.person.factories.PersonFactory')
+    email = factory.LazyAttribute(lambda obj: obj.person.email())
+    affiliation = factory.Faker('company')
+    order = factory.LazyAttribute(lambda o: o.document.rfcauthor_set.count() + 1)
+
 class WgDocumentAuthorFactory(DocumentAuthorFactory):
     document = factory.SubFactory(WgDraftFactory)
 
diff --git a/ietf/doc/management/commands/reset_rfc_authors.py b/ietf/doc/management/commands/reset_rfc_authors.py
deleted file mode 100644
index e2ab5f1208..0000000000
--- a/ietf/doc/management/commands/reset_rfc_authors.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# Copyright The IETF Trust 2024, All Rights Reserved
-
-# Reset an RFC's authors to those of the draft it came from
-from django.core.management.base import BaseCommand, CommandError
-
-from ietf.doc.models import Document, DocEvent
-from ietf.person.models import Person
-
-
-class Command(BaseCommand):
-    def add_arguments(self, parser):
-        parser.add_argument("rfcnum", type=int, help="RFC number to modify")
-        parser.add_argument(
-            "--force",
-            action="store_true",
-            help="reset even if RFC already has authors",
-        )
-
-    def handle(self, *args, **options):
-        try:
-            rfc = Document.objects.get(type="rfc", rfc_number=options["rfcnum"])
-        except Document.DoesNotExist:
-            raise CommandError(
-                f"rfc{options['rfcnum']} does not exist in the Datatracker."
-            )
-
-        draft = rfc.came_from_draft()
-        if draft is None:
-            raise CommandError(f"{rfc.name} did not come from a draft. Can't reset.")
-
-        orig_authors = rfc.documentauthor_set.all()
-        if orig_authors.exists():
-            # Potentially dangerous, so refuse unless "--force" is specified
-            if not options["force"]:
-                raise CommandError(
-                    f"{rfc.name} already has authors. Not resetting. Use '--force' to reset anyway."
-                )
-            removed_auth_names = list(orig_authors.values_list("person__name", flat=True))
-            rfc.documentauthor_set.all().delete()
-            DocEvent.objects.create(
-                doc=rfc,
-                by=Person.objects.get(name="(System)"),
-                type="edited_authors",
-                desc=f"Removed all authors: {', '.join(removed_auth_names)}",
-            )
-            self.stdout.write(
-                self.style.SUCCESS(
-                    f"Removed author(s): {', '.join(removed_auth_names)}"
-                )
-            )
-
-        for author in draft.documentauthor_set.all():
-            # Copy the author but point at the new doc.
-            # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances
-            author.pk = None
-            author.id = None
-            author._state.adding = True
-            author.document = rfc
-            author.save()
-            self.stdout.write(
-                self.style.SUCCESS(f"Added author {author.person.name} <{author.email}>")
-            )
-        auth_names = draft.documentauthor_set.values_list("person__name", flat=True)
-        DocEvent.objects.create(
-            doc=rfc,
-            by=Person.objects.get(name="(System)"),
-            type="edited_authors",
-            desc=f"Set authors from rev {draft.rev} of {draft.name}: {', '.join(auth_names)}",
-        )
diff --git a/ietf/doc/management/commands/tests.py b/ietf/doc/management/commands/tests.py
deleted file mode 100644
index 8244d87266..0000000000
--- a/ietf/doc/management/commands/tests.py
+++ /dev/null
@@ -1,72 +0,0 @@
-# Copyright The IETF Trust 2024, All Rights Reserved
-# -*- coding: utf-8 -*-
-
-from io import StringIO
-
-from django.core.management import call_command, CommandError
-
-from ietf.doc.factories import DocumentAuthorFactory, WgDraftFactory, WgRfcFactory
-from ietf.doc.models import Document, DocumentAuthor
-from ietf.utils.test_utils import TestCase
-
-
-class CommandTests(TestCase):
-    @staticmethod
-    def _call_command(command_name, *args, **options):
-        """Call command, capturing (and suppressing) output"""
-        out = StringIO()
-        err = StringIO()
-        options["stdout"] = out
-        options["stderr"] = err
-        call_command(command_name, *args, **options)
-        return out.getvalue(), err.getvalue()
-
-    def test_reset_rfc_authors(self):
-        command_name = "reset_rfc_authors"
-
-        draft = WgDraftFactory()
-        DocumentAuthorFactory.create_batch(3, document=draft)
-        rfc = WgRfcFactory()  # rfc does not yet have a draft
-        DocumentAuthorFactory.create_batch(3, document=rfc)
-        bad_rfc_num = (
-            1
-            + Document.objects.filter(rfc_number__isnull=False)
-            .order_by("-rfc_number")
-            .first()
-            .rfc_number
-        )
-        docauthor_fields = [
-            field.name
-            for field in DocumentAuthor._meta.get_fields()
-            if field.name not in ["document", "id"]
-        ]
-
-        with self.assertRaises(CommandError, msg="Cannot reset a bad RFC number"):
-            self._call_command(command_name, bad_rfc_num)
-
-        with self.assertRaises(CommandError, msg="Cannot reset an RFC with no draft"):
-            self._call_command(command_name, rfc.rfc_number)
-
-        with self.assertRaises(CommandError, msg="Cannot force-reset an RFC with no draft"):
-            self._call_command(command_name, rfc.rfc_number, "--force")
-
-        # Link the draft to the rfc
-        rfc.targets_related.create(relationship_id="became_rfc", source=draft)
-
-        with self.assertRaises(CommandError, msg="Cannot reset an RFC with authors"):
-            self._call_command(command_name, rfc.rfc_number)
-
-        # Calling with force should work
-        self._call_command(command_name, rfc.rfc_number, "--force")
-        self.assertCountEqual(
-            draft.documentauthor_set.values(*docauthor_fields),
-            rfc.documentauthor_set.values(*docauthor_fields),
-        )
-
-        # Calling on an RFC with no authors should also work
-        rfc.documentauthor_set.all().delete()
-        self._call_command(command_name, rfc.rfc_number)
-        self.assertCountEqual(
-            draft.documentauthor_set.values(*docauthor_fields),
-            rfc.documentauthor_set.values(*docauthor_fields),
-        )
diff --git a/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py b/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py
new file mode 100644
index 0000000000..e0d8560e6f
--- /dev/null
+++ b/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py
@@ -0,0 +1,41 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("doc", "0026_change_wg_state_descriptions"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="dochistory",
+            name="title",
+            field=models.CharField(
+                max_length=255,
+                validators=[
+                    django.core.validators.ProhibitNullCharactersValidator,  # type:ignore
+                    django.core.validators.RegexValidator(
+                        message="Please enter a string without control characters.",
+                        regex="^[^\x01-\x1f]*$",
+                    ),
+                ],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="document",
+            name="title",
+            field=models.CharField(
+                max_length=255,
+                validators=[
+                    django.core.validators.ProhibitNullCharactersValidator,  # type:ignore
+                    django.core.validators.RegexValidator(
+                        message="Please enter a string without control characters.",
+                        regex="^[^\x01-\x1f]*$",
+                    ),
+                ],
+            ),
+        ),
+    ]
diff --git a/ietf/doc/migrations/0028_rfcauthor.py b/ietf/doc/migrations/0028_rfcauthor.py
new file mode 100644
index 0000000000..776dc22eb1
--- /dev/null
+++ b/ietf/doc/migrations/0028_rfcauthor.py
@@ -0,0 +1,84 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+import django.db.models.deletion
+import ietf.utils.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("person", "0005_alter_historicalperson_pronouns_selectable_and_more"),
+        ("doc", "0027_alter_dochistory_title_alter_document_title"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RfcAuthor",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("titlepage_name", models.CharField(max_length=128)),
+                ("is_editor", models.BooleanField(default=False)),
+                (
+                    "affiliation",
+                    models.CharField(
+                        blank=True,
+                        help_text="Organization/company used by author for submission",
+                        max_length=100,
+                    ),
+                ),
+                (
+                    "country",
+                    models.CharField(
+                        blank=True,
+                        help_text="Country used by author for submission",
+                        max_length=255,
+                    ),
+                ),
+                ("order", models.IntegerField(default=1)),
+                (
+                    "document",
+                    ietf.utils.models.ForeignKey(
+                        limit_choices_to={"type_id": "rfc"},
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="doc.document",
+                    ),
+                ),
+                (
+                    "email",
+                    ietf.utils.models.ForeignKey(
+                        blank=True,
+                        help_text="Email address used by author for submission",
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="person.email",
+                    ),
+                ),
+                (
+                    "person",
+                    ietf.utils.models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="person.person",
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["document", "order"],
+                "indexes": [
+                    models.Index(
+                        fields=["document", "order"],
+                        name="doc_rfcauth_documen_6b5dc4_idx",
+                    )
+                ],
+            },
+        ),
+    ]
diff --git a/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py b/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py
new file mode 100644
index 0000000000..60837c5cb2
--- /dev/null
+++ b/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py
@@ -0,0 +1,30 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("doc", "0028_rfcauthor"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="EditedRfcAuthorsDocEvent",
+            fields=[
+                (
+                    "docevent_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="doc.docevent",
+                    ),
+                ),
+            ],
+            bases=("doc.docevent",),
+        ),
+    ]
diff --git a/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py b/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py
new file mode 100644
index 0000000000..9ee858b2e8
--- /dev/null
+++ b/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py
@@ -0,0 +1,41 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("doc", "0029_editedrfcauthorsdocevent"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="dochistory",
+            name="title",
+            field=models.CharField(
+                max_length=255,
+                validators=[
+                    django.core.validators.ProhibitNullCharactersValidator(),
+                    django.core.validators.RegexValidator(
+                        message="Please enter a string without control characters.",
+                        regex="^[^\x01-\x1f]*$",
+                    ),
+                ],
+            ),
+        ),
+        migrations.AlterField(
+            model_name="document",
+            name="title",
+            field=models.CharField(
+                max_length=255,
+                validators=[
+                    django.core.validators.ProhibitNullCharactersValidator(),
+                    django.core.validators.RegexValidator(
+                        message="Please enter a string without control characters.",
+                        regex="^[^\x01-\x1f]*$",
+                    ),
+                ],
+            ),
+        ),
+    ]
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index 8bb79b64ed..cce9203d09 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -1,7 +1,8 @@
-# Copyright The IETF Trust 2010-2025, All Rights Reserved
+# Copyright The IETF Trust 2010-2026, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
+from collections import namedtuple
 import datetime
 import logging
 import os
@@ -20,7 +21,11 @@
 from django.core import checks
 from django.core.files.base import File
 from django.core.cache import caches
-from django.core.validators import URLValidator, RegexValidator
+from django.core.validators import (
+    URLValidator,
+    RegexValidator,
+    ProhibitNullCharactersValidator,
+)
 from django.urls import reverse as urlreverse
 from django.contrib.contenttypes.models import ContentType
 from django.conf import settings
@@ -107,7 +112,13 @@ class DocumentInfo(models.Model):
     time = models.DateTimeField(default=timezone.now) # should probably have auto_now=True
 
     type = ForeignKey(DocTypeName, blank=True, null=True) # Draft, Agenda, Minutes, Charter, Discuss, Guideline, Email, Review, Issue, Wiki, External ...
-    title = models.CharField(max_length=255, validators=[validate_no_control_chars, ])
+    title = models.CharField(
+        max_length=255,
+        validators=[
+            ProhibitNullCharactersValidator(),
+            validate_no_control_chars,
+        ],
+    )
 
     states = models.ManyToManyField(State, blank=True) # plain state (Active/Expired/...), IESG state, stream state
     tags = models.ManyToManyField(DocTagName, blank=True) # Revised ID Needed, ExternalParty, AD Followup, ...
@@ -407,9 +418,55 @@ def friendly_state(self):
         else:
             return state.name
 
+    def author_names(self):
+        """Author names as a list of strings"""
+        names = []
+        if self.type_id == "rfc" and self.rfcauthor_set.exists():
+            for author in self.rfcauthor_set.select_related("person"):
+                if author.person:
+                    names.append(author.person.name)
+                else:
+                    # titlepage_name cannot be blank
+                    names.append(author.titlepage_name)
+        else:
+            names = [
+                author.person.name
+                for author in self.documentauthor_set.select_related("person")
+            ]
+        return names
+
+    def author_persons_or_names(self):
+        """Authors as a list of named tuples with person and/or titlepage_name"""
+        Author = namedtuple("Author", "person titlepage_name")
+        persons_or_names = []
+        if self.type_id=="rfc" and self.rfcauthor_set.exists():
+            for author in self.rfcauthor_set.select_related("person"):
+                persons_or_names.append(Author(person=author.person, titlepage_name=author.titlepage_name))
+        else:
+            for author in self.documentauthor_set.select_related("person"):
+                persons_or_names.append(Author(person=author.person, titlepage_name=""))
+        return persons_or_names
+
+    def author_persons(self):
+        """Authors as a list of Persons
+        
+        Omits any RfcAuthors with a null person field.
+        """
+        if self.type_id == "rfc" and self.rfcauthor_set.exists():
+            authors_qs = self.rfcauthor_set.filter(person__isnull=False)
+        else:
+            authors_qs = self.documentauthor_set.all()
+        return [a.person for a in authors_qs.select_related("person")]
+
     def author_list(self):
+        """List of author emails"""
+        author_qs = (
+            self.rfcauthor_set
+            if self.type_id == "rfc" and self.rfcauthor_set.exists()
+            else self.documentauthor_set
+        ).select_related("email").order_by("order")
         best_addresses = []
-        for author in self.documentauthor_set.all():
+        for author in author_qs:
             if author.email:
                 if author.email.active or not author.email.person:
                     best_addresses.append(author.email.address)
@@ -417,9 +474,6 @@ def author_list(self):
                     best_addresses.append(author.email.person.email_address())
         return ", ".join(best_addresses)
 
-    def authors(self):
-        return [ a.person for a in self.documentauthor_set.all() ]
-
     # This, and several other ballot related functions here, assume that there is only one active ballot for a document at any point in time.
     # If that assumption is violated, they will only expose the most recently created ballot
     def ballot_open(self, ballot_type_slug):
@@ -721,7 +775,14 @@ def referenced_by_rfcs_as_rfc_or_draft(self):
         if self.type_id == "rfc" and self.came_from_draft():
             refs_to |= self.came_from_draft().referenced_by_rfcs()
         return refs_to
-    
+
+    def sent_to_rfc_editor_event(self):
+        if self.stream_id == "ietf":
+            return self.docevent_set.filter(type="iesg_approved").order_by("-time").first()
+        elif self.stream_id in ["editorial", "iab", "irtf", "ise"]:
+            return self.docevent_set.filter(type="requested_publication").order_by("-time").first()
+        else:
+            return None
     class Meta:
         abstract = True
 
@@ -845,6 +906,45 @@ def is_approved_downref(self):
 
         return False
 
+class RfcAuthor(models.Model):
+    """Captures the authors of an RFC as represented on the RFC title page.
+
+    This deviates from DocumentAuthor in that it does not get moved into the DocHistory
+    hierarchy as documents are saved. It will attempt to preserve email, country, and affiliation
+    from the DocumentAuthor objects associated with the draft leading to this RFC (which
+    may be wrong if the author moves or changes affiliation while the document is in the
+    queue).
+
+    It does not, at this time, attempt to capture the authors from anything _but_ the title
+    page. The datatracker may know more about such authors based on information from the draft
+    leading to the RFC, and future work may take that into account.
+
+    Once doc.rfcauthor_set.exists() for a doc of type `rfc`, doc.documentauthor_set should be 
+    ignored.
+    """
+
+    document = ForeignKey(
+        "Document",
+        on_delete=models.CASCADE,
+        limit_choices_to={"type_id": "rfc"},  # only affects ModelForms (e.g., admin)
+    )
+    titlepage_name = models.CharField(max_length=128, blank=False)
+    is_editor = models.BooleanField(default=False)
+    person = ForeignKey(Person, null=True, blank=True, on_delete=models.PROTECT)
+    email = ForeignKey(Email, help_text="Email address used by author for submission", blank=True, null=True, on_delete=models.PROTECT)
+    affiliation = models.CharField(max_length=100, blank=True, help_text="Organization/company used by author for submission")
+    country = models.CharField(max_length=255, blank=True, help_text="Country used by author for submission")
+    order = models.IntegerField(default=1)
+
+    def __str__(self):
+        return u"%s %s (%s)" % (self.document.name, self.person, self.order)
+
+    class Meta:
+        ordering=["document", "order"]
+        indexes=[
+            models.Index(fields=["document", "order"])
+        ]
+
 class DocumentAuthorInfo(models.Model):
     person = ForeignKey(Person)
     # email should only be null for some historic documents
@@ -894,7 +994,7 @@ class Meta:
     def role_for_doc(self):
         """Brief string description of this person's relationship to the doc"""
         roles = []
-        if self.person in self.document.authors():
+        if self.person in self.document.author_persons():
             roles.append('Author')
         if self.person == self.document.ad:
             roles.append('Responsible AD')
@@ -920,7 +1020,18 @@ def role_for_doc(self):
     'invalid'
 )
 
+
+SUBSERIES_DOC_TYPE_IDS = ("bcp", "fyi", "std")
+
+
+class DocumentQuerySet(models.QuerySet):
+    def subseries_docs(self):
+        return self.filter(type_id__in=SUBSERIES_DOC_TYPE_IDS)
+
+
 class Document(StorableMixin, DocumentInfo):
+    objects = DocumentQuerySet.as_manager()
+
     name = models.CharField(max_length=255, validators=[validate_docname,], unique=True)           # immutable
     
     action_holders = models.ManyToManyField(Person, through=DocumentActionHolder, blank=True)
@@ -1581,6 +1692,11 @@ class EditedAuthorsDocEvent(DocEvent):
     """
     basis = models.CharField(help_text="What is the source or reasoning for the changes to the author list",max_length=255)
 
+
+class EditedRfcAuthorsDocEvent(DocEvent):
+    """Change to the RfcAuthor list for a document"""
+
+
 class BofreqEditorDocEvent(DocEvent):
     """ Capture the proponents of a BOF Request."""
     editors = models.ManyToManyField('person.Person', blank=True)
diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py
index 157a3ad556..556465a522 100644
--- a/ietf/doc/resources.py
+++ b/ietf/doc/resources.py
@@ -17,8 +17,9 @@
     InitialReviewDocEvent, DocHistoryAuthor, BallotDocEvent, RelatedDocument,
     RelatedDocHistory, BallotPositionDocEvent, AddedMessageEvent, SubmissionDocEvent,
     ReviewRequestDocEvent, ReviewAssignmentDocEvent, EditedAuthorsDocEvent, DocumentURL,
-    IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, 
-    BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject)
+    IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder,
+    BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject, RfcAuthor,
+    EditedRfcAuthorsDocEvent)
 
 from ietf.name.resources import BallotPositionNameResource, DocTypeNameResource
 class BallotTypeResource(ModelResource):
@@ -650,6 +651,31 @@ class Meta:
 api.doc.register(EditedAuthorsDocEventResource())
 
 
+
+from ietf.person.resources import PersonResource
+class EditedRfcAuthorsDocEventResource(ModelResource):
+    by               = ToOneField(PersonResource, 'by')
+    doc              = ToOneField(DocumentResource, 'doc')
+    docevent_ptr     = ToOneField(DocEventResource, 'docevent_ptr')
+    class Meta:
+        queryset = EditedRfcAuthorsDocEvent.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'editedrfcauthorsdocevent'
+        ordering = ['id', ]
+        filtering = { 
+            "id": ALL,
+            "time": ALL,
+            "type": ALL,
+            "rev": ALL,
+            "desc": ALL,
+            "by": ALL_WITH_RELATIONS,
+            "doc": ALL_WITH_RELATIONS,
+            "docevent_ptr": ALL_WITH_RELATIONS,
+        }
+api.doc.register(EditedRfcAuthorsDocEventResource())
+
+
 from ietf.name.resources import DocUrlTagNameResource
 class DocumentURLResource(ModelResource):
     doc              = ToOneField(DocumentResource, 'doc')
@@ -865,3 +891,28 @@ class Meta:
             "deleted": ALL,
         }
 api.doc.register(StoredObjectResource())
+
+
+from ietf.person.resources import EmailResource, PersonResource
+class RfcAuthorResource(ModelResource):
+    document         = ToOneField(DocumentResource, 'document')
+    person           = ToOneField(PersonResource, 'person', null=True)
+    email            = ToOneField(EmailResource, 'email', null=True)
+    class Meta:
+        queryset = RfcAuthor.objects.all()
+        serializer = api.Serializer()
+        cache = SimpleCache()
+        #resource_name = 'rfcauthor'
+        ordering = ['id', ]
+        filtering = { 
+            "id": ALL,
+            "titlepage_name": ALL,
+            "is_editor": ALL,
+            "affiliation": ALL,
+            "country": ALL,
+            "order": ALL,
+            "document": ALL_WITH_RELATIONS,
+            "person": ALL_WITH_RELATIONS,
+            "email": ALL_WITH_RELATIONS,
+        }
+api.doc.register(RfcAuthorResource())
diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py
new file mode 100644
index 0000000000..05647d9ce1
--- /dev/null
+++ b/ietf/doc/serializers.py
@@ -0,0 +1,316 @@
+# Copyright The IETF Trust 2024-2026, All Rights Reserved
+"""django-rest-framework serializers"""
+
+from dataclasses import dataclass
+from typing import Literal, ClassVar
+
+from django.db.models.manager import BaseManager
+from django.db.models.query import QuerySet
+from drf_spectacular.utils import extend_schema_field
+from rest_framework import serializers
+
+from ietf.group.serializers import GroupSerializer
+from ietf.name.serializers import StreamNameSerializer
+from .models import Document, DocumentAuthor, RfcAuthor
+
+
+class RfcAuthorSerializer(serializers.ModelSerializer):
+    """Serializer for an RfcAuthor / DocumentAuthor in a response"""
+    datatracker_person_path = serializers.URLField(
+        source="person.get_absolute_url",
+        required=False,
+        help_text="URL for person link (relative to datatracker base URL)",
+    )
+
+    class Meta:
+        model = RfcAuthor
+        fields = [
+            "titlepage_name",
+            "is_editor",
+            "person",
+            "email",  # relies on email.pk being email.address
+            "affiliation",
+            "country",
+            "datatracker_person_path",
+        ]
+
+    def to_representation(self, instance):
+        """instance -> primitive data types
+        
+        Translates a DocumentAuthor into an equivalent RfcAuthor we can use the same
+        serializer for either type.
+        """
+        if isinstance(instance, DocumentAuthor):
+            # create a non-persisted RfcAuthor as a shim - do not save it!
+            document_author = instance
+            instance = RfcAuthor(
+                titlepage_name=document_author.person.plain_name(),
+                is_editor=False,
+                person=document_author.person,
+                email=document_author.email,
+                affiliation=document_author.affiliation,
+                country=document_author.country,
+                order=document_author.order,
+            )
+        return super().to_representation(instance)
+
+    def validate(self, data):
+        email = data.get("email")
+        if email is not None:
+            person = data.get("person")
+            if person is None:
+                raise serializers.ValidationError(
+                    {
+                        "email": "cannot have an email without a person",
+                    },
+                    code="email-without-person",
+                )
+            if email.person_id != person.pk:
+                raise serializers.ValidationError(
+                    {
+                        "email": "email must belong to person",
+                    },
+                    code="email-person-mismatch",
+                )
+        return data
+
+
+@dataclass
+class DocIdentifier:
+    type: Literal["doi", "issn"]
+    value: str
+
+
+class DocIdentifierSerializer(serializers.Serializer):
+    type = serializers.ChoiceField(choices=["doi", "issn"])
+    value = serializers.CharField()
+
+
+type RfcStatusSlugT = Literal[
+    "std", "ps", "ds", "bcp", "inf", "exp", "hist", "unkn", "not-issued",
+]
+
+
+@dataclass
+class RfcStatus:
+    """Helper to extract the 'Status' from an RFC document for serialization"""
+
+    slug: RfcStatusSlugT
+
+    # Names that aren't just the slug itself. ClassVar annotation prevents dataclass from treating this as a field.
+    fancy_names: ClassVar[dict[RfcStatusSlugT, str]] = {
+        "std": "internet standard",
+        "ps": "proposed standard",
+        "ds": "draft standard",
+        "bcp": "best current practice",
+        "inf": "informational",
+        "exp": "experimental",
+        "hist": "historic",
+        "unkn": "unknown",
+    }
+
+    # ClassVar annotation prevents dataclass from treating this as a field
+    stdlevelname_slug_map: ClassVar[dict[str, RfcStatusSlugT]] = {
+        "bcp": "bcp",
+        "ds": "ds",
+        "exp": "exp",
+        "hist": "hist",
+        "inf": "inf",
+        "std": "std",
+        "ps": "ps",
+        "unkn": "unkn",
+    }
+
+    # ClassVar annotation prevents dataclass from treating this as a field
+    status_slugs: ClassVar[list[RfcStatusSlugT]] = sorted(
+        # TODO implement "not-issued" RFCs
+        set(stdlevelname_slug_map.values()) | {"not-issued"}
+    )
+
+    @property
+    def name(self):
+        return RfcStatus.fancy_names.get(self.slug, self.slug)
+
+    @classmethod
+    def from_document(cls, doc: Document):
+        """Decide the status that applies to a document"""
+        return cls(
+            slug=(cls.stdlevelname_slug_map.get(doc.std_level.slug, "unkn")),
+        )
+
+    @classmethod
+    def filter(cls, queryset, name, value: list[RfcStatusSlugT]):
+        """Filter a queryset by status
+
+        This is basically the inverse of the from_document() method. Given a status name, filter
+        the queryset to those in that status. The queryset should be a Document queryset.
+        """
+        interesting_slugs = [
+            stdlevelname_slug
+            for stdlevelname_slug, status_slug in cls.stdlevelname_slug_map.items()
+            if status_slug in value
+        ]
+        if len(interesting_slugs) == 0:
+            return queryset.none()
+        return queryset.filter(std_level__slug__in=interesting_slugs)
+
+
+class RfcStatusSerializer(serializers.Serializer):
+    """Status serializer for a Document instance"""
+
+    slug = serializers.ChoiceField(choices=RfcStatus.status_slugs)
+    name = serializers.CharField()
+
+    def to_representation(self, instance: Document):
+        return super().to_representation(instance=RfcStatus.from_document(instance))
+
+
+class RelatedDraftSerializer(serializers.Serializer):
+    id = serializers.IntegerField(source="source.id")
+    name = serializers.CharField(source="source.name")
+    title = serializers.CharField(source="source.title")
+
+
+class RelatedRfcSerializer(serializers.Serializer):
+    id = serializers.IntegerField(source="target.id")
+    number = serializers.IntegerField(source="target.rfc_number")
+    title = serializers.CharField(source="target.title")
+
+
+class ReverseRelatedRfcSerializer(serializers.Serializer):
+    id = serializers.IntegerField(source="source.id")
+    number = serializers.IntegerField(source="source.rfc_number")
+    title = serializers.CharField(source="source.title")
+
+
+class ContainingSubseriesSerializer(serializers.Serializer):
+    name = serializers.CharField(source="source.name")
+    type = serializers.CharField(source="source.type_id")
+
+
+class RfcMetadataSerializer(serializers.ModelSerializer):
+    """Serialize metadata of an RFC"""
+
+    RFC_FORMATS = ("xml", "txt", "html", "htmlized", "pdf", "ps")
+
+    number = serializers.IntegerField(source="rfc_number")
+    published = serializers.DateField()
+    status = RfcStatusSerializer(source="*")
+    authors = serializers.SerializerMethodField()
+    group = GroupSerializer()
+    area = GroupSerializer(source="group.area", required=False)
+    stream = StreamNameSerializer()
+    identifiers = serializers.SerializerMethodField()
+    draft = serializers.SerializerMethodField()
+    obsoletes = RelatedRfcSerializer(many=True, read_only=True)
+    obsoleted_by = ReverseRelatedRfcSerializer(many=True, read_only=True)
+    updates = RelatedRfcSerializer(many=True, read_only=True)
+    updated_by = ReverseRelatedRfcSerializer(many=True, read_only=True)
+    subseries = ContainingSubseriesSerializer(many=True, read_only=True)
+    see_also = serializers.ListField(child=serializers.CharField(), read_only=True)
+    formats = serializers.MultipleChoiceField(choices=RFC_FORMATS)
+    keywords = serializers.ListField(child=serializers.CharField(), read_only=True)
+    errata = serializers.ListField(child=serializers.CharField(), read_only=True)
+
+    class Meta:
+        model = Document
+        fields = [
+            "number",
+            "title",
+            "published",
+            "status",
+            "pages",
+            "authors",
+            "group",
+            "area",
+            "stream",
+            "identifiers",
+            "obsoletes",
+            "obsoleted_by",
+            "updates",
+            "updated_by",
+            "subseries",
+            "see_also",
+            "draft",
+            "abstract",
+            "formats",
+            "keywords",
+            "errata",
+        ]
+
+
+    @extend_schema_field(RfcAuthorSerializer(many=True))
+    def get_authors(self, doc: Document):
+        # If doc has any RfcAuthors, use those, otherwise fall back to DocumentAuthors
+        author_queryset: QuerySet[RfcAuthor] | QuerySet[DocumentAuthor] = (
+            doc.rfcauthor_set.all()
+            if doc.rfcauthor_set.exists()
+            else doc.documentauthor_set.all()
+        )
+        # RfcAuthorSerializer can deal with DocumentAuthor instances
+        return RfcAuthorSerializer(
+            instance=author_queryset,
+            many=True,
+        ).data
+
+    @extend_schema_field(DocIdentifierSerializer(many=True))
+    def get_identifiers(self, doc: Document):
+        identifiers = []
+        if doc.rfc_number:
+            identifiers.append(
+                DocIdentifier(type="doi", value=f"10.17487/RFC{doc.rfc_number:04d}")
+            )
+        return DocIdentifierSerializer(instance=identifiers, many=True).data
+
+    @extend_schema_field(RelatedDraftSerializer)
+    def get_draft(self, object):
+        try:
+            related_doc = object.drafts[0]
+        except IndexError:
+            return None
+        return RelatedDraftSerializer(related_doc).data
+
+
+class RfcSerializer(RfcMetadataSerializer):
+    """Serialize an RFC, including its metadata and text content if available"""
+
+    text = serializers.CharField(allow_null=True)
+
+    class Meta:
+        model = RfcMetadataSerializer.Meta.model
+        fields = RfcMetadataSerializer.Meta.fields + ["text"]
+
+
+class SubseriesContentListSerializer(serializers.ListSerializer):
+    """ListSerializer that gets its object from item.target"""
+
+    def to_representation(self, data):
+        """
+        List of object instances -> List of dicts of primitive datatypes.
+        """
+        # Dealing with nested relationships, data can be a Manager,
+        # so, first get a queryset from the Manager if needed
+        iterable = data.all() if isinstance(data, BaseManager) else data
+        # Serialize item.target instead of item itself
+        return [self.child.to_representation(item.target) for item in iterable]
+
+
+class SubseriesContentSerializer(RfcMetadataSerializer):
+    """Serialize RFC contained in a subseries doc"""
+
+    class Meta(RfcMetadataSerializer.Meta):
+        list_serializer_class = SubseriesContentListSerializer
+
+
+class SubseriesDocSerializer(serializers.ModelSerializer):
+    """Serialize a subseries document (e.g., a BCP or STD)"""
+
+    contents = SubseriesContentSerializer(many=True)
+
+    class Meta:
+        model = Document
+        fields = [
+            "name",
+            "type",
+            "contents",
+        ]
diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index 16dcfb7754..f92c9648e6 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -39,11 +39,15 @@
 from ietf.doc.models import ( Document, DocRelationshipName, RelatedDocument, State,
     DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType,
     EditedAuthorsDocEvent, StateType)
-from ietf.doc.factories import ( DocumentFactory, DocEventFactory, CharterFactory,
-    ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, 
-    IndividualRfcFactory, StateDocEventFactory, BallotPositionDocEventFactory, 
-    BallotDocEventFactory, DocumentAuthorFactory, NewRevisionDocEventFactory,
-    StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory)
+from ietf.doc.factories import (DocumentFactory, DocEventFactory, CharterFactory,
+                                ConflictReviewFactory, WgDraftFactory,
+                                IndividualDraftFactory, WgRfcFactory,
+                                IndividualRfcFactory, StateDocEventFactory,
+                                BallotPositionDocEventFactory,
+                                BallotDocEventFactory, DocumentAuthorFactory,
+                                NewRevisionDocEventFactory,
+                                StatusChangeFactory, DocExtResourceFactory,
+                                RgDraftFactory, BcpFactory, RfcAuthorFactory)
 from ietf.doc.forms import NotifyForm
 from ietf.doc.fields import SearchableDocumentsField
 from ietf.doc.utils import (
@@ -979,7 +983,7 @@ def test_edit_authors_permissions(self):
         # Relevant users not authorized to edit authors
         unauthorized_usernames = [
             'plain',
-            *[author.user.username for author in draft.authors()],
+            *[author.user.username for author in draft.author_persons()],
             draft.group.get_chair().person.user.username,
             'ad'
         ]
@@ -994,7 +998,7 @@ def test_edit_authors_permissions(self):
         self.client.logout()
 
         # Try to add an author via POST - still only the secretary should be able to do this.
-        orig_authors = draft.authors()
+        orig_authors = draft.author_persons()
         post_data = self.make_edit_authors_post_data(
             basis='permission test',
             authors=draft.documentauthor_set.all(),
@@ -1012,12 +1016,12 @@ def test_edit_authors_permissions(self):
         for username in unauthorized_usernames:
             login_testing_unauthorized(self, username, url, method='post', request_kwargs=dict(data=post_data))
             draft = Document.objects.get(pk=draft.pk)
-            self.assertEqual(draft.authors(), orig_authors)  # ensure draft author list was not modified
+            self.assertEqual(draft.author_persons(), orig_authors)  # ensure draft author list was not modified
         login_testing_unauthorized(self, 'secretary', url, method='post', request_kwargs=dict(data=post_data))
         r = self.client.post(url, post_data)
         self.assertEqual(r.status_code, 302)
         draft = Document.objects.get(pk=draft.pk)
-        self.assertEqual(draft.authors(), orig_authors + [new_auth_person])
+        self.assertEqual(draft.author_persons(), orig_authors + [new_auth_person])
 
     def make_edit_authors_post_data(self, basis, authors):
         """Helper to generate edit_authors POST data for a set of authors"""
@@ -1365,8 +1369,8 @@ def test_edit_authors_edit_fields(self):
             basis=change_reason
         )
 
-        old_address = draft.authors()[0].email()
-        new_email = EmailFactory(person=draft.authors()[0], address=f'changed-{old_address}')
+        old_address = draft.author_persons()[0].email()
+        new_email = EmailFactory(person=draft.author_persons()[0], address=f'changed-{old_address}')
         post_data['author-0-email'] = new_email.address
         post_data['author-1-affiliation'] = 'University of Nowhere'
         post_data['author-2-country'] = 'Chile'
@@ -1399,17 +1403,17 @@ def test_edit_authors_edit_fields(self):
         country_event = change_events.filter(desc__icontains='changed country').first()
 
         self.assertIsNotNone(email_event)
-        self.assertIn(draft.authors()[0].name, email_event.desc)
+        self.assertIn(draft.author_persons()[0].name, email_event.desc)
         self.assertIn(before[0]['email'], email_event.desc)
         self.assertIn(after[0]['email'], email_event.desc)
 
         self.assertIsNotNone(affiliation_event)
-        self.assertIn(draft.authors()[1].name, affiliation_event.desc)
+        self.assertIn(draft.author_persons()[1].name, affiliation_event.desc)
         self.assertIn(before[1]['affiliation'], affiliation_event.desc)
         self.assertIn(after[1]['affiliation'], affiliation_event.desc)
 
         self.assertIsNotNone(country_event)
-        self.assertIn(draft.authors()[2].name, country_event.desc)
+        self.assertIn(draft.author_persons()[2].name, country_event.desc)
         self.assertIn(before[2]['country'], country_event.desc)
         self.assertIn(after[2]['country'], country_event.desc)
 
@@ -1863,13 +1867,63 @@ def test_document_ballot_needed_positions(self):
 
     def test_document_json(self):
         doc = IndividualDraftFactory()
-
+        author = DocumentAuthorFactory(document=doc)
+        
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_json", kwargs=dict(name=doc.name)))
         self.assertEqual(r.status_code, 200)
         data = r.json()
-        self.assertEqual(doc.name, data['name'])
-        self.assertEqual(doc.pages,data['pages'])
+        self.assertEqual(data["name"], doc.name)
+        self.assertEqual(data["pages"], doc.pages)
+        self.assertEqual(
+            data["authors"],
+            [
+                {
+                    "name": author.person.name,
+                    "email": author.email.address,
+                    "affiliation": author.affiliation,
+                }
+            ]
+        )
 
+    def test_document_json_rfc(self):
+        doc = IndividualRfcFactory()
+        old_style_author = DocumentAuthorFactory(document=doc)
+        url = urlreverse("ietf.doc.views_doc.document_json", kwargs=dict(name=doc.name))
+
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        data = r.json()
+        self.assertEqual(data["name"], doc.name)
+        self.assertEqual(data["pages"], doc.pages)
+        self.assertEqual(
+            data["authors"],
+            [
+                {
+                    "name": old_style_author.person.name,
+                    "email": old_style_author.email.address,
+                    "affiliation": old_style_author.affiliation,
+                }
+            ]
+        )
+    
+        new_style_author = RfcAuthorFactory(document=doc)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        data = r.json()
+        self.assertEqual(data["name"], doc.name)
+        self.assertEqual(data["pages"], doc.pages)
+        self.assertEqual(
+            data["authors"],
+            [
+                {
+                    "name": new_style_author.titlepage_name,
+                    "email": new_style_author.email.address,
+                    "affiliation": new_style_author.affiliation,
+                }
+            ]
+        )
+
+    
     def test_writeup(self):
         doc = IndividualDraftFactory(states = [('draft','active'),('draft-iesg','iesg-eva')],)
 
diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py
index cebeac1f27..21a873c5c0 100644
--- a/ietf/doc/tests_draft.py
+++ b/ietf/doc/tests_draft.py
@@ -140,7 +140,7 @@ def test_change_state(self):
         self.assertEqual(draft.get_state_slug("draft-iesg"), "review-e")
         self.assertTrue(not draft.tags.filter(slug="ad-f-up"))
         self.assertTrue(draft.tags.filter(slug="need-rev"))
-        self.assertCountEqual(draft.action_holders.all(), [ad] + draft.authors())
+        self.assertCountEqual(draft.action_holders.all(), [ad] + draft.author_persons())
         self.assertEqual(draft.docevent_set.count(), events_before + 3)
         self.assertTrue("Test comment" in draft.docevent_set.all()[0].desc)
         self.assertTrue("Changed action holders" in draft.docevent_set.all()[1].desc)
@@ -179,7 +179,7 @@ def test_pull_from_rfc_queue(self):
             states=[('draft-iesg','rfcqueue')],
         )
         DocEventFactory(type='started_iesg_process',by=ad,doc=draft,rev=draft.rev,desc="Started IESG Process")
-        draft.action_holders.add(*(draft.authors()))
+        draft.action_holders.add(*(draft.author_persons()))
 
         url = urlreverse('ietf.doc.views_draft.change_state', kwargs=dict(name=draft.name))
         login_testing_unauthorized(self, "secretary", url)
@@ -279,7 +279,7 @@ def test_request_last_call(self):
             states=[('draft-iesg','ad-eval')],
         )
         DocEventFactory(type='started_iesg_process',by=ad,doc=draft,rev=draft.rev,desc="Started IESG Process")
-        draft.action_holders.add(*(draft.authors()))
+        draft.action_holders.add(*(draft.author_persons()))
 
         self.client.login(username="secretary", password="secretary+password")
         url = urlreverse('ietf.doc.views_draft.change_state', kwargs=dict(name=draft.name))
@@ -1369,7 +1369,7 @@ def _test_changing_ah(action_holders, reason):
 
         _test_changing_ah([doc.ad, doc.shepherd.person], 'this is a first test')
         _test_changing_ah([doc.ad], 'this is a second test')
-        _test_changing_ah(doc.authors(), 'authors can do it, too')
+        _test_changing_ah(doc.author_persons(), 'authors can do it, too')
         _test_changing_ah([], 'clear it back out')
 
     def test_doc_change_action_holders_as_doc_manager(self):
diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py
index 8c1fc99ffe..82d1b5c232 100644
--- a/ietf/doc/tests_review.py
+++ b/ietf/doc/tests_review.py
@@ -822,7 +822,7 @@ def test_complete_review_upload_content(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, assignment.review_request.team.list_email)
-        for author in assignment.review_request.doc.authors():
+        for author in assignment.review_request.doc.author_persons():
             self.assertContains(r, author.formatted_email())
 
         # faulty post
diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index 2bd9a3d314..0715471551 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -13,7 +13,7 @@
 from dataclasses import dataclass
 from hashlib import sha384
 from pathlib import Path
-from typing import Iterator, Optional, Union
+from typing import Iterator, Optional, Union, Iterable
 from zoneinfo import ZoneInfo
 
 from django.conf import settings
@@ -33,7 +33,14 @@
 from ietf.community.models import CommunityList
 from ietf.community.utils import docs_tracked_by_community_list
 
-from ietf.doc.models import Document, DocHistory, State, DocumentAuthor, DocHistoryAuthor
+from ietf.doc.models import (
+    DocHistory,
+    DocHistoryAuthor,
+    Document,
+    DocumentAuthor,
+    RfcAuthor,
+    State, EditedRfcAuthorsDocEvent,
+)
 from ietf.doc.models import RelatedDocument, RelatedDocHistory, BallotType, DocReminder
 from ietf.doc.models import DocEvent, ConsensusDocEvent, BallotDocEvent, IRSGBallotDocEvent, NewRevisionDocEvent, StateDocEvent
 from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent, BallotPositionDocEvent
@@ -534,7 +541,7 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None,
             doc.action_holders.clear()
         if tags.removed("need-rev"):
             # Removed the 'need-rev' tag - drop authors from the action holders list
-            DocumentActionHolder.objects.filter(document=doc, person__in=doc.authors()).delete()
+            DocumentActionHolder.objects.filter(document=doc, person__in=doc.author_persons()).delete()
         elif tags.added("need-rev"):
             # Remove the AD if we're asking for a new revision
             DocumentActionHolder.objects.filter(document=doc, person=doc.ad).delete()
@@ -549,7 +556,7 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None,
                 doc.action_holders.add(doc.ad)
         # Authors get the action if a revision is needed
         if tags.added("need-rev"):
-            for auth in doc.authors():
+            for auth in doc.author_persons():
                 doc.action_holders.add(auth)
 
         # Now create an event if we changed the set
@@ -561,6 +568,40 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None,
         )
 
 
+def _change_field_and_describe(
+    author: DocumentAuthor | RfcAuthor,
+    field: str,
+    newval,
+    field_display_name: str | None = None,
+):
+    # make the change
+    oldval = getattr(author, field)
+    setattr(author, field, newval)
+
+    was_empty = oldval is None or len(str(oldval)) == 0
+    now_empty = newval is None or len(str(newval)) == 0
+
+    # describe the change
+    if oldval == newval:
+        return None
+    else:
+        if field_display_name is None:
+            field_display_name = field
+
+        if was_empty and not now_empty:
+            return 'set {field} to "{new}"'.format(
+                field=field_display_name, new=newval
+            )
+        elif now_empty and not was_empty:
+            return 'cleared {field} (was "{old}")'.format(
+                field=field_display_name, old=oldval
+            )
+        else:
+            return 'changed {field} from "{old}" to "{new}"'.format(
+                field=field_display_name, old=oldval, new=newval
+            )
+
+
 def update_documentauthors(doc, new_docauthors, by=None, basis=None):
     """Update the list of authors for a document
 
@@ -573,27 +614,6 @@ def update_documentauthors(doc, new_docauthors, by=None, basis=None):
     used. These objects will not be saved, their attributes will be used to create new
     DocumentAuthor instances. (The document and order fields will be ignored.)
     """
-    def _change_field_and_describe(auth, field, newval):
-        # make the change
-        oldval = getattr(auth, field)
-        setattr(auth, field, newval)
-        
-        was_empty = oldval is None or len(str(oldval)) == 0
-        now_empty = newval is None or len(str(newval)) == 0
-        
-        # describe the change
-        if oldval == newval:
-            return None
-        else:
-            if was_empty and not now_empty:
-                return 'set {field} to "{new}"'.format(field=field, new=newval)
-            elif now_empty and not was_empty:
-                return 'cleared {field} (was "{old}")'.format(field=field, old=oldval)
-            else:
-                return 'changed {field} from "{old}" to "{new}"'.format(
-                    field=field, old=oldval, new=newval
-                )
-
     persons = []
     changes = []  # list of change descriptions
 
@@ -637,6 +657,111 @@ def _change_field_and_describe(auth, field, newval):
         ) for change in changes
     ] 
 
+
+def update_rfcauthors(
+    rfc: Document, new_rfcauthors: Iterable[RfcAuthor], by: Person | None = None
+) -> Iterable[EditedRfcAuthorsDocEvent]:
+    def _find_matching_author(
+        author_to_match: RfcAuthor, existing_authors: Iterable[RfcAuthor]
+    ) -> RfcAuthor | None:
+        """Helper to find a matching existing author"""
+        if author_to_match.person_id is not None:
+            for candidate in existing_authors:
+                if candidate.person_id == author_to_match.person_id:
+                    return candidate
+            return None  # no match
+        # author does not have a person, match on titlepage name
+        for candidate in existing_authors:
+            if candidate.titlepage_name == author_to_match.titlepage_name:
+                return candidate
+        return None  # no match
+
+    def _rfcauthor_from_documentauthor(docauthor: DocumentAuthor) -> RfcAuthor:
+        """Helper to create an equivalent RfcAuthor from a DocumentAuthor"""
+        return RfcAuthor(
+            document_id=docauthor.document_id,
+            titlepage_name=docauthor.person.plain_name(),  # closest thing we have
+            is_editor=False,
+            person_id=docauthor.person_id,
+            affiliation=docauthor.affiliation,
+            country=docauthor.country,
+            order=docauthor.order,
+        )
+
+    # Is this the first time this document is getting an RfcAuthor? If so, the
+    # updates will need to account for the model change.
+    converting_from_docauthors = not rfc.rfcauthor_set.exists()
+    
+    if converting_from_docauthors:
+        original_authors = [
+            _rfcauthor_from_documentauthor(da) for da in rfc.documentauthor_set.all()
+        ]
+    else:
+        original_authors = list(rfc.rfcauthor_set.all())
+
+    authors_to_commit = []
+    changes = []
+    for order, new_author in enumerate(new_rfcauthors):
+        matching_author = _find_matching_author(new_author, original_authors)
+        if matching_author is not None:
+            # Update existing matching author using new_author data
+            authors_to_commit.append(matching_author)
+            original_authors.remove(matching_author)  # avoid reuse
+            # Describe changes to this author
+            author_changes = []
+            # Update fields other than order
+            for field in ["titlepage_name", "is_editor", "affiliation", "country"]:
+                author_changes.append(
+                    _change_field_and_describe(
+                        matching_author,
+                        field,
+                        getattr(new_author, field),
+                        # List titlepage_name as "name" in logs
+                        "name" if field == "titlepage_name" else field,
+                    )
+                )
+            # Update order
+            author_changes.append(
+                _change_field_and_describe(matching_author, "order", order + 1)
+            )
+            matching_author.save()
+            author_change_summary = ", ".join(
+                [ch for ch in author_changes if ch is not None]
+            )
+            if len(author_change_summary) > 0:
+                changes.append(
+                    'Changed author "{name}": {summary}'.format(
+                        name=matching_author.titlepage_name,
+                        summary=author_change_summary,
+                    )
+                )
+        else:
+            # No author matched, so update the new_author and use that
+            new_author.document = rfc
+            new_author.order = order + 1
+            new_author.save()
+            changes.append(f'Added "{new_author.titlepage_name}" as author')
+    # Any authors left in original_authors are no longer in the list, so remove them
+    for removed_author in original_authors:
+        # Skip actual removal of old authors if we are converting from the
+        # DocumentAuthor models - the original_authors were just stand-ins anyway.
+        if not converting_from_docauthors:
+            removed_author.delete()
+        changes.append(f'Removed "{removed_author.titlepage_name}" as author')
+    # Create DocEvents, but leave it up to caller to save
+    if by is None:
+        by = Person.objects.get(name="(System)")
+    return [
+        EditedRfcAuthorsDocEvent(
+            type="edited_authors",
+            by=by,
+            doc=rfc,
+            desc=change,
+        )
+        for change in changes
+    ]
+
+
 def update_reminder(doc, reminder_type_slug, event, due_date):
     reminder_type = DocReminderTypeName.objects.get(slug=reminder_type_slug)
 
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 5564904504..0578da1b77 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -1653,11 +1653,18 @@ def extract_name(s):
     data["state"] = extract_name(doc.get_state())
     data["intended_std_level"] = extract_name(doc.intended_std_level)
     data["std_level"] = extract_name(doc.std_level)
+    author_qs = (
+        doc.rfcauthor_set
+        if doc.type_id == "rfc" and doc.rfcauthor_set.exists()
+        else doc.documentauthor_set
+    ).select_related("person", "email").order_by("order")
     data["authors"] = [
-        dict(name=author.person.name,
-             email=author.email.address if author.email else None,
-             affiliation=author.affiliation)
-        for author in doc.documentauthor_set.all().select_related("person", "email").order_by("order")
+        {
+            "name": author.titlepage_name if hasattr(author, "titlepage_name") else author.person.name,
+            "email": author.email.address if author.email else None,
+            "affiliation": author.affiliation,
+        }
+        for author in author_qs
     ]
     data["shepherd"] = doc.shepherd.formatted_email() if doc.shepherd else None
     data["ad"] = doc.ad.role_email("ad").formatted_email() if doc.ad else None
@@ -1941,9 +1948,9 @@ def edit_action_holders(request, name):
     role_ids = dict()  # maps role slug to list of Person IDs (assumed numeric in the JavaScript)
     extra_prefetch = []  # list of Person objects to prefetch for select2 field
 
-    if len(doc.authors()) > 0:
+    authors = doc.author_persons()
+    if len(authors) > 0:
         doc_role_labels.append(dict(slug='authors', label='Authors'))
-        authors = doc.authors()
         role_ids['authors'] = [p.pk for p in authors]
         extra_prefetch += authors
 
diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py
index 3b67061b05..4232d77f6c 100644
--- a/ietf/doc/views_search.py
+++ b/ietf/doc/views_search.py
@@ -219,7 +219,7 @@ def retrieve_search_results(form, all_types=False):
             queries.extend([Q(targets_related__source__name__icontains=look_for, targets_related__relationship_id="became_rfc")])
 
         combined_query = reduce(operator.or_, queries)
-        docs = docs.filter(combined_query).distinct()
+        docs = docs.filter(combined_query)
 
     # rfc/active/old check buttons
     allowed_draft_states = []
@@ -229,20 +229,23 @@ def retrieve_search_results(form, all_types=False):
         allowed_draft_states.extend(['repl', 'expired', 'auth-rm', 'ietf-rm'])
 
     docs = docs.filter(Q(states__slug__in=allowed_draft_states) |
-                       ~Q(type__slug='draft')).distinct()
+                       ~Q(type__slug='draft'))
 
     # radio choices
     by = query["by"]
     if by == "author":
         docs = docs.filter(
             Q(documentauthor__person__alias__name__icontains=query["author"]) |
-            Q(documentauthor__person__email__address__icontains=query["author"])
+            Q(documentauthor__person__email__address__icontains=query["author"]) |
+            Q(rfcauthor__person__alias__name__icontains=query["author"]) |
+            Q(rfcauthor__person__email__address__icontains=query["author"]) |
+            Q(rfcauthor__titlepage_name__icontains=query["author"])
         )
     elif by == "group":
         docs = docs.filter(group__acronym__iexact=query["group"])
     elif by == "area":
         docs = docs.filter(Q(group__type="wg", group__parent=query["area"]) |
-                           Q(group=query["area"])).distinct()
+                           Q(group=query["area"]))
     elif by == "ad":
         docs = docs.filter(ad=query["ad"])
     elif by == "state":
@@ -255,6 +258,8 @@ def retrieve_search_results(form, all_types=False):
     elif by == "stream":
         docs = docs.filter(stream=query["stream"])
 
+    docs=docs.distinct()
+
     return docs
 
 
diff --git a/ietf/group/models.py b/ietf/group/models.py
index 2d5e7c4e6f..a7e3c6616e 100644
--- a/ietf/group/models.py
+++ b/ietf/group/models.py
@@ -111,6 +111,9 @@ def active_wgs(self):
     def closed_wgs(self):
         return self.wgs().exclude(state__in=Group.ACTIVE_STATE_IDS)
 
+    def areas(self):
+        return self.get_queryset().filter(type="area")
+
     def with_meetings(self):
         return self.get_queryset().filter(type__features__has_meetings=True)
 
diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py
new file mode 100644
index 0000000000..08e6bba81a
--- /dev/null
+++ b/ietf/group/serializers.py
@@ -0,0 +1,11 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+"""django-rest-framework serializers"""
+from rest_framework import serializers
+
+from .models import Group
+
+
+class GroupSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Group
+        fields = ["acronym", "name", "type"]
diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py
index 1f634278be..0df667fbd2 100644
--- a/ietf/ietfauth/utils.py
+++ b/ietf/ietfauth/utils.py
@@ -287,7 +287,7 @@ def is_individual_draft_author(user, doc):
     if not hasattr(user, 'person'):
         return False
 
-    if user.person in doc.authors():
+    if user.person in doc.author_persons():
         return True
 
     return False
diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py
index 665c99dc43..0a43ff2c27 100644
--- a/ietf/ipr/views.py
+++ b/ietf/ipr/views.py
@@ -81,7 +81,8 @@ def get_document_emails(ipr):
 
         addrs = gather_address_lists('ipr_posted_on_doc',doc=doc).as_strings(compact=False)
 
-        author_names = ', '.join(a.person.name for a in doc.documentauthor_set.select_related("person"))
+        # Get a list of author names for the salutation in the body of the email
+        author_names = ', '.join(doc.author_names())
     
         context = dict(
             settings=settings,
diff --git a/ietf/name/serializers.py b/ietf/name/serializers.py
new file mode 100644
index 0000000000..a764f56051
--- /dev/null
+++ b/ietf/name/serializers.py
@@ -0,0 +1,11 @@
+# Copyright The IETF Trust 2024, All Rights Reserved
+"""django-rest-framework serializers"""
+from rest_framework import serializers
+
+from .models import StreamName
+
+
+class StreamNameSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = StreamName
+        fields = ["slug", "name", "desc"]
diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py
index dcdb9ef836..b6e8c57da7 100644
--- a/ietf/nomcom/tests.py
+++ b/ietf/nomcom/tests.py
@@ -1,5 +1,4 @@
-# Copyright The IETF Trust 2012-2023, All Rights Reserved
-# -*- coding: utf-8 -*-
+# Copyright The IETF Trust 2012-2025, All Rights Reserved
 
 
 import datetime
@@ -27,8 +26,14 @@
 from ietf.api.views import EmailIngestionError
 from ietf.dbtemplate.factories import DBTemplateFactory
 from ietf.dbtemplate.models import DBTemplate
-from ietf.doc.factories import DocEventFactory, WgDocumentAuthorFactory, \
-                               NewRevisionDocEventFactory, DocumentAuthorFactory
+from ietf.doc.factories import (
+    DocEventFactory,
+    WgDocumentAuthorFactory,
+    NewRevisionDocEventFactory,
+    DocumentAuthorFactory,
+    RfcAuthorFactory,
+    WgDraftFactory, WgRfcFactory,
+)
 from ietf.group.factories import GroupFactory, GroupHistoryFactory, RoleFactory, RoleHistoryFactory
 from ietf.group.models import Group, Role
 from ietf.meeting.factories import MeetingFactory, AttendedFactory, RegistrationFactory
@@ -45,10 +50,20 @@
                                   nomcom_kwargs_for_year, provide_private_key_to_test_client, \
                                   key
 from ietf.nomcom.tasks import send_nomcom_reminders_task
-from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, \
-                              get_hash_nominee_position, is_eligible, list_eligible, \
-                              get_eligibility_date, suggest_affiliation, ingest_feedback_email, \
-                              decorate_volunteers_with_qualifications, send_reminders, _is_time_to_send_reminder
+from ietf.nomcom.utils import (
+    get_nomcom_by_year,
+    make_nomineeposition,
+    get_hash_nominee_position,
+    is_eligible,
+    list_eligible,
+    get_eligibility_date,
+    suggest_affiliation,
+    ingest_feedback_email,
+    decorate_volunteers_with_qualifications,
+    send_reminders,
+    _is_time_to_send_reminder,
+    get_qualified_author_queryset,
+)
 from ietf.person.factories import PersonFactory, EmailFactory
 from ietf.person.models import Email, Person
 from ietf.utils.mail import outbox, empty_outbox, get_payload_text
@@ -2440,6 +2455,86 @@ def test_get_eligibility_date(self):
         NomComFactory(group__acronym=f'nomcom{this_year}', first_call_for_volunteers=datetime.date(this_year,5,6))
         self.assertEqual(get_eligibility_date(),datetime.date(this_year,5,6))
 
+    def test_get_qualified_author_queryset(self):
+        """get_qualified_author_queryset implements the eligiblity rules correctly
+
+        This is not an exhaustive test of corner cases. Overlaps considerably with
+        rfc8989EligibilityTests.test_elig_by_author().
+        """
+        people = PersonFactory.create_batch(2)
+        extra_person = PersonFactory()
+        base_qs = Person.objects.filter(pk__in=[person.pk for person in people])
+        now = datetime.datetime.now(tz=datetime.UTC)
+        one_year = datetime.timedelta(days=365)
+
+        # Authors with no qualifying drafts
+        self.assertCountEqual(
+            get_qualified_author_queryset(base_qs, now - 5 * one_year, now), []
+        )
+
+        # Authors with one qualifying draft
+        approved_draft = WgDraftFactory(authors=people, states=[("draft", "active")])
+        DocEventFactory(
+            type="iesg_approved",
+            doc=approved_draft,
+            time=now - 4 * one_year,
+        )
+        self.assertCountEqual(
+            get_qualified_author_queryset(base_qs, now - 5 * one_year, now), []
+        )
+
+        # Create a draft that was published into an RFC. Give it an extra author who
+        # should not be eligible.
+        published_draft = WgDraftFactory(authors=people, states=[("draft", "rfc")])
+        DocEventFactory(
+            type="iesg_approved",
+            doc=published_draft,
+            time=now - 5.5 * one_year,  # < 6 years ago
+        )
+        rfc = WgRfcFactory(
+            authors=people + [extra_person],
+            group=published_draft.group,
+        )
+        DocEventFactory(
+            type="published_rfc",
+            doc=rfc,
+            time=now - 0.5 * one_year,  # < 1 year ago
+        )
+        # Period 6 years ago to 1 year ago - authors are eligible due to the
+        # iesg-approved draft in this window
+        self.assertCountEqual(
+            get_qualified_author_queryset(base_qs, now - 6 * one_year, now - one_year),
+            people,
+        )
+
+        # Period 5 years ago to now - authors are eligible due to the RFC publication
+        self.assertCountEqual(
+            get_qualified_author_queryset(base_qs, now - 5 * one_year, now),
+            people,
+        )
+        
+        # Use the extra_person to check that a single doc can't count both as an
+        # RFC _and_ an approved draft. Use an eligibility interval that includes both
+        # the approval and the RFC publication
+        self.assertCountEqual(
+            get_qualified_author_queryset(base_qs, now - 6 * one_year, now),
+            people,  # does not include extra_person!
+        )
+
+        # Now add an RfcAuthor for only one of the two authors to the RFC. This should
+        # remove the other author from the eligibility list because the DocumentAuthor
+        # records are no longer used.
+        RfcAuthorFactory(
+            document=rfc,
+            person=people[0],
+            titlepage_name="P. Zero",
+            email=people[0].email_set.first(),
+        )
+        self.assertCountEqual(
+            get_qualified_author_queryset(base_qs, now - 5 * one_year, now),
+            [people[0]],
+        )
+
 
 class rfc8713EligibilityTests(TestCase):
 
@@ -2724,33 +2819,41 @@ def test_elig_by_author(self):
             ineligible = set()
 
             p = PersonFactory()
-            ineligible.add(p)
-
+            ineligible.add(p)  # no RFCs or iesg-approved drafts
             p = PersonFactory()
-            da = WgDocumentAuthorFactory(person=p)
-            DocEventFactory(type='published_rfc',doc=da.document,time=middle_date)
-            ineligible.add(p)
+            doc = WgRfcFactory(authors=[p])
+            DocEventFactory(type='published_rfc', doc=doc, time=middle_date)
+            ineligible.add(p)  # only one RFC
 
             p = PersonFactory()
-            da = WgDocumentAuthorFactory(person=p)
+            da = WgDocumentAuthorFactory(
+                person=p,
+                document__states=[("draft", "active"), ("draft-rfceditor", "ref")],
+            )
             DocEventFactory(type='iesg_approved',doc=da.document,time=last_date)
-            da = WgDocumentAuthorFactory(person=p)
-            DocEventFactory(type='published_rfc',doc=da.document,time=first_date)
-            eligible.add(p)
+            doc = WgRfcFactory(authors=[p])
+            DocEventFactory(type='published_rfc', doc=doc, time=first_date)
+            eligible.add(p)  # one RFC and one iesg-approved draft
 
             p = PersonFactory()
-            da = WgDocumentAuthorFactory(person=p)
+            da = WgDocumentAuthorFactory(
+                person=p,
+                document__states=[("draft", "active"), ("draft-rfceditor", "ref")],
+            )
             DocEventFactory(type='iesg_approved',doc=da.document,time=middle_date)
-            da = WgDocumentAuthorFactory(person=p)
-            DocEventFactory(type='published_rfc',doc=da.document,time=day_before_first_date)
-            ineligible.add(p)
+            doc = WgRfcFactory(authors=[p])
+            DocEventFactory(type='published_rfc', doc=doc, time=day_before_first_date)
+            ineligible.add(p)  # RFC is out of the eligibility window
 
             p = PersonFactory()
-            da = WgDocumentAuthorFactory(person=p)
+            da = WgDocumentAuthorFactory(
+                person=p,
+                document__states=[("draft", "active"), ("draft-rfceditor", "ref")],
+            )
             DocEventFactory(type='iesg_approved',doc=da.document,time=day_after_last_date)
-            da = WgDocumentAuthorFactory(person=p)
-            DocEventFactory(type='published_rfc',doc=da.document,time=middle_date)
-            ineligible.add(p)
+            doc = WgRfcFactory(authors=[p])
+            DocEventFactory(type='published_rfc', doc=doc, time=middle_date)
+            ineligible.add(p)  # iesg approval is outside the eligibility window
 
             for person in eligible:
                 self.assertTrue(is_eligible(person,nomcom))
@@ -2878,15 +2981,38 @@ def test_volunteer(self):
 
     def test_suggest_affiliation(self):
         person = PersonFactory()
-        self.assertEqual(suggest_affiliation(person), '')
-        da = DocumentAuthorFactory(person=person,affiliation='auth_affil')
+        self.assertEqual(suggest_affiliation(person), "")
+        rfc_da = DocumentAuthorFactory(
+            person=person,
+            document__type_id="rfc",
+            affiliation="",
+        )
+        rfc = rfc_da.document
+        DocEventFactory(doc=rfc, type="published_rfc")
+        self.assertEqual(suggest_affiliation(person), "")
+
+        rfc_da.affiliation = "rfc_da_affil"
+        rfc_da.save()
+        self.assertEqual(suggest_affiliation(person), "rfc_da_affil")
+
+        rfc_ra = RfcAuthorFactory(person=person, document=rfc, affiliation="")
+        self.assertEqual(suggest_affiliation(person), "")
+
+        rfc_ra.affiliation = "rfc_ra_affil"
+        rfc_ra.save()
+        self.assertEqual(suggest_affiliation(person), "rfc_ra_affil")
+
+        da = DocumentAuthorFactory(person=person, affiliation="auth_affil")
         NewRevisionDocEventFactory(doc=da.document)
-        self.assertEqual(suggest_affiliation(person), 'auth_affil')
+        self.assertEqual(suggest_affiliation(person), "auth_affil")
+
         nc = NomComFactory()
-        nc.volunteer_set.create(person=person,affiliation='volunteer_affil')
-        self.assertEqual(suggest_affiliation(person), 'volunteer_affil')
-        RegistrationFactory(person=person, affiliation='meeting_affil')
-        self.assertEqual(suggest_affiliation(person), 'meeting_affil')
+        nc.volunteer_set.create(person=person, affiliation="volunteer_affil")
+        self.assertEqual(suggest_affiliation(person), "volunteer_affil")
+
+        RegistrationFactory(person=person, affiliation="meeting_affil")
+        self.assertEqual(suggest_affiliation(person), "meeting_affil")
+
 
 class VolunteerDecoratorUnitTests(TestCase):
     def test_decorate_volunteers_with_qualifications(self):
@@ -2922,10 +3048,10 @@ def test_decorate_volunteers_with_qualifications(self):
 
         author_person = PersonFactory()
         for i in range(2):
-            da = WgDocumentAuthorFactory(person=author_person)
+            doc = WgRfcFactory(authors=[author_person])
             DocEventFactory(
                 type='published_rfc',
-                doc=da.document,
+                doc=doc,
                 time=datetime.datetime(
                     elig_date.year - 3,
                     elig_date.month,
diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py
index dd651c2941..a2ab680df6 100644
--- a/ietf/nomcom/utils.py
+++ b/ietf/nomcom/utils.py
@@ -18,7 +18,7 @@
 from email.utils import parseaddr
 from textwrap import dedent
 
-from django.db.models import Q, Count
+from django.db.models import Q, Count, F, QuerySet
 from django.conf import settings
 from django.contrib.sites.models import Site
 from django.core.exceptions import ObjectDoesNotExist
@@ -27,7 +27,7 @@
 from django.shortcuts import get_object_or_404
 
 from ietf.dbtemplate.models import DBTemplate
-from ietf.doc.models import DocEvent, NewRevisionDocEvent
+from ietf.doc.models import DocEvent, NewRevisionDocEvent, Document
 from ietf.group.models import Group, Role
 from ietf.person.models import Email, Person
 from ietf.mailtrigger.utils import gather_address_lists
@@ -576,6 +576,70 @@ def get_8989_eligibility_querysets(date, base_qs):
 def get_9389_eligibility_querysets(date, base_qs):
     return get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable=three_of_five_eligible_9389)
 
+
+def get_qualified_author_queryset(
+    base_qs: QuerySet[Person],
+    eligibility_period_start: datetime.datetime,
+    eligibility_period_end: datetime.datetime,
+):
+    """Filter a Person queryset, keeping those qualified by RFC 8989's author path
+    
+    The author path is defined by "path 3" in section 4 of RFC 8989. It qualifies
+    a person who has been a front-page listed author or editor of at least two IETF-
+    stream RFCs within the last five years. An I-D in the RFC Editor queue that was
+    approved by the IESG is treated as an RFC, using the date of entry to the RFC
+    Editor queue as the date for qualification.
+    
+    This method does not strictly enforce "in the RFC Editor queue" for IESG-approved
+    drafts when computing eligibility. In the overwhelming majority of cases, an IESG-
+    approved draft immediately enters the queue and goes on to be published, so this
+    simplification makes the calculation much easier and virtually never affects
+    eligibility.  
+    
+    Arguments eligibility_period_start and eligibility_period_end are datetimes that
+    mark the start and end of the eligibility period. These should be five years apart.
+    """
+    # First, get the RFCs using publication date
+    qualifying_rfc_pub_events = DocEvent.objects.filter(
+        type='published_rfc',
+        time__gte=eligibility_period_start,
+        time__lte=eligibility_period_end,
+    )
+    qualifying_rfcs = Document.objects.filter(
+        type_id="rfc",
+        docevent__in=qualifying_rfc_pub_events
+    ).annotate(
+        rfcauthor_count=Count("rfcauthor")
+    )
+    rfcs_with_rfcauthors = qualifying_rfcs.filter(rfcauthor_count__gt=0).distinct()
+    rfcs_without_rfcauthors = qualifying_rfcs.filter(rfcauthor_count=0).distinct()
+
+    # Second, get the IESG-approved I-Ds excluding any we're already counting as rfcs
+    qualifying_approval_events = DocEvent.objects.filter(
+        type='iesg_approved',
+        time__gte=eligibility_period_start,
+        time__lte=eligibility_period_end,
+    )
+    qualifying_drafts = Document.objects.filter(
+        type_id="draft",
+        docevent__in=qualifying_approval_events,
+    ).exclude(
+        relateddocument__relationship_id="became_rfc",
+        relateddocument__target__in=qualifying_rfcs,
+    ).distinct()
+
+    return base_qs.filter(
+        Q(documentauthor__document__in=qualifying_drafts)
+        | Q(rfcauthor__document__in=rfcs_with_rfcauthors)
+        | Q(documentauthor__document__in=rfcs_without_rfcauthors)
+    ).annotate(
+        document_author_count=Count('documentauthor'),
+        rfc_author_count=Count("rfcauthor")
+    ).annotate(
+        authorship_count=F("document_author_count") + F("rfc_author_count")
+    ).filter(authorship_count__gte=2)
+
+
 def get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable):
     if not base_qs:
         base_qs = Person.objects.all()
@@ -608,14 +672,7 @@ def get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable):
          )
     ).distinct()
 
-    rfc_pks = set(DocEvent.objects.filter(type='published_rfc', time__gte=five_years_ago, time__lte=date_as_dt).values_list('doc__pk', flat=True))
-    iesgappr_pks = set(DocEvent.objects.filter(type='iesg_approved', time__gte=five_years_ago, time__lte=date_as_dt).values_list('doc__pk',flat=True))
-    qualifying_pks = rfc_pks.union(iesgappr_pks.difference(rfc_pks))
-    author_qs = base_qs.filter(
-            documentauthor__document__pk__in=qualifying_pks
-        ).annotate(
-            document_author_count = Count('documentauthor')
-        ).filter(document_author_count__gte=2)
+    author_qs = get_qualified_author_queryset(base_qs, five_years_ago, date_as_dt)
     return three_of_five_qs, officer_qs, author_qs
 
 def list_eligible_8989(date, base_qs=None):
@@ -691,18 +748,42 @@ def three_of_five_eligible_9389(previous_five, queryset=None):
             counts[id] += 1
     return queryset.filter(pk__in=[id for id, count in counts.items() if count >= 3])
 
-def suggest_affiliation(person):
+def suggest_affiliation(person) -> str:
+    """Heuristically suggest a current affiliation for a Person"""
     recent_meeting = person.registration_set.order_by('-meeting__date').first()
-    affiliation = recent_meeting.affiliation if recent_meeting else ''
-    if not affiliation:
-        recent_volunteer = person.volunteer_set.order_by('-nomcom__group__acronym').first()
-        if recent_volunteer:
-            affiliation = recent_volunteer.affiliation 
-    if not affiliation:
-        recent_draft_revision =  NewRevisionDocEvent.objects.filter(doc__type_id='draft',doc__documentauthor__person=person).order_by('-time').first()
-        if recent_draft_revision:
-            affiliation = recent_draft_revision.doc.documentauthor_set.filter(person=person).first().affiliation
-    return affiliation
+    if recent_meeting and recent_meeting.affiliation:
+        return recent_meeting.affiliation
+
+    recent_volunteer = person.volunteer_set.order_by('-nomcom__group__acronym').first()
+    if recent_volunteer and recent_volunteer.affiliation:
+        return recent_volunteer.affiliation 
+
+    recent_draft_revision =  NewRevisionDocEvent.objects.filter(
+        doc__type_id="draft",
+        doc__documentauthor__person=person,
+    ).order_by("-time").first()
+    if recent_draft_revision:
+        draft_author = recent_draft_revision.doc.documentauthor_set.filter(
+            person=person
+        ).first()
+        if draft_author and draft_author.affiliation:
+            return draft_author.affiliation
+    
+    recent_rfc_publication = DocEvent.objects.filter(
+        Q(doc__documentauthor__person=person) | Q(doc__rfcauthor__person=person),
+        doc__type_id="rfc",
+        type="published_rfc",
+    ).order_by("-time").first()
+    if recent_rfc_publication:
+        rfc = recent_rfc_publication.doc
+        if rfc.rfcauthor_set.exists():
+            rfc_author = rfc.rfcauthor_set.filter(person=person).first()
+        else:
+            rfc_author = rfc.documentauthor_set.filter(person=person).first()
+        if rfc_author and rfc_author.affiliation:
+            return rfc_author.affiliation
+    return ""
+
 
 def extract_volunteers(year):
     nomcom = get_nomcom_by_year(year)
diff --git a/ietf/person/models.py b/ietf/person/models.py
index 03cf0c87fb..3ab89289a6 100644
--- a/ietf/person/models.py
+++ b/ietf/person/models.py
@@ -87,7 +87,7 @@ def short(self):
         else:
             prefix, first, middle, last, suffix = self.ascii_parts()
             return (first and first[0]+"." or "")+(middle or "")+" "+last+(suffix and " "+suffix or "")
-    def plain_name(self):
+    def plain_name(self) -> str:
         if not hasattr(self, '_cached_plain_name'):
             if self.plain:
                 self._cached_plain_name = self.plain
@@ -203,7 +203,10 @@ def has_drafts(self):
 
     def rfcs(self):
         from ietf.doc.models import Document
-        rfcs = list(Document.objects.filter(documentauthor__person=self, type='rfc'))
+        # When RfcAuthors are populated, this may over-return if an author is dropped
+        # from the author list between the final draft and the published RFC. Should
+        # ignore DocumentAuthors when an RfcAuthor exists for a draft.
+        rfcs = list(Document.objects.filter(type="rfc").filter(models.Q(documentauthor__person=self)|models.Q(rfcauthor__person=self)).distinct())
         rfcs.sort(key=lambda d: d.name )
         return rfcs
 
@@ -266,11 +269,16 @@ def available_api_endpoints(self):
     def cdn_photo_url(self, size=80):
         if self.photo:
             if settings.SERVE_CDN_PHOTOS:
+                if settings.SERVER_MODE != "production":
+                    original_media_dir = settings.MEDIA_URL
+                    settings.MEDIA_URL = "https://www.ietf.org/lib/dt/media/"
                 source_url = self.photo.url
                 if source_url.startswith(settings.IETF_HOST_URL):
                     source_url = source_url[len(settings.IETF_HOST_URL):]
                 elif source_url.startswith('/'):
                     source_url = source_url[1:]
+                if settings.SERVER_MODE != "production":
+                    settings.MEDIA_URL = original_media_dir
                 return f'{settings.IETF_HOST_URL}cdn-cgi/image/fit=scale-down,width={size},height={size}/{source_url}'
             else:
                 datatracker_photo_path = urlreverse('ietf.person.views.photo', kwargs={'email_or_name': self.email()})
diff --git a/ietf/secr/telechat/tests.py b/ietf/secr/telechat/tests.py
index fa26d33a5c..91ccde2187 100644
--- a/ietf/secr/telechat/tests.py
+++ b/ietf/secr/telechat/tests.py
@@ -256,7 +256,7 @@ def test_doc_detail_post_update_state_action_holder_automation(self):
         self.assertEqual(response.status_code,302)
         draft = Document.objects.get(name=draft.name)
         self.assertEqual(draft.get_state('draft-iesg').slug,'defer')
-        self.assertCountEqual(draft.action_holders.all(), [draft.ad] + draft.authors())
+        self.assertCountEqual(draft.action_holders.all(), [draft.ad] + draft.author_persons())
         self.assertEqual(draft.docevent_set.filter(type='changed_action_holders').count(), 1)
 
         # Removing need-rev should remove authors
@@ -273,7 +273,7 @@ def test_doc_detail_post_update_state_action_holder_automation(self):
 
         # Setting to approved should remove all action holders
         # noinspection DjangoOrm
-        draft.action_holders.add(*(draft.authors()))  # add() with through model ok in Django 2.2+
+        draft.action_holders.add(*(draft.author_persons()))  # add() with through model ok in Django 2.2+
         response = self.client.post(url,{
             'submit': 'update_state',
             'state': State.objects.get(type_id='draft-iesg', slug='approved').pk,
diff --git a/ietf/settings.py b/ietf/settings.py
index 05eab0f12f..fedd313ca0 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -22,6 +22,7 @@
 warnings.filterwarnings("ignore", message="The django.utils.timezone.utc alias is deprecated.", module="oidc_provider")
 warnings.filterwarnings("ignore", message="The django.utils.datetime_safe module is deprecated.", module="tastypie")
 warnings.filterwarnings("ignore", message="The USE_DEPRECATED_PYTZ setting,")  # https://github.com/ietf-tools/datatracker/issues/5635
+warnings.filterwarnings("ignore", message="The is_dst argument to make_aware\\(\\)")  # caused by django-filters when USE_DEPRECATED_PYTZ is true 
 warnings.filterwarnings("ignore", message="The USE_L10N setting is deprecated.")  # https://github.com/ietf-tools/datatracker/issues/5648
 warnings.filterwarnings("ignore", message="django.contrib.auth.hashers.CryptPasswordHasher is deprecated.")  # https://github.com/ietf-tools/datatracker/issues/5663
 
@@ -502,6 +503,7 @@ def skip_unreadable_post(record):
     'django_celery_results',
     'corsheaders',
     'django_markup',
+    'django_filters',
     'oidc_provider',
     'drf_spectacular',
     'drf_standardized_errors',
diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index 216fc7de6b..400d0d8c7d 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -595,7 +595,7 @@ def submit_existing(self, formats, change_authors=True, group_type='wg', stream_
         TestBlobstoreManager().emptyTestBlobstores()
 
         def _assert_authors_are_action_holders(draft, expect=True):
-            for author in draft.authors():
+            for author in draft.author_persons():
                 if expect:
                     self.assertIn(author, draft.action_holders.all())
                 else:
diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index a0c7dd8511..9a7c358a6d 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -1268,7 +1268,7 @@ def process_submission_text(filename, revision):
     if title:
         title = _normalize_title(title)
 
-    # Translation taable drops \r, \n, <, >.
+    # Translation table drops \r, \n, <, >.
     trans_table = str.maketrans("", "", "\r\n<>")
     authors = [
         {
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index b3234a87e2..cdcdeb5989 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -468,14 +468,18 @@ def update_docs_from_rfc_index(
             doc.set_state(rfc_published_state)
             if draft:
                 doc.formal_languages.set(draft.formal_languages.all())
-                for author in draft.documentauthor_set.all():
+                # Create authors based on the last draft in the datatracker. This
+                # path will go away when we publish via the modernized RPC workflow
+                # but until then, these are the only data we have for authors that
+                # are easily connected to Person records.
+                for documentauthor in draft.documentauthor_set.all():
                     # Copy the author but point at the new doc.
                     # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances
-                    author.pk = None
-                    author.id = None
-                    author._state.adding = True
-                    author.document = doc
-                    author.save()
+                    documentauthor.pk = None
+                    documentauthor.id = None
+                    documentauthor._state.adding = True
+                    documentauthor.document = doc
+                    documentauthor.save()
 
         if draft:
             draft_events = []
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index 3432f6214a..888920ae9d 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -446,7 +446,7 @@ def test_rfc_index(self):
 
         rfc_doc = Document.objects.filter(rfc_number=1234, type_id="rfc").first()
         self.assertIsNotNone(rfc_doc, "RFC document should have been created")
-        self.assertEqual(rfc_doc.authors(), draft_doc.authors())
+        self.assertEqual(rfc_doc.author_persons_or_names(), draft_doc.author_persons_or_names())
         rfc_events = rfc_doc.docevent_set.all()
         self.assertEqual(len(rfc_events), 8)
         expected_events = [
diff --git a/ietf/templates/doc/document_info.html b/ietf/templates/doc/document_info.html
index 71050f9d41..d6d8d43071 100644
--- a/ietf/templates/doc/document_info.html
+++ b/ietf/templates/doc/document_info.html
@@ -87,7 +87,7 @@
     {% endif %}
     <tr>
         <td></td>
-        <th scope="row">Author{% if doc.pk %}{{ doc.authors|pluralize }}{% endif %}</th>
+        <th scope="row">Author{% if doc.pk %}{{ doc.author_persons_or_names|pluralize }}{% endif %}</th>
         <td class="edit">
             {% if can_edit_authors %}
                 <a class="btn btn-primary btn-sm"
@@ -96,8 +96,8 @@
         </td>
         <td>
             {# Implementation that uses the current primary email for each author #}
-            {% if doc.pk %}{% for author in doc.authors %}
-                {% person_link author %}{% if not forloop.last %},{% endif %}
+            {% if doc.pk %}{% for author in doc.author_persons_or_names %}
+                {% if author.person %}{% person_link author.person %}{% else %}{{ author.titlepage_name }}{% endif %}{% if not forloop.last %},{% endif %}
             {% endfor %}{% endif %}
             {% if document_html and not snapshot or document_html and doc.rev == latest_rev%}
                 <br>
diff --git a/ietf/templates/doc/index_active_drafts.html b/ietf/templates/doc/index_active_drafts.html
index 06ea2c4ff5..607385f56f 100644
--- a/ietf/templates/doc/index_active_drafts.html
+++ b/ietf/templates/doc/index_active_drafts.html
@@ -29,7 +29,7 @@ <h1>Active Internet-Drafts</h1>
     </p>
     {% for group in groups %}
         <h2 class="mt-3" id="id-{{ group.acronym }}">{{ group.name }} ({{ group.acronym }})</h2>
-        {% for d in group.active_drafts %}
+        {% for d in group.active_drafts %}{# n.b., d is a dict, not a Document #}
             <div class="card mb-3">
                 <div class="card-body">
                     <b>{{ d.title }}.</b>
diff --git a/ietf/templates/doc/opengraph.html b/ietf/templates/doc/opengraph.html
index 4fe39b6209..1c8c5abe91 100644
--- a/ietf/templates/doc/opengraph.html
+++ b/ietf/templates/doc/opengraph.html
@@ -1,4 +1,4 @@
-{# Copyright The IETF Trust 2016-2020, All Rights Reserved #}
+{# Copyright The IETF Trust 2016-2025, All Rights Reserved #}
 {% load origin %}
 {% load static %}
 {% load ietf_filters %}
@@ -36,7 +36,7 @@
 {% else %}{# TODO: We need a card image for individual I-Ds. #}
 <meta property="article:section" content="Individual Internet-Draft">
 {% endif %}
-{% if doc.pk %}{% for author in doc.documentauthor_set.all %}<meta property="article:author" content="{{ author.person.name}}">
+{% if doc.pk %}{% for author_name in doc.author_names %}<meta property="article:author" content="{{ author_name }}">
 {% endfor %}{% endif %}
 {% if published %}<meta property="article:published_time" content="{{ published.time|date:'Y-m-d' }}">{% endif %}
 {% if expires %}<meta property="article:expiration_time" content="{{ expires.time|date:'Y-m-d' }}">{% endif %}
\ No newline at end of file
diff --git a/ietf/templates/doc/review/request_info.html b/ietf/templates/doc/review/request_info.html
index 9ad126d59e..51aea10a02 100644
--- a/ietf/templates/doc/review/request_info.html
+++ b/ietf/templates/doc/review/request_info.html
@@ -74,13 +74,13 @@
                 <td>{% person_link review_req.requested_by %}</td>
             </tr>
         {% endif %}
-        {% if review_req.doc.authors %}
+        {% if review_req.doc.author_persons_or_names %}
             <tr>
                 <td></td>
                 <th scope="row">Authors</th>
                 <td>
-                    {% for author in review_req.doc.authors %}
-                        {% person_link author %}{% if not forloop.last %},{% endif %}
+                    {% for person, tp_name in review_req.doc.author_persons_or_names %}
+                        {% if person %}{% person_link person %}{% else %}<span>{{ tp_name }}</span>{% endif %}{% if not forloop.last %},{% endif %}
                     {% endfor %}
                 </td>
             </tr>
diff --git a/ietf/templates/group/manage_review_requests.html b/ietf/templates/group/manage_review_requests.html
index 99b23c138a..d240ef24fa 100644
--- a/ietf/templates/group/manage_review_requests.html
+++ b/ietf/templates/group/manage_review_requests.html
@@ -66,10 +66,10 @@ <h1>
                                     <span class="badge rounded-pill text-bg-info">Auto-suggested</span>
                                     <br>
                                 {% endif %}
-                                {% if r.doc.authors %}
+                                {% if r.doc.author_persons_or_names %}
                                     <span class="fw-bold">Authors:</span>
-                                    {% for person in r.doc.authors %}
-                                        {% person_link person %}{% if not forloop.last %},{% endif %}
+                                    {% for person, tp_name in r.doc.author_persons_or_names %}
+                                        {% if person %}{% person_link person %}{% else %}<span>{{ tp_name }}</span>{% endif %}{% if not forloop.last %},{% endif %}
                                     {% endfor %}
                                     <br>
                                 {% endif %}
diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py
index 86c5a0c1c3..5faf83d93f 100644
--- a/ietf/utils/test_utils.py
+++ b/ietf/utils/test_utils.py
@@ -38,6 +38,7 @@
 import re
 import email
 import html5lib
+import rest_framework.test
 import requests_mock
 import shutil
 import sys
@@ -312,3 +313,11 @@ def tearDown(self):
             shutil.rmtree(dir)
         self.requests_mock.stop()
         super().tearDown()
+
+
+class APITestCase(TestCase):
+    """Test case that uses rest_framework's APIClient
+    
+    This is equivalent to rest_framework.test.APITestCase, but picks up our 
+    """
+    client_class = rest_framework.test.APIClient
diff --git a/ietf/utils/validators.py b/ietf/utils/validators.py
index 92a20f5a26..a99de72724 100644
--- a/ietf/utils/validators.py
+++ b/ietf/utils/validators.py
@@ -33,8 +33,9 @@
 # Note that this is an instantiation of the regex validator, _not_ the
 # regex-string validator defined right below
 validate_no_control_chars = RegexValidator(
-                                    regex="^[^\x00-\x1f]*$",
-                                    message="Please enter a string without control characters." )
+    regex="^[^\x01-\x1f]*$",
+    message="Please enter a string without control characters.",
+)
 
 
 @deconstructible
diff --git a/mypy.ini b/mypy.ini
index 19df7ec9b0..4acaf98c95 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -2,6 +2,9 @@
 
 ignore_missing_imports = True
 
+# allow PEP 695 type aliases (flag needed until mypy >= 1.13) 
+enable_incomplete_feature = NewGenericSyntax
+
 plugins =
     mypy_django_plugin.main
 
diff --git a/requirements.txt b/requirements.txt
index 02a4cf5fd0..3f89f6f16c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -19,8 +19,10 @@ django-analytical>=3.2.0
 django-bootstrap5>=25.1
 django-celery-beat>=2.7.0,<2.8.0  # pin until https://github.com/celery/django-celery-beat/issues/875 is resolved, then revisit
 django-celery-results>=2.6.0
+django-csp>=3.7
 django-cors-headers>=4.7.0
 django-debug-toolbar>=6.0.0
+django-filter>=24.3
 django-markup>=1.10    # Limited use - need to reconcile against direct use of markdown
 django-oidc-provider==0.8.2    # 0.8.3 changes logout flow and claim return
 django-simple-history>=3.10.1
@@ -50,7 +52,7 @@ markdown>=3.8.0
 types-markdown>=3.8.0
 mock>=5.2.0  # should replace with unittest.mock and remove dependency
 types-mock>=5.2.0
-mypy~=1.7.0  # Version requirements determined by django-stubs.
+mypy~=1.11.2  # Version requirements loosely determined by django-stubs.
 oic>=1.7.0  # Used only by tests
 opentelemetry-sdk>=1.38.0
 opentelemetry-instrumentation-django>=0.59b0

From c845aa788b7d1b215636a7730756365c6d27ebd8 Mon Sep 17 00:00:00 2001
From: rjsparks <10996692+rjsparks@users.noreply.github.com>
Date: Wed, 14 Jan 2026 18:11:50 +0000
Subject: [PATCH 540/656] ci: update base image target version to 20260114T1756

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index e05ad51d1b..41ff295eec 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20251219T2152
+FROM ghcr.io/ietf-tools/datatracker-app-base:20260114T1756
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index f8afdadf36..3ad31c7e25 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20251219T2152
+20260114T1756

From c1e8eb658ce6a204f450cd6d61857d49f6bbfcee Mon Sep 17 00:00:00 2001
From: Nicolas Giard <github@ngpixel.com>
Date: Thu, 15 Jan 2026 12:01:00 -0500
Subject: [PATCH 541/656] ci: Increase wait-for-completion timeouts in
 build.yml

---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 5e91445202..d97889fbb8 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -426,7 +426,7 @@ jobs:
           token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
           inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }'
           wait-for-completion: true
-          wait-for-completion-timeout: 30m
+          wait-for-completion-timeout: 60m
           wait-for-completion-interval: 30s
           display-workflow-run-url: false
 
@@ -453,7 +453,7 @@ jobs:
           token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
           inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }'
           wait-for-completion: true
-          wait-for-completion-timeout: 60m
+          wait-for-completion-timeout: 120m
           wait-for-completion-interval: 20s
           display-workflow-run-url: false
 

From ac2fced3113f16e54eb84fd2b5c1b3000f0f2c87 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 15 Jan 2026 14:58:04 -0400
Subject: [PATCH 542/656] fix: remove notice from bofreq template (#10265)

---
 ietf/templates/doc/bofreq/new_bofreq.html | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/ietf/templates/doc/bofreq/new_bofreq.html b/ietf/templates/doc/bofreq/new_bofreq.html
index c6aa0054f9..cda6f73b90 100644
--- a/ietf/templates/doc/bofreq/new_bofreq.html
+++ b/ietf/templates/doc/bofreq/new_bofreq.html
@@ -1,5 +1,5 @@
 {% extends "base.html" %}
-{# Copyright The IETF Trust 2021, All Rights Reserved #}
+{# Copyright The IETF Trust 2021-2026, All Rights Reserved #}
 {% load origin django_bootstrap5 static textfilters %}
 {% block title %}Start a new BOF Request{% endblock %}
 {% block content %}
@@ -12,21 +12,6 @@ <h1>Start a new BOF Request</h1>
         <li>Process: <a href="https://www.rfc-editor.org/rfc/rfc2418.html#section-2.4">RFC 2418 Section 2.4</a></li>
         <li>Considerations for having a successful BoF: <a href="https://www.rfc-editor.org/rfc/rfc5434.html">RFC 5434</a></li>
     </ul>
-    {# The following block needs to be commented out after the BoF deadline and re-opened before next BoF request opening #}
-    <hr>
-    <p class="lead">
-        Announcement for IETF 124: The IESG and the IAB have organized <em>Ask Me Anything</em> (AMA) virtual sessions 
-        for the community to help proponents who are interested in putting up BoF proposals for IETF 124
-        (see also the <a href="https://mailarchive.ietf.org/arch/msg/ietf-announce/mtAYV3Mu5RfJPR-przzr-Nzxhwc/">IETF-announce</a> email):
-    </p>
-    <ul class="lead">
-        <li>28th of August 13:00-14:00 UTC 
-            <a href="https://datatracker.ietf.org/meeting/interim-2025-iesg-22/sessions/iesg.ics" title="iCalendar entry" aria-label="iCalendar entry"><i class="bi bi-calendar"></i></a></li>
-        <li>28th of August 19:00-20:00 UTC 
-            <a href="https://datatracker.ietf.org/meeting/interim-2025-iesg-23/sessions/iesg.ics" title="iCalendar entry" aria-label="iCalendar entry"><i class="bi bi-calendar"></i></a></li>
-    </ul>
-    <hr>
-    {# End of the temporary block #}
     <p>
         The IAB will also attempt to provide BoF Shepherds as described in their document on the subject only on request from the IESG. 
         If you feel that your BoF would benefit from an IAB BoF Shepherd, please discuss this with your Area Director.

From 0b6e887c36b37f3b66cf653360471f5151cd7796 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 16 Jan 2026 15:38:26 -0400
Subject: [PATCH 543/656] test: more robust URL coverage; improve badly formed
 test (#10270)

* test: use bad ID with correct format

* test: use Django resolver for URL coverage

* chore: f-strings are hard
---
 ietf/api/tests_views_rpc.py |  5 +++-
 ietf/utils/test_runner.py   | 56 ++++++++++++++++++++++++++++++-------
 2 files changed, 50 insertions(+), 11 deletions(-)

diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py
index 032b4b9495..ece2af1b85 100644
--- a/ietf/api/tests_views_rpc.py
+++ b/ietf/api/tests_views_rpc.py
@@ -6,6 +6,7 @@
 from django.conf import settings
 from django.core.files.base import ContentFile
 from django.db.models import Max
+from django.db.models.functions import Coalesce
 from django.test.utils import override_settings
 from django.urls import reverse as urlreverse
 
@@ -22,7 +23,9 @@ def test_draftviewset_references(self):
         viewname = "ietf.api.purple_api.draft-references"
 
         # non-existent draft
-        bad_id = Document.objects.aggregate(unused_id=Max("id") + 100)["unused_id"]
+        bad_id = Document.objects.aggregate(unused_id=Coalesce(Max("id"), 0) + 100)[
+            "unused_id"
+        ]
         url = urlreverse(viewname, kwargs={"doc_id": bad_id})
         # Without credentials
         r = self.client.get(url)
diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py
index 1a3d4e5c3d..df8ed1fd61 100644
--- a/ietf/utils/test_runner.py
+++ b/ietf/utils/test_runner.py
@@ -70,7 +70,7 @@
 from django.template.loaders.filesystem import Loader as BaseLoader
 from django.test.runner import DiscoverRunner
 from django.core.management import call_command
-from django.urls import URLResolver # type: ignore
+from django.urls import URLResolver, resolve, Resolver404  # type: ignore
 from django.template.backends.django import DjangoTemplates
 from django.template.backends.django import Template  # type: ignore[attr-defined]
 from django.utils import timezone
@@ -88,6 +88,26 @@
 from mypy_boto3_s3.service_resource import Bucket
 
 
+class UrlCoverageWarning(UserWarning):
+    """Warning category for URL coverage-related warnings"""
+    pass
+
+
+class UninterestingPatternWarning(UrlCoverageWarning):
+    """Warning category for unexpected URL match patterns
+    
+    These are common, caused by tests that hit a URL that is not selected for
+    coverage checking. The warning is in place to help with a putative future 
+    review of whether we're selecting the right patterns to check for coverage. 
+    """
+    pass
+
+
+# Configure warnings for reasonable output quantity
+warnings.simplefilter("once", UrlCoverageWarning)
+warnings.simplefilter("ignore", UninterestingPatternWarning)
+
+
 loaded_templates: set[str] = set()
 visited_urls: set[str] = set()
 test_database_name: Optional[str] = None
@@ -550,21 +570,37 @@ def ignore_pattern(regex, pattern):
                         )
                         or pattern.callback == django.views.static.serve)
 
-            patterns = [(regex, re.compile(regex, re.U), obj) for regex, obj in url_patterns
-                        if not ignore_pattern(regex, obj)]
+            patterns ={
+                regex: obj
+                for regex, obj in url_patterns
+                if not ignore_pattern(regex, obj)
+            }
 
             covered = set()
             for url in visited_urls:
-                for regex, compiled, obj in patterns:
-                    if regex not in covered and compiled.match(url[1:]): # strip leading /
-                        covered.add(regex)
-                        break
+                try:
+                    resolved = resolve(url)  # let Django resolve the URL for us
+                except Resolver404:
+                    warnings.warn(
+                        f"Unable to resolve visited URL {url}", UrlCoverageWarning
+                    )
+                    continue
+                if resolved.route not in patterns:
+                    warnings.warn(
+                        f"WARNING: url resolved to an unexpected pattern (url='{url}', "
+                        f"resolved to r'{resolved.route}'",
+                        UninterestingPatternWarning,
+                    )
+                    continue
+                covered.add(resolved.route)
 
             self.runner.coverage_data["url"] = {
-                "coverage": 1.0*len(covered)/len(patterns),
-                "covered": dict( (k, (o.lookup_str, k in covered)) for k,p,o in patterns ),
+                "coverage": 1.0 * len(covered) / len(patterns),
+                "covered": dict(
+                    (k, (o.lookup_str, k in covered)) for k, o in patterns.items()
+                ),
                 "format": 4,
-                }
+            }
 
             self.report_test_result("url")
         else:

From 0e9e18efa187857a8992aee867d659d3c0276d2f Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 16 Jan 2026 16:21:49 -0400
Subject: [PATCH 544/656] chore: suppress expected warnings (#10272)

---
 ietf/utils/test_runner.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py
index df8ed1fd61..a23416e87f 100644
--- a/ietf/utils/test_runner.py
+++ b/ietf/utils/test_runner.py
@@ -90,7 +90,11 @@
 
 class UrlCoverageWarning(UserWarning):
     """Warning category for URL coverage-related warnings"""
-    pass
+    # URLs for which we don't expect patterns to match
+    IGNORE_URLS = (
+        "/_doesnotexist/",
+        "/sitemap.xml.",
+    )
 
 
 class UninterestingPatternWarning(UrlCoverageWarning):
@@ -581,9 +585,10 @@ def ignore_pattern(regex, pattern):
                 try:
                     resolved = resolve(url)  # let Django resolve the URL for us
                 except Resolver404:
-                    warnings.warn(
-                        f"Unable to resolve visited URL {url}", UrlCoverageWarning
-                    )
+                    if url not in UrlCoverageWarning.IGNORE_URLS:
+                        warnings.warn(
+                            f"Unable to resolve visited URL {url}", UrlCoverageWarning
+                        )
                     continue
                 if resolved.route not in patterns:
                     warnings.warn(

From 337a2311cbef23d46e4dc290742215de764a701c Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 16 Jan 2026 14:28:12 -0600
Subject: [PATCH 545/656] feat: rsync rfc content, store in blob, rebuild
 references (#10255)

* feat: rsync rfc content, store in blob, rebuild references

* fix: isolate subprocess. Guard against missing file

* fix: correct variable initialization. guard against unnecessary call

* test: mock rsync task calls

* fix: use list for typing rather than List

* fix: string formatting

* fix: generalize error string when there are no files to parse

* fix: use delete_on_close with NamedTemporaryFile

* fix: mtime is less distracting than m_time

* fix: store the notprepped file on the fs

* fix: typo

* fix: fetch json, remove unneeded unlink

* chore: ruff

* fix: use list for typing

* fix: typo

* feat: bulk load rfcs into blob storage

* fix: restrict the rsync_helper to rsync

* test: test ietf.sync.utils

* chore: honor typing choices

* test: sync task tests

* refactor: isolate the rsync from-file construction and test it

* chore: ruff

* fix: reflect current changes in older test

* fix: address incorrect test assumption

* chore: adhere to task naming conventions
---
 ietf/doc/tasks.py        | 21 ++++++++++
 ietf/doc/tests_utils.py  |  4 +-
 ietf/doc/utils.py        | 48 +++++++++++++++--------
 ietf/settings.py         |  1 +
 ietf/sync/tasks.py       | 50 ++++++++++++++++++++++++
 ietf/sync/tests.py       | 82 +++++++++++++++++++++++++++++++++++++++-
 ietf/sync/tests_utils.py | 82 ++++++++++++++++++++++++++++++++++++++++
 ietf/sync/utils.py       | 69 +++++++++++++++++++++++++++++++++
 8 files changed, 338 insertions(+), 19 deletions(-)
 create mode 100644 ietf/sync/tests_utils.py
 create mode 100644 ietf/sync/utils.py

diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py
index 4f7fe37782..02b7c2a07d 100644
--- a/ietf/doc/tasks.py
+++ b/ietf/doc/tasks.py
@@ -29,6 +29,7 @@
 from .utils import (
     generate_idnits2_rfc_status,
     generate_idnits2_rfcs_obsoleted,
+    rebuild_reference_relations,
     update_or_create_draft_bibxml_file,
     ensure_draft_bibxml_path_exists,
     investigate_fragment,
@@ -128,3 +129,23 @@ def investigate_fragment_task(name_fragment: str):
         "name_fragment": name_fragment,
         "results": investigate_fragment(name_fragment),
     }
+
+@shared_task
+def rebuild_reference_relations_task(doc_names: list[str]):
+    log.log(f"Task: Rebuilding reference relations for {doc_names}")
+    for doc in Document.objects.filter(name__in=doc_names, type__in=["rfc", "draft"]):
+        filenames = dict()
+        base = (
+            settings.RFC_PATH
+            if doc.type_id == "rfc"
+            else settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR
+        )
+        stem = doc.name if doc.type_id == "rfc" else f"{doc.name}-{doc.rev}"
+        for ext in ["xml", "txt"]:
+            path = Path(base) / f"{stem}.{ext}"
+            if path.is_file():
+                filenames[ext] = str(path)
+        if len(filenames) > 0:
+            rebuild_reference_relations(doc, filenames)
+        else:
+            log.log(f"Found no content for {stem}")
diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py
index ef71f6ae6e..a2784bc85e 100644
--- a/ietf/doc/tests_utils.py
+++ b/ietf/doc/tests_utils.py
@@ -389,13 +389,13 @@ def test_requires_txt_or_xml(self):
         result = rebuild_reference_relations(self.doc, {})
         self.assertCountEqual(result.keys(), ['errors'])
         self.assertEqual(len(result['errors']), 1)
-        self.assertIn('No Internet-Draft text available', result['errors'][0],
+        self.assertIn('No file available', result['errors'][0],
                       'Error should be reported if no Internet-Draft file is given')
 
         result = rebuild_reference_relations(self.doc, {'md': 'cant-do-this.md'})
         self.assertCountEqual(result.keys(), ['errors'])
         self.assertEqual(len(result['errors']), 1)
-        self.assertIn('No Internet-Draft text available', result['errors'][0],
+        self.assertIn('No file available', result['errors'][0],
                       'Error should be reported if no XML or plaintext file is given')
 
     @patch.object(XMLDraft, 'get_refs')
diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index 0715471551..42fab7d472 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -941,50 +941,66 @@ def rebuild_reference_relations(doc, filenames):
 
     filenames should be a dict mapping file ext (i.e., type) to the full path of each file.
     """
-    if doc.type.slug != 'draft':
+    if doc.type.slug not in ["draft", "rfc"]:
         return None
+    
+    log.log(f"Rebuilding reference relations for {doc.name}")
 
     # try XML first
-    if 'xml' in filenames:
-        refs = XMLDraft(filenames['xml']).get_refs()
-    elif 'txt' in filenames:
-        filename = filenames['txt']
+    if "xml" in filenames:
+        refs = XMLDraft(filenames["xml"]).get_refs()
+    elif "txt" in filenames:
+        filename = filenames["txt"]
         try:
             refs = draft.PlaintextDraft.from_file(filename).get_refs()
         except IOError as e:
-            return { 'errors': ["%s :%s" %  (e.strerror, filename)] }
+            return {"errors": [f"{e.strerror}: {filename}"]}
     else:
-        return {'errors': ['No Internet-Draft text available for rebuilding reference relations. Need XML or plaintext.']}
+        return {
+            "errors": [
+                "No file available for rebuilding reference relations. Need XML or plaintext."
+            ]
+        }
 
-    doc.relateddocument_set.filter(relationship__slug__in=['refnorm','refinfo','refold','refunk']).delete()
+    doc.relateddocument_set.filter(
+        relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"]
+    ).delete()
 
     warnings = []
     errors = []
     unfound = set()
-    for ( ref, refType ) in refs.items():
+    for ref, refType in refs.items():
         refdoc = Document.objects.filter(name=ref)
         if not refdoc and re.match(r"^draft-.*-\d{2}$", ref):
             refdoc = Document.objects.filter(name=ref[:-3])
         count = refdoc.count()
         if count == 0:
-            unfound.add( "%s" % ref )
+            unfound.add("%s" % ref)
             continue
         elif count > 1:
-            errors.append("Too many Document objects found for %s"%ref)
+            errors.append("Too many Document objects found for %s" % ref)
         else:
             # Don't add references to ourself
             if doc != refdoc[0]:
-                RelatedDocument.objects.get_or_create( source=doc, target=refdoc[ 0 ], relationship=DocRelationshipName.objects.get( slug='ref%s' % refType ) )
+                RelatedDocument.objects.get_or_create(
+                    source=doc,
+                    target=refdoc[0],
+                    relationship=DocRelationshipName.objects.get(
+                        slug="ref%s" % refType
+                    ),
+                )
     if unfound:
-        warnings.append('There were %d references with no matching Document'%len(unfound))
+        warnings.append(
+            "There were %d references with no matching Document" % len(unfound)
+        )
 
     ret = {}
     if errors:
-        ret['errors']=errors
+        ret["errors"] = errors
     if warnings:
-        ret['warnings']=warnings
+        ret["warnings"] = warnings
     if unfound:
-        ret['unfound']=list(unfound)
+        ret["unfound"] = list(unfound)
 
     return ret
 
diff --git a/ietf/settings.py b/ietf/settings.py
index fedd313ca0..fd8d86a1ab 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -811,6 +811,7 @@ def skip_unreadable_post(record):
     "polls",
     "procmaterials",
     "review",
+    "rfc",
     "slides",
     "staging",
     "statchg",
diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py
index e4174d3729..fc75a056ed 100644
--- a/ietf/sync/tasks.py
+++ b/ietf/sync/tasks.py
@@ -4,6 +4,8 @@
 #
 import datetime
 import io
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 import requests
 
 from celery import shared_task
@@ -12,9 +14,11 @@
 from django.utils import timezone
 
 from ietf.doc.models import DocEvent, RelatedDocument
+from ietf.doc.tasks import rebuild_reference_relations_task
 from ietf.sync import iana
 from ietf.sync import rfceditor
 from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue
+from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper
 from ietf.utils import log
 from ietf.utils.timezone import date_today
 
@@ -65,11 +69,16 @@ def rfc_editor_index_update_task(full_index=False):
     if len(errata_data) < rfceditor.MIN_ERRATA_RESULTS:
         log.log("Not enough errata entries, only %s" % len(errata_data))
         return  # failed
+    newly_published = set()
     for rfc_number, changes, doc, rfc_published in rfceditor.update_docs_from_rfc_index(
         index_data, errata_data, skip_older_than_date=skip_date
     ):
         for c in changes:
             log.log("RFC%s, %s: %s" % (rfc_number, doc.name, c))
+        if rfc_published:
+            newly_published.add(rfc_number)
+    if len(newly_published) > 0:
+        rsync_rfcs_from_rfceditor_task.delay(list(newly_published))
 
 
 @shared_task
@@ -222,3 +231,44 @@ def fix_subseries_docevents_task():
         DocEvent.objects.filter(type="sync_from_rfc_editor", desc=desc).update(
             time=obsoleting_time
         )
+
+@shared_task
+def rsync_rfcs_from_rfceditor_task(rfc_numbers: list[int]):
+    log.log(f"Rsyncing rfcs from rfc-editor: {rfc_numbers}")
+    from_file = None
+    with NamedTemporaryFile(mode="w", delete_on_close=False) as fp:
+        fp.write(build_from_file_content(rfc_numbers))
+        fp.close()
+        from_file = Path(fp.name)
+        rsync_helper(
+            [
+                "-a",
+                "--ignore-existing",
+                f"--include-from={from_file}",
+                "--exclude=*",
+                "rsync.rfc-editor.org::rfcs/",
+                f"{settings.RFC_PATH}",
+            ]
+        )
+    load_rfcs_into_blobdb(rfc_numbers)
+
+    rebuild_reference_relations_task.delay([f"rfc{num}" for num in rfc_numbers])
+
+
+@shared_task
+def load_rfcs_into_blobdb_task(start: int, end: int):
+    """Move file content for rfcs from rfc{start} to rfc{end} inclusive
+
+    As this is expected to be removed once the blobdb is populated, it
+    will truncate its work to a coded max end.
+    This will not overwrite any existing blob content, and will only
+    log a small complaint if asked to load a non-exsiting RFC.
+    """
+    # Protect us from ourselves
+    if end < start:
+        return
+    if start < 1:
+        start = 1
+    if end > 11000:  # Arbitrarily chosen
+        end = 11000
+    load_rfcs_into_blobdb(list(range(start, end + 1)))
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index 888920ae9d..bcc87a43aa 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -889,8 +889,9 @@ class TaskTests(TestCase):
     @mock.patch("ietf.sync.tasks.rfceditor.update_docs_from_rfc_index")
     @mock.patch("ietf.sync.tasks.rfceditor.parse_index")
     @mock.patch("ietf.sync.tasks.requests.get")
+    @mock.patch("ietf.sync.tasks.rsync_rfcs_from_rfceditor_task.delay")
     def test_rfc_editor_index_update_task(
-        self, requests_get_mock, parse_index_mock, update_docs_mock
+        self, rsync_task_mock, requests_get_mock, parse_index_mock, update_docs_mock
     ) -> None:  # the annotation here prevents mypy from complaining about annotation-unchecked
         """rfc_editor_index_update_task calls helpers correctly
         
@@ -922,6 +923,7 @@ def json(self):
         rfc = RfcFactory()
     
         # Test with full_index = False
+        rsync_task_mock.return_value = None
         requests_get_mock.side_effect = (index_response, errata_response)  # will step through these
         parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS)
         update_docs_mock.return_value = (
@@ -947,10 +949,13 @@ def json(self):
         )
         self.assertIsNotNone(update_docs_kwargs["skip_older_than_date"])
 
+        self.assertFalse(rsync_task_mock.called)
+
         # Test again with full_index = True
         requests_get_mock.reset_mock()
         parse_index_mock.reset_mock()
         update_docs_mock.reset_mock()
+        rsync_task_mock.reset_mock()
         requests_get_mock.side_effect = (index_response, errata_response)  # will step through these
         tasks.rfc_editor_index_update_task(full_index=True)
 
@@ -971,40 +976,67 @@ def json(self):
         )
         self.assertIsNone(update_docs_kwargs["skip_older_than_date"])
 
+        self.assertFalse(rsync_task_mock.called)
+
+        # Test again where the index would cause a new RFC to come into existance
+        requests_get_mock.reset_mock()
+        parse_index_mock.reset_mock()
+        update_docs_mock.reset_mock()
+        rsync_task_mock.reset_mock()
+        requests_get_mock.side_effect = (
+            index_response,
+            errata_response,
+        )  # will step through these
+        update_docs_mock.return_value = (
+            (rfc.rfc_number, ("something changed",), rfc, True),
+        )
+        tasks.rfc_editor_index_update_task(full_index=True)
+        self.assertTrue(rsync_task_mock.called)
+        rsync_task_args, rsync_task_kwargs = rsync_task_mock.call_args
+        self.assertEqual((([rfc.rfc_number],), {}), (rsync_task_args, rsync_task_kwargs))
+
         # Test error handling
         requests_get_mock.reset_mock()
         parse_index_mock.reset_mock()
         update_docs_mock.reset_mock()
+        rsync_task_mock.reset_mock()
         requests_get_mock.side_effect = requests.Timeout  # timeout on every get()
         tasks.rfc_editor_index_update_task(full_index=False)
         self.assertFalse(parse_index_mock.called)
         self.assertFalse(update_docs_mock.called)
+        self.assertFalse(rsync_task_mock.called)
         
         requests_get_mock.reset_mock()
         parse_index_mock.reset_mock()
         update_docs_mock.reset_mock()
+        rsync_task_mock.reset_mock()
         requests_get_mock.side_effect = [index_response, requests.Timeout]  # timeout second get()
         tasks.rfc_editor_index_update_task(full_index=False)
         self.assertFalse(update_docs_mock.called)
+        self.assertFalse(rsync_task_mock.called)
 
         requests_get_mock.reset_mock()
         parse_index_mock.reset_mock()
         update_docs_mock.reset_mock()
+        rsync_task_mock.reset_mock()
         requests_get_mock.side_effect = [index_response, errata_response]
         # feed in an index that is too short
         parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS - 1)
         tasks.rfc_editor_index_update_task(full_index=False)
         self.assertTrue(parse_index_mock.called)
         self.assertFalse(update_docs_mock.called)
+        self.assertFalse(rsync_task_mock.called)
 
         requests_get_mock.reset_mock()
         parse_index_mock.reset_mock()
         update_docs_mock.reset_mock()
+        rsync_task_mock.reset_mock()
         requests_get_mock.side_effect = [index_response, errata_response]
         errata_response.json_length = rfceditor.MIN_ERRATA_RESULTS - 1  # too short
         parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS)
         tasks.rfc_editor_index_update_task(full_index=False)
         self.assertFalse(update_docs_mock.called)
+        self.assertFalse(rsync_task_mock.called)
 
     @override_settings(RFC_EDITOR_QUEUE_URL="https://rfc-editor.example.com/queue/")
     @mock.patch("ietf.sync.tasks.update_drafts_from_queue")
@@ -1134,3 +1166,51 @@ def test_iana_protocols_update_task(
         self.assertTrue(requests_get_mock.called)
         self.assertFalse(parse_protocols_mock.called)
         self.assertFalse(update_rfc_log_mock.called)
+
+    @mock.patch("ietf.sync.tasks.rsync_helper")
+    @mock.patch("ietf.sync.tasks.load_rfcs_into_blobdb")
+    @mock.patch("ietf.sync.tasks.rebuild_reference_relations_task.delay")
+    def test_rsync_rfcs_from_rfceditor_task(
+        self,
+        rebuild_relations_mock,
+        load_blobs_mock,
+        rsync_helper_mock,
+    ):
+        tasks.rsync_rfcs_from_rfceditor_task([12345, 54321])
+        self.assertTrue(rsync_helper_mock.called)
+        self.assertTrue(load_blobs_mock.called)
+        load_blobs_args, load_blobs_kwargs = load_blobs_mock.call_args
+        self.assertEqual(load_blobs_args, ([12345, 54321],))
+        self.assertEqual(load_blobs_kwargs, {})
+        self.assertTrue(rebuild_relations_mock.called)
+        rebuild_args, rebuild_kwargs = rebuild_relations_mock.call_args
+        self.assertEqual(rebuild_args, (["rfc12345", "rfc54321"],))
+        self.assertEqual(rebuild_kwargs, {})
+
+    @mock.patch("ietf.sync.tasks.load_rfcs_into_blobdb")
+    def test_load_rfcs_into_blobdb_task(
+        self,
+        load_blobs_mock,
+    ):
+        tasks.load_rfcs_into_blobdb_task(5, 3)
+        self.assertFalse(load_blobs_mock.called)
+        load_blobs_mock.reset_mock()
+        tasks.load_rfcs_into_blobdb_task(-1, 1)
+        self.assertTrue(load_blobs_mock.called)
+        mock_args, mock_kwargs = load_blobs_mock.call_args
+        self.assertEqual(mock_args, ([1],))
+        self.assertEqual(mock_kwargs, {})
+        load_blobs_mock.reset_mock()
+        tasks.load_rfcs_into_blobdb_task(10999, 50000)
+        self.assertTrue(load_blobs_mock.called)
+        mock_args, mock_kwargs = load_blobs_mock.call_args
+        self.assertEqual(mock_args, ([10999, 11000],))
+        self.assertEqual(mock_kwargs, {})
+        load_blobs_mock.reset_mock()
+        tasks.load_rfcs_into_blobdb_task(3261, 3263)
+        self.assertTrue(load_blobs_mock.called)
+        mock_args, mock_kwargs = load_blobs_mock.call_args
+        self.assertEqual(mock_args, ([3261, 3262, 3263],))
+        self.assertEqual(mock_kwargs, {})
+
+
diff --git a/ietf/sync/tests_utils.py b/ietf/sync/tests_utils.py
new file mode 100644
index 0000000000..eb4b4ddf74
--- /dev/null
+++ b/ietf/sync/tests_utils.py
@@ -0,0 +1,82 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from django.test import override_settings
+from ietf import settings
+from ietf.doc.storage_utils import exists_in_storage, retrieve_str
+from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper
+from ietf.utils.test_utils import TestCase
+
+
+class RsyncHelperTests(TestCase):
+    def test_rsync_helper(self):
+        with (
+            TemporaryDirectory() as source_dir,
+            TemporaryDirectory() as dest_dir,
+        ):
+            with (Path(source_dir) / "canary.txt").open("w") as canary_source_file:
+                canary_source_file.write("chirp")
+            rsync_helper(
+                [
+                    "-a",
+                    f"{source_dir}/",
+                    f"{dest_dir}/",
+                ]
+            )
+            with (Path(dest_dir) / "canary.txt").open("r") as canary_dest_file:
+                chirp = canary_dest_file.read()
+            self.assertEqual(chirp, "chirp")
+
+    def test_build_from_file_content(self):
+        content = build_from_file_content([12345, 54321])
+        self.assertEqual(
+            content,
+            """prerelease/
+rfc12345.txt
+rfc12345.html
+rfc12345.xml
+rfc12345.pdf
+rfc12345.ps
+rfc12345.json
+prerelease/rfc12345.notprepped.xml
+rfc54321.txt
+rfc54321.html
+rfc54321.xml
+rfc54321.pdf
+rfc54321.ps
+rfc54321.json
+prerelease/rfc54321.notprepped.xml
+""",
+        )
+
+
+class RfcBlobUploadTests(TestCase):
+    def test_load_rfcs_into_blobdb(self):
+        with TemporaryDirectory() as faux_rfc_path:
+            with override_settings(RFC_PATH=faux_rfc_path):
+                rfc_path = Path(faux_rfc_path)
+                (rfc_path / "prerelease").mkdir()
+                for num in [12345, 54321]:
+                    for ext in settings.RFC_FILE_TYPES + ("json",):
+                        with (rfc_path / f"rfc{num}.{ext}").open("w") as f:
+                            f.write(ext)
+                    with (rfc_path / "rfc{num}.bogon").open("w") as f:
+                        f.write("bogon")
+                    with (rfc_path / "prerelease" / f"rfc{num}.notprepped.xml").open(
+                        "w"
+                    ) as f:
+                        f.write("notprepped")
+                load_rfcs_into_blobdb([12345, 54321])
+                for num in [12345, 54321]:
+                    for ext in settings.RFC_FILE_TYPES + ("json",):
+                        self.assertEqual(
+                            retrieve_str("rfc", f"{ext}/rfc{num}.{ext}"),
+                            ext,
+                        )
+                    self.assertFalse(exists_in_storage("rfc", f"bogon/rfc{num}.bogon"))
+                    self.assertEqual(
+                        retrieve_str("rfc", f"notprepped/rfc{num}.notprepped.xml"),
+                        "notprepped",
+                    )
diff --git a/ietf/sync/utils.py b/ietf/sync/utils.py
new file mode 100644
index 0000000000..5b5f8ff559
--- /dev/null
+++ b/ietf/sync/utils.py
@@ -0,0 +1,69 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+import datetime
+import subprocess
+
+from pathlib import Path
+
+from django.conf import settings
+
+from ietf.utils import log
+from ietf.doc.storage_utils import AlreadyExistsError, store_bytes
+
+
+def rsync_helper(subprocess_arg_array: list[str]):
+    subprocess.run(["/usr/bin/rsync"]+subprocess_arg_array)
+
+def build_from_file_content(rfc_numbers: list[int]) -> str:
+    types_to_sync = settings.RFC_FILE_TYPES + ("json",)
+    lines = []
+    lines.append("prerelease/")
+    for num in rfc_numbers:
+        for ext in types_to_sync:
+            lines.append(f"rfc{num}.{ext}")
+        lines.append(f"prerelease/rfc{num}.notprepped.xml")
+    return "\n".join(lines)+"\n"
+
+def load_rfcs_into_blobdb(numbers: list[int]):
+    types_to_load = settings.RFC_FILE_TYPES + ("json",)
+    for num in numbers:
+        for ext in types_to_load:
+            fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}"
+            if fs_path.is_file():
+                with fs_path.open("rb") as f:
+                    bytes = f.read()
+                mtime = fs_path.stat().st_mtime
+                try:
+                    store_bytes(
+                        kind="rfc",
+                        name=f"{ext}/rfc{num}.{ext}",
+                        content=bytes,
+                        allow_overwrite=False,  # Intentionally not allowing overwrite.
+                        doc_name=f"rfc{num}",
+                        doc_rev=None,
+                        # Not setting content_type
+                        mtime=datetime.datetime.fromtimestamp(mtime, tz=datetime.UTC),
+                    )
+                except AlreadyExistsError as e:
+                    log.log(str(e))
+
+        # store the not-prepped xml
+        name = f"rfc{num}.notprepped.xml"
+        source = Path(settings.RFC_PATH) / "prerelease" / name
+        if source.is_file():
+            with open(source, "rb") as f:
+                bytes = f.read()
+            mtime = source.stat().st_mtime
+            try:
+                store_bytes(
+                    kind="rfc",
+                    name=f"notprepped/{name}",
+                    content=bytes,
+                    allow_overwrite=False,  # Intentionally not allowing overwrite.
+                    doc_name=f"rfc{num}",
+                    doc_rev=None,
+                    # Not setting content_type
+                    mtime=datetime.datetime.fromtimestamp(mtime, tz=datetime.UTC),
+                )
+            except AlreadyExistsError as e:
+                log.log(str(e))

From 37888b36b3f7e2262b857363ff7b3b4486dbcbe2 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 16 Jan 2026 16:28:53 -0400
Subject: [PATCH 546/656] feat: store blobs+set mtime in RFC publish API
 (#10260)

* feat: set mtime for RFC pub files

* chore: add rfc storage

* refactor: destination helper is fs-specific

* feat: RFC files->blobstore in publish API

* test: test blob writing

* chore: remove completed todo comment
---
 ietf/api/serializers_rpc.py |  7 +++++
 ietf/api/tests_views_rpc.py | 58 +++++++++++++++++++++++++++++++++----
 ietf/api/views_rpc.py       | 57 ++++++++++++++++++++++++++++++++----
 3 files changed, 111 insertions(+), 11 deletions(-)

diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py
index 2223f04aeb..f2e735be7a 100644
--- a/ietf/api/serializers_rpc.py
+++ b/ietf/api/serializers_rpc.py
@@ -5,6 +5,7 @@
 
 from django.db import transaction
 from django.urls import reverse as urlreverse
+from django.utils import timezone
 from drf_spectacular.types import OpenApiTypes
 from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
@@ -571,6 +572,12 @@ class RfcFileSerializer(serializers.Serializer):
             "file types, but filenames are otherwise ignored."
         ),
     )
+    mtime = serializers.DateTimeField(
+        required=False,
+        default=timezone.now,
+        default_timezone=datetime.UTC,
+        help_text="Modification timestamp to apply to uploaded files",
+    )
     replace = serializers.BooleanField(
         required=False,
         default=False,
diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py
index ece2af1b85..ecb50ee76c 100644
--- a/ietf/api/tests_views_rpc.py
+++ b/ietf/api/tests_views_rpc.py
@@ -10,6 +10,7 @@
 from django.test.utils import override_settings
 from django.urls import reverse as urlreverse
 
+from ietf.blobdb.models import Blob
 from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory
 from ietf.doc.models import RelatedDocument, Document
 from ietf.group.factories import RoleFactory, GroupFactory
@@ -259,6 +260,31 @@ def _valid_post_data():
             )
             self.assertEqual(r.status_code, 400)
 
+            # Put a file in the way. Post should fail because replace = False
+            file_in_the_way = (rfc_path / f"rfc{unused_rfc_number}.txt")
+            file_in_the_way.touch()
+            r = self.client.post(
+                url,
+                _valid_post_data(),
+                format="multipart",
+                headers={"X-Api-Key": "valid-token"},
+            )
+            self.assertEqual(r.status_code, 409)  # conflict
+            file_in_the_way.unlink()
+            
+            # Put a blob in the way. Post should fail because replace = False
+            blob_in_the_way = Blob.objects.create(
+                bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b""
+            )
+            r = self.client.post(
+                url,
+                _valid_post_data(),
+                format="multipart",
+                headers={"X-Api-Key": "valid-token"},
+            )
+            self.assertEqual(r.status_code, 409)  # conflict
+            blob_in_the_way.delete()
+
             # valid post
             r = self.client.post(
                 url,
@@ -267,21 +293,41 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 200)
-            for suffix in [".xml", ".txt", ".html", ".pdf", ".json"]:
+            for extension in ["xml", "txt", "html", "pdf", "json"]:
+                filename = f"rfc{unused_rfc_number}.{extension}"
                 self.assertEqual(
-                    (rfc_path / f"rfc{unused_rfc_number}")
-                    .with_suffix(suffix)
+                    (rfc_path / filename)
                     .read_text(),
-                    f"This is {suffix}",
-                    f"{suffix} file should contain the expected content",
+                    f"This is .{extension}",
+                    f"{extension} file should contain the expected content",
+                )
+                self.assertEqual(
+                    bytes(
+                        Blob.objects.get(
+                            bucket="rfc", name=f"{extension}/{filename}"
+                        ).content
+                    ),
+                    f"This is .{extension}".encode("utf-8"),
+                    f"{extension} blob should contain the expected content",
                 )
+            # special case for notprepped
+            notprepped_fn = f"rfc{unused_rfc_number}.notprepped.xml"
             self.assertEqual(
                 (
-                    rfc_path / "prerelease" / f"rfc{unused_rfc_number}.notprepped.xml"
+                    rfc_path / "prerelease" / notprepped_fn
                 ).read_text(),
                 "This is .notprepped.xml",
                 ".notprepped.xml file should contain the expected content",
             )
+            self.assertEqual(
+                bytes(
+                    Blob.objects.get(
+                        bucket="rfc", name=f"notprepped/{notprepped_fn}"
+                    ).content
+                ),
+                b"This is .notprepped.xml",
+                ".notprepped.xml blob should contain the expected content",
+            )
 
             # re-post with replace = False should now fail
             r = self.client.post(
diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index fce174ab72..542836a857 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -1,4 +1,5 @@
 # Copyright The IETF Trust 2023-2026, All Rights Reserved
+import os
 import shutil
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -35,6 +36,7 @@
 )
 from ietf.doc.models import Document, DocHistory, RfcAuthor
 from ietf.doc.serializers import RfcAuthorSerializer
+from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage
 from ietf.person.models import Email, Person
 
 
@@ -366,8 +368,8 @@ class RfcPubFilesView(APIView):
     api_key_endpoint = "ietf.api.views_rpc"
     parser_classes = [parsers.MultiPartParser]
 
-    def _destination(self, filename: str | Path) -> Path:
-        """Destination for an uploaded RFC file
+    def _fs_destination(self, filename: str | Path) -> Path:
+        """Destination for an uploaded RFC file in the filesystem
         
         Strips any path components in filename and returns an absolute Path.
         """
@@ -378,6 +380,23 @@ def _destination(self, filename: str | Path) -> Path:
             return rfc_path / "prerelease" / filename.name
         return rfc_path / filename.name
 
+    def _blob_destination(self, filename: str | Path) -> str:
+        """Destination name for an uploaded RFC file in the blob store
+        
+        Strips any path components in filename and returns an absolute Path.
+        """
+        filename = Path(filename)  # could potentially have directory components
+        extension = "".join(filename.suffixes)
+        if extension == ".notprepped.xml":
+            file_type = "notprepped"
+        elif extension[0] == ".":
+            file_type = extension[1:]
+        else:
+            raise serializers.ValidationError(
+                f"Extension does not begin with '.'!? ({filename})",
+            )
+        return f"{file_type}/{filename.name}"
+
     @extend_schema(
         operation_id="upload_rfc_files",
         summary="Upload files for a published RFC",
@@ -394,10 +413,17 @@ def post(self, request):
         uploaded_files = serializer.validated_data["contents"]  # list[UploadedFile]
         replace = serializer.validated_data["replace"]
         dest_stem = f"rfc{rfc.rfc_number}"
+        mtime = serializer.validated_data["mtime"]
+        mtimestamp = mtime.timestamp()
+        blob_kind = "rfc"
 
         # List of files that might exist for an RFC
         possible_rfc_files = [
-            self._destination(dest_stem + ext)
+            self._fs_destination(dest_stem + ext)
+            for ext in serializer.allowed_extensions
+        ]
+        possible_rfc_blobs = [
+            self._blob_destination(dest_stem + ext)
             for ext in serializer.allowed_extensions
         ]
         if not replace:
@@ -408,6 +434,14 @@ def post(self, request):
                         "File(s) already exist for this RFC",
                         code="files-exist",
                     )
+            for possible_existing_blob in possible_rfc_blobs:
+                if exists_in_storage(
+                    kind=blob_kind, name=possible_existing_blob
+                ):
+                    raise Conflict(
+                        "Blob(s) already exist for this RFC",
+                        code="blobs-exist",
+                    )
 
         with TemporaryDirectory() as tempdir:
             # Save files in a temporary directory. Use the uploaded filename
@@ -421,14 +455,27 @@ def post(self, request):
                 with tempfile_path.open("wb") as dest:
                     for chunk in upfile.chunks():
                         dest.write(chunk)
+                os.utime(tempfile_path, (mtimestamp, mtimestamp))
                 files_to_move.append(tempfile_path)
             # copy files to final location, removing any existing ones first if the
             # remove flag was set
             if replace:
                 for possible_existing_file in possible_rfc_files:
                     possible_existing_file.unlink(missing_ok=True)
+                for possible_existing_blob in possible_rfc_blobs:
+                    remove_from_storage(
+                        blob_kind, possible_existing_blob, warn_if_missing=False
+                    )
             for ftm in files_to_move:
-                shutil.move(ftm, self._destination(ftm))
-                # todo store in blob storage as well (need a bucket for RFCs)
+                with ftm.open("rb") as f:
+                    store_file(
+                        kind=blob_kind,
+                        name=self._blob_destination(ftm),
+                        file=f,
+                        doc_name=rfc.name,
+                        doc_rev=rfc.rev,  # expect None, but match whatever it is
+                        mtime=mtime,
+                    )
+                shutil.move(ftm, self._fs_destination(ftm))
 
         return Response(NotificationAckSerializer().data)

From 4aeb36e01803cf1fc93cc931aa53438417a75900 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 16 Jan 2026 16:46:48 -0400
Subject: [PATCH 547/656] chore: order is_auth'd middleware correctly (#10225)

Needs to be earlier than ConditionalGetMiddleware or the flag
will not be set for 304 responses.
---
 ietf/settings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/settings.py b/ietf/settings.py
index fd8d86a1ab..1cda79e21b 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -454,6 +454,7 @@ def skip_unreadable_post(record):
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
+    "ietf.middleware.is_authenticated_header_middleware",
     "django.middleware.http.ConditionalGetMiddleware",
     "simple_history.middleware.HistoryRequestMiddleware",
     # comment in this to get logging of SQL insert and update statements:
@@ -464,7 +465,6 @@ def skip_unreadable_post(record):
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
     "django.middleware.security.SecurityMiddleware",
     "ietf.middleware.unicode_nfkc_normalization_middleware",
-    "ietf.middleware.is_authenticated_header_middleware",
 ]
 
 ROOT_URLCONF = 'ietf.urls'

From 3ff0154ce5044802c586961c8216e74393bbf6f6 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 16 Jan 2026 16:48:24 -0400
Subject: [PATCH 548/656] feat: include tickets in RegistrationResource
 (#10223)

---
 ietf/meeting/resources.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py
index 88562a88fe..490b75f925 100644
--- a/ietf/meeting/resources.py
+++ b/ietf/meeting/resources.py
@@ -21,7 +21,13 @@
                                  Attended,
                                  Registration, RegistrationTicket)
 
-from ietf.name.resources import MeetingTypeNameResource
+from ietf.name.resources import (
+    AttendanceTypeNameResource,
+    MeetingTypeNameResource,
+    RegistrationTicketTypeNameResource,
+)
+
+
 class MeetingResource(ModelResource):
     type             = ToOneField(MeetingTypeNameResource, 'type')
     schedule         = ToOneField('ietf.meeting.resources.ScheduleResource', 'schedule', null=True)
@@ -437,11 +443,16 @@ class Meta:
         }
 api.meeting.register(AttendedResource())
 
-from ietf.meeting.resources import MeetingResource
 from ietf.person.resources import PersonResource
 class RegistrationResource(ModelResource):
     meeting          = ToOneField(MeetingResource, 'meeting')
     person           = ToOneField(PersonResource, 'person', null=True)
+    tickets          = ToManyField(
+        'ietf.meeting.resources.RegistrationTicketResource',
+        'tickets',
+        full=True,
+    )
+    
     class Meta:
         queryset = Registration.objects.all()
         serializer = api.Serializer()
@@ -456,13 +467,17 @@ class Meta:
             "country_code": ALL,
             "email": ALL,
             "attended": ALL,
+            "checkedin": ALL,
             "meeting": ALL_WITH_RELATIONS,
             "person": ALL_WITH_RELATIONS,
+            "tickets": ALL_WITH_RELATIONS,
         }
 api.meeting.register(RegistrationResource())
 
 class RegistrationTicketResource(ModelResource):
     registration          = ToOneField(RegistrationResource, 'registration')
+    attendance_type       = ToOneField(AttendanceTypeNameResource, 'attendance_type')
+    ticket_type           = ToOneField(RegistrationTicketTypeNameResource, 'ticket_type')
     class Meta:
         queryset = RegistrationTicket.objects.all()
         serializer = api.Serializer()
@@ -471,8 +486,8 @@ class Meta:
         ordering = ['id', ]
         filtering = { 
             "id": ALL,
-            "ticket_type": ALL,
-            "attendance_type": ALL,
+            "ticket_type": ALL_WITH_RELATIONS,
+            "attendance_type": ALL_WITH_RELATIONS,
             "registration": ALL_WITH_RELATIONS,
         }
 api.meeting.register(RegistrationTicketResource())

From d53ceff5d408c0fc9daab4ebe7144e2507309fbf Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 21 Jan 2026 11:43:46 -0400
Subject: [PATCH 549/656] chore(dev): adjust dev settings for RFC file upload
 (#10287)

* chore(dev): disable nginx max_body_size

* chore(dev): create missing dest dir

Create the "prerelease" directory, if needed, when accepting RFC
publication files. Gated not to impact production, where this directory
should already exist.
---
 docker/configs/nginx-proxy.conf | 1 +
 ietf/api/views_rpc.py           | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/docker/configs/nginx-proxy.conf b/docker/configs/nginx-proxy.conf
index 3068cc71d7..5a9ae31ad0 100644
--- a/docker/configs/nginx-proxy.conf
+++ b/docker/configs/nginx-proxy.conf
@@ -4,6 +4,7 @@ server {
 
     proxy_read_timeout 1d;
     proxy_send_timeout 1d;
+    client_max_body_size 0;  # disable checking
     
     root /var/www/html;
     index index.html index.htm index.nginx-debian.html;
diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index 542836a857..ea9c6348ca 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -476,6 +476,12 @@ def post(self, request):
                         doc_rev=rfc.rev,  # expect None, but match whatever it is
                         mtime=mtime,
                     )
-                shutil.move(ftm, self._fs_destination(ftm))
+                destination = self._fs_destination(ftm)
+                if (
+                    settings.SERVER_MODE != "production"
+                    and not destination.parent.exists()
+                ):
+                    destination.parent.mkdir()
+                shutil.move(ftm, destination)
 
         return Response(NotificationAckSerializer().data)

From 50653e961556cf803b43707fad17c51077c68870 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 21 Jan 2026 11:44:49 -0400
Subject: [PATCH 550/656] fix: add Blob.__str__() (#10284)

Keeps the model in sync with mailarchive's verison
---
 ietf/blobdb/models.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ietf/blobdb/models.py b/ietf/blobdb/models.py
index fa7831f203..27325ada5d 100644
--- a/ietf/blobdb/models.py
+++ b/ietf/blobdb/models.py
@@ -64,6 +64,9 @@ class Meta:
             ),
         ]
 
+    def __str__(self):
+        return f"{self.bucket}:{self.name}"
+
     def save(self, **kwargs):
         db = get_blobdb()
         with transaction.atomic(using=db):

From 4ff48057e8dfc68d467effa396854dd7fdbede9a Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 21 Jan 2026 09:50:02 -0600
Subject: [PATCH 551/656] fix: restrict rfc bulkload to those with a Document
 object (#10289)

* fix: restrict rfc bulkload to those with a Document object

* fix: only query Document once

* test: update test to match check for existing Document
---
 ietf/sync/tests_utils.py |  2 ++
 ietf/sync/utils.py       | 64 ++++++++++++++++++++++------------------
 2 files changed, 38 insertions(+), 28 deletions(-)

diff --git a/ietf/sync/tests_utils.py b/ietf/sync/tests_utils.py
index eb4b4ddf74..bb4a859e30 100644
--- a/ietf/sync/tests_utils.py
+++ b/ietf/sync/tests_utils.py
@@ -5,6 +5,7 @@
 
 from django.test import override_settings
 from ietf import settings
+from ietf.doc.factories import RfcFactory
 from ietf.doc.storage_utils import exists_in_storage, retrieve_str
 from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper
 from ietf.utils.test_utils import TestCase
@@ -59,6 +60,7 @@ def test_load_rfcs_into_blobdb(self):
                 rfc_path = Path(faux_rfc_path)
                 (rfc_path / "prerelease").mkdir()
                 for num in [12345, 54321]:
+                    RfcFactory(rfc_number=num)
                     for ext in settings.RFC_FILE_TYPES + ("json",):
                         with (rfc_path / f"rfc{num}.{ext}").open("w") as f:
                             f.write(ext)
diff --git a/ietf/sync/utils.py b/ietf/sync/utils.py
index 5b5f8ff559..b3bdd8d206 100644
--- a/ietf/sync/utils.py
+++ b/ietf/sync/utils.py
@@ -6,8 +6,8 @@
 from pathlib import Path
 
 from django.conf import settings
-
 from ietf.utils import log
+from ietf.doc.models import Document
 from ietf.doc.storage_utils import AlreadyExistsError, store_bytes
 
 
@@ -26,17 +26,42 @@ def build_from_file_content(rfc_numbers: list[int]) -> str:
 
 def load_rfcs_into_blobdb(numbers: list[int]):
     types_to_load = settings.RFC_FILE_TYPES + ("json",)
+    rfc_docs = Document.objects.filter(type="rfc", rfc_number__in=numbers).values_list("rfc_number", flat=True)
     for num in numbers:
-        for ext in types_to_load:
-            fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}"
-            if fs_path.is_file():
-                with fs_path.open("rb") as f:
+        if num in rfc_docs:
+            for ext in types_to_load:
+                fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}"
+                if fs_path.is_file():
+                    with fs_path.open("rb") as f:
+                        bytes = f.read()
+                    mtime = fs_path.stat().st_mtime
+                    try:
+                        store_bytes(
+                            kind="rfc",
+                            name=f"{ext}/rfc{num}.{ext}",
+                            content=bytes,
+                            allow_overwrite=False,  # Intentionally not allowing overwrite.
+                            doc_name=f"rfc{num}",
+                            doc_rev=None,
+                            # Not setting content_type
+                            mtime=datetime.datetime.fromtimestamp(
+                                mtime, tz=datetime.UTC
+                            ),
+                        )
+                    except AlreadyExistsError as e:
+                        log.log(str(e))
+
+            # store the not-prepped xml
+            name = f"rfc{num}.notprepped.xml"
+            source = Path(settings.RFC_PATH) / "prerelease" / name
+            if source.is_file():
+                with open(source, "rb") as f:
                     bytes = f.read()
-                mtime = fs_path.stat().st_mtime
+                mtime = source.stat().st_mtime
                 try:
                     store_bytes(
                         kind="rfc",
-                        name=f"{ext}/rfc{num}.{ext}",
+                        name=f"notprepped/{name}",
                         content=bytes,
                         allow_overwrite=False,  # Intentionally not allowing overwrite.
                         doc_name=f"rfc{num}",
@@ -46,24 +71,7 @@ def load_rfcs_into_blobdb(numbers: list[int]):
                     )
                 except AlreadyExistsError as e:
                     log.log(str(e))
-
-        # store the not-prepped xml
-        name = f"rfc{num}.notprepped.xml"
-        source = Path(settings.RFC_PATH) / "prerelease" / name
-        if source.is_file():
-            with open(source, "rb") as f:
-                bytes = f.read()
-            mtime = source.stat().st_mtime
-            try:
-                store_bytes(
-                    kind="rfc",
-                    name=f"notprepped/{name}",
-                    content=bytes,
-                    allow_overwrite=False,  # Intentionally not allowing overwrite.
-                    doc_name=f"rfc{num}",
-                    doc_rev=None,
-                    # Not setting content_type
-                    mtime=datetime.datetime.fromtimestamp(mtime, tz=datetime.UTC),
-                )
-            except AlreadyExistsError as e:
-                log.log(str(e))
+        else:
+            log.log(
+                f"Skipping loading rfc{num} into blobdb as no matching Document exists"
+            )

From a28594eecbe5000950ccd6a12bcadd5c61018183 Mon Sep 17 00:00:00 2001
From: Ryan Cross <rcross@amsl.com>
Date: Wed, 21 Jan 2026 09:55:37 -0800
Subject: [PATCH 552/656] feat: add option to email users about duplicate
 accounts. Fixes #8174. (#9850)

* feat: add option to email users about duplicate accounts. Fixes #8174.

* fix: split person merge into two views

* fix: use form for validation

* fix: update text of merge request email

* fix: update copyright date

* fix: use custom field classes in MergeRequestForm
---
 ietf/person/forms.py                          | 21 ++++-
 ietf/person/tests.py                          | 34 ++++++--
 ietf/person/urls.py                           |  4 +
 ietf/person/views.py                          | 86 ++++++++++++++++---
 ietf/templates/person/merge.html              | 36 +-------
 ietf/templates/person/merge_request_email.txt | 23 +++++
 ietf/templates/person/merge_submit.html       | 57 ++++++++++++
 ietf/templates/person/send_merge_request.html | 20 +++++
 ietf/utils/fields.py                          | 23 ++++-
 9 files changed, 254 insertions(+), 50 deletions(-)
 create mode 100644 ietf/templates/person/merge_request_email.txt
 create mode 100644 ietf/templates/person/merge_submit.html
 create mode 100644 ietf/templates/person/send_merge_request.html

diff --git a/ietf/person/forms.py b/ietf/person/forms.py
index 81ee362561..7eef8aa17b 100644
--- a/ietf/person/forms.py
+++ b/ietf/person/forms.py
@@ -1,15 +1,26 @@
-# Copyright The IETF Trust 2018-2020, All Rights Reserved
+# Copyright The IETF Trust 2018-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
 from django import forms
+
 from ietf.person.models import Person
+from ietf.utils.fields import MultiEmailField, NameAddrEmailField
 
 
 class MergeForm(forms.Form):
     source = forms.IntegerField(label='Source Person ID')
     target = forms.IntegerField(label='Target Person ID')
 
+    def __init__(self, *args, **kwargs):
+        self.readonly = False
+        if 'readonly' in kwargs:
+            self.readonly = kwargs.pop('readonly')
+        super().__init__(*args, **kwargs)
+        if self.readonly:
+            self.fields['source'].widget.attrs['readonly'] = True
+            self.fields['target'].widget.attrs['readonly'] = True
+
     def clean_source(self):
         return self.get_person(self.cleaned_data['source'])
 
@@ -21,3 +32,11 @@ def get_person(self, pk):
             return Person.objects.get(pk=pk)
         except Person.DoesNotExist:
             raise forms.ValidationError("ID does not exist")
+
+
+class MergeRequestForm(forms.Form):
+    to = MultiEmailField()
+    frm = NameAddrEmailField()
+    reply_to = MultiEmailField()
+    subject = forms.CharField()
+    body = forms.CharField(widget=forms.Textarea)
diff --git a/ietf/person/tests.py b/ietf/person/tests.py
index 6326362fd8..f55d8b8a34 100644
--- a/ietf/person/tests.py
+++ b/ietf/person/tests.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2014-2022, All Rights Reserved
+# Copyright The IETF Trust 2014-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -10,7 +10,6 @@
 from PIL import Image
 from pyquery import PyQuery
 
-
 from django.core.exceptions import ValidationError
 from django.http import HttpRequest
 from django.test import override_settings
@@ -23,6 +22,7 @@
 from ietf.community.models import CommunityList
 from ietf.group.factories import RoleFactory
 from ietf.group.models import Group
+from ietf.message.models import Message
 from ietf.nomcom.models import NomCom
 from ietf.nomcom.test_data import nomcom_test_data
 from ietf.nomcom.factories import NomComFactory, NomineeFactory, NominationFactory, FeedbackFactory, PositionFactory
@@ -208,13 +208,13 @@ def test_merge(self):
     def test_merge_with_params(self):
         p1 = get_person_no_user()
         p2 = PersonFactory()
-        url = urlreverse("ietf.person.views.merge") + "?source={}&target={}".format(p1.pk, p2.pk)
+        url = urlreverse("ietf.person.views.merge_submit") + "?source={}&target={}".format(p1.pk, p2.pk)
         login_testing_unauthorized(self, "secretary", url)
         r = self.client.get(url)
         self.assertContains(r, 'retaining login', status_code=200)
 
     def test_merge_with_params_bad_id(self):
-        url = urlreverse("ietf.person.views.merge") + "?source=1000&target=2000"
+        url = urlreverse("ietf.person.views.merge_submit") + "?source=1000&target=2000"
         login_testing_unauthorized(self, "secretary", url)
         r = self.client.get(url)
         self.assertContains(r, 'ID does not exist', status_code=200)
@@ -222,7 +222,7 @@ def test_merge_with_params_bad_id(self):
     def test_merge_post(self):
         p1 = get_person_no_user()
         p2 = PersonFactory()
-        url = urlreverse("ietf.person.views.merge")
+        url = urlreverse("ietf.person.views.merge_submit")
         expected_url = urlreverse("ietf.secr.rolodex.views.view", kwargs={'id': p2.pk})
         login_testing_unauthorized(self, "secretary", url)
         data = {'source': p1.pk, 'target': p2.pk}
@@ -451,6 +451,30 @@ def test_dots(self):
         ncchair = RoleFactory(group__acronym='nomcom2020',group__type_id='nomcom',name_id='chair').person
         self.assertEqual(get_dots(ncchair),['nomcom'])
 
+    def test_send_merge_request(self):
+        empty_outbox()
+        message_count_before = Message.objects.count()
+        source = PersonFactory()
+        target = PersonFactory()
+        url = urlreverse('ietf.person.views.send_merge_request')
+        url = url + f'?source={source.pk}&target={target.pk}'
+        login_testing_unauthorized(self, 'secretary', url)
+        r = self.client.get(url)
+        initial = r.context['form'].initial
+        subject = 'Action requested: Merging possible duplicate IETF Datatracker accounts'
+        self.assertEqual(initial['to'], ', '.join([source.user.username, target.user.username]))
+        self.assertEqual(initial['subject'], subject)
+        self.assertEqual(initial['reply_to'], 'support@ietf.org')
+        self.assertEqual(r.status_code, 200)
+        r = self.client.post(url, data=initial)
+        self.assertEqual(r.status_code, 302)
+        self.assertEqual(len(outbox), 1)
+        self.assertIn(source.user.username, outbox[0]['To'])
+        message_count_after = Message.objects.count()
+        message = Message.objects.last()
+        self.assertEqual(message_count_after, message_count_before + 1)
+        self.assertIn(source.user.username, message.to)
+
 
 class TaskTests(TestCase):
     @mock.patch("ietf.person.tasks.log.log")
diff --git a/ietf/person/urls.py b/ietf/person/urls.py
index 867646fe39..f3eccd04b7 100644
--- a/ietf/person/urls.py
+++ b/ietf/person/urls.py
@@ -1,8 +1,12 @@
+# Copyright The IETF Trust 2009-2025, All Rights Reserved
+# -*- coding: utf-8 -*-
 from ietf.person import views, ajax
 from ietf.utils.urls import url
 
 urlpatterns = [
     url(r'^merge/?$', views.merge),
+    url(r'^merge/submit/?$', views.merge_submit),
+    url(r'^merge/send_request/?$', views.send_merge_request),
     url(r'^search/(?P<model_name>(person|email))/$', views.ajax_select2_search),
     url(r'^(?P<personid>[0-9]+)/email.json$', ajax.person_email_json),
     url(r'^(?P<email_or_name>[^/]+)$', views.profile),
diff --git a/ietf/person/views.py b/ietf/person/views.py
index a37b164311..d0b5912431 100644
--- a/ietf/person/views.py
+++ b/ietf/person/views.py
@@ -1,14 +1,16 @@
-# Copyright The IETF Trust 2012-2020, All Rights Reserved
+# Copyright The IETF Trust 2012-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
 from io import StringIO, BytesIO
 from PIL import Image
 
+from django.conf import settings
 from django.contrib import messages
 from django.db.models import Q
 from django.http import HttpResponse, Http404
 from django.shortcuts import render, redirect
+from django.template.loader import render_to_string
 from django.utils import timezone
 
 import debug                            # pyflakes:ignore
@@ -16,8 +18,9 @@
 from ietf.ietfauth.utils import role_required
 from ietf.person.models import Email, Person
 from ietf.person.fields import select2_id_name_json
-from ietf.person.forms import MergeForm
+from ietf.person.forms import MergeForm, MergeRequestForm
 from ietf.person.utils import handle_users, merge_persons, lookup_persons
+from ietf.utils.mail import send_mail_text
 
 
 def ajax_select2_search(request, model_name):
@@ -98,16 +101,19 @@ def photo(request, email_or_name):
 @role_required("Secretariat")
 def merge(request):
     form = MergeForm()
-    method = 'get'
+    return render(request, 'person/merge.html', {'form': form})
+
+
+@role_required("Secretariat")
+def merge_submit(request):
     change_details = ''
     warn_messages = []
     source = None
     target = None
 
     if request.method == "GET":
-        form = MergeForm()
         if request.GET:
-            form = MergeForm(request.GET)
+            form = MergeForm(request.GET, readonly=True)
             if form.is_valid():
                 source = form.cleaned_data.get('source')
                 target = form.cleaned_data.get('target')
@@ -116,12 +122,9 @@ def merge(request):
                     if source.user.last_login and target.user.last_login and source.user.last_login > target.user.last_login:
                         warn_messages.append('WARNING: The most recently used login is being deleted!')
                 change_details = handle_users(source, target, check_only=True)
-                method = 'post'
-            else:
-                method = 'get'
 
     if request.method == "POST":
-        form = MergeForm(request.POST)
+        form = MergeForm(request.POST, readonly=True)
         if form.is_valid():
             source = form.cleaned_data.get('source')
             source_id = source.id
@@ -136,11 +139,72 @@ def merge(request):
                 messages.error(request, output)
             return redirect('ietf.secr.rolodex.views.view', id=target.pk)
 
-    return render(request, 'person/merge.html', {
+    return render(request, 'person/merge_submit.html', {
         'form': form,
-        'method': method,
         'change_details': change_details,
         'source': source,
         'target': target,
         'warn_messages': warn_messages,
     })
+
+
+@role_required("Secretariat")
+def send_merge_request(request):
+    if request.method == 'GET':
+        merge_form = MergeForm(request.GET)
+        if merge_form.is_valid():
+            source = merge_form.cleaned_data['source']
+            target = merge_form.cleaned_data['target']
+            to = []
+            if source.email():
+                to.append(source.email().address)
+            if target.email():
+                to.append(target.email().address)
+            if source.user:
+                source_account = source.user.username
+            else:
+                source_account = source.email()
+            if target.user:
+                target_account = target.user.username
+            else:
+                target_account = target.email()
+            sender_name = request.user.person.name
+            subject = 'Action requested: Merging possible duplicate IETF Datatracker accounts'
+            context = {
+                'source_account': source_account,
+                'target_account': target_account,
+                'sender_name': sender_name,
+            }
+            body = render_to_string('person/merge_request_email.txt', context)
+            initial = {
+                'to': ', '.join(to),
+                'frm': settings.DEFAULT_FROM_EMAIL,
+                'reply_to': 'support@ietf.org',
+                'subject': subject,
+                'body': body,
+                'by': request.user.person.pk,
+            }
+            form = MergeRequestForm(initial=initial)
+        else:
+            messages.error(request, "Error requesting merge email: " + merge_form.errors.as_text())
+            return redirect("ietf.person.views.merge")
+
+    if request.method == 'POST':
+        form = MergeRequestForm(request.POST)
+        if form.is_valid():
+            extra = {"Reply-To": form.cleaned_data.get("reply_to")}
+            send_mail_text(
+                request,
+                form.cleaned_data.get("to"),
+                form.cleaned_data.get("frm"),
+                form.cleaned_data.get("subject"),
+                form.cleaned_data.get("body"),
+                extra=extra,
+            )
+
+            messages.success(request, "The merge confirmation email was sent.")
+            return redirect("ietf.person.views.merge")
+
+    return render(request, "person/send_merge_request.html", {
+        "form": form,
+    })
diff --git a/ietf/templates/person/merge.html b/ietf/templates/person/merge.html
index 36499ecdbc..5c3e6b0938 100644
--- a/ietf/templates/person/merge.html
+++ b/ietf/templates/person/merge.html
@@ -1,5 +1,5 @@
+{# Copyright The IETF Trust 2018-2025, All Rights Reserved #}
 {% extends "base.html" %}
-{# Copyright The IETF Trust 2015, All Rights Reserved #}
 {% load static %}
 {% load django_bootstrap5 %}
 {% block title %}Merge Persons{% endblock %}
@@ -8,45 +8,17 @@ <h1>Merge Person Records</h1>
     <p class="alert alert-info my-3">
         This tool will merge two Person records into one.  If both records have logins and you want to retain the one on the left, use the Swap button to swap source and target records.
     </p>
-    <form method="{{ method }}">
-        {% if method == 'post' %}
-            {% csrf_token %}
-        {% endif %}
+    <form method="GET" action="{% url 'ietf.person.views.merge_submit' %}">
         <div class="row mt-3">
             <div class="col-md-6">
                 {% bootstrap_field form.source %}
-                {% if source %}
-                    {% with person=source %}
-                        {% include "person/person_info.html" %}
-                    {% endwith %}
-                {% endif %}
             </div>
             <div class="col-md-6">
                 {% bootstrap_field form.target %}
-                {% if target %}
-                    {% with person=target %}
-                        {% include "person/person_info.html" %}
-                    {% endwith %}
-                {% endif %}
             </div>
         </div>
-        {% if change_details %}<div class="alert alert-info my-3" role="alert">{{ change_details }}</div>{% endif %}
-        {% if warn_messages %}
-            {% for message in warn_messages %}<div class="alert alert-warning my-3" role="alert">{{ message }}</div>{% endfor %}
-        {% endif %}
-        {% if method == 'post' %}
-            <a class="btn btn-primary"
-               href="{% url 'ietf.person.views.merge' %}?source={{ target.pk }}&amp;target={{ source.pk }}"
-               role="button">
-                Swap
-            </a>
-        {% endif %}
-        <button type="submit" class="btn btn-warning">
-            {% if method == 'post' %}
-                Merge
-            {% else %}
-                Submit
-            {% endif %}
+        <button type="submit" class="btn btn-warning" title="Get Person information">
+            Get Info
         </button>
     </form>
 {% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/person/merge_request_email.txt b/ietf/templates/person/merge_request_email.txt
new file mode 100644
index 0000000000..0a695f036c
--- /dev/null
+++ b/ietf/templates/person/merge_request_email.txt
@@ -0,0 +1,23 @@
+Hello,
+
+We have identified multiple IETF Datatracker accounts that may represent a single person:
+
+https://datatracker.ietf.org/person/{{ source_account }}
+
+and 
+
+https://datatracker.ietf.org/person/{{ target_account }}
+
+If this is so then it is important that we merge the accounts.
+
+This email is being sent to the primary emails associated with each Datatracker account.
+
+Please respond to this message individually from the email account(s) you control so we can take the appropriate action.
+
+If these should be merged, please identify which account you would like to keep the login credentials from.
+
+If you are associated with but no longer have access to one of the email accounts, then please let us know and we will follow up to determine how to proceed.
+
+
+{{ sender_name }}
+IETF Support
\ No newline at end of file
diff --git a/ietf/templates/person/merge_submit.html b/ietf/templates/person/merge_submit.html
new file mode 100644
index 0000000000..30e1999f81
--- /dev/null
+++ b/ietf/templates/person/merge_submit.html
@@ -0,0 +1,57 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% extends "base.html" %}
+{% load static %}
+{% load django_bootstrap5 %}
+{% block title %}Merge Persons{% endblock %}
+{% block content %}
+    <h1>Merge Person Records</h1>
+    <p class="alert alert-info my-3">
+        This tool will merge two Person records into one.  If both records have logins and you want to retain the one on the left, use the Swap button to swap source and target records.
+    </p>
+    <form method="post">
+        {% csrf_token %}
+        <div class="row mt-3">
+            <div class="col-md-6">
+                {% bootstrap_field form.source %}
+                {% if source %}
+                    {% with person=source %}
+                        {% include "person/person_info.html" %}
+                    {% endwith %}
+                {% endif %}
+            </div>
+            <div class="col-md-6">
+                {% bootstrap_field form.target %}
+                {% if target %}
+                    {% with person=target %}
+                        {% include "person/person_info.html" %}
+                    {% endwith %}
+                {% endif %}
+            </div>
+        </div>
+        {% if change_details %}<div class="alert alert-info my-3" role="alert">{{ change_details }}</div>{% endif %}
+        {% if warn_messages %}
+            {% for message in warn_messages %}<div class="alert alert-warning my-3" role="alert">{{ message }}</div>{% endfor %}
+        {% endif %}
+
+        <a class="btn btn-primary"
+           href="{% url 'ietf.person.views.merge_submit' %}?source={{ target.pk }}&amp;target={{ source.pk }}"
+           role="button"
+           title="Swap source and target">
+            Swap
+        </a>
+
+        <button type="submit" class="btn btn-warning" title="Merge Person records">
+            Merge
+        </button>
+
+        <a class="btn btn-primary"
+           href="{% url 'ietf.person.views.send_merge_request' %}?source={{ source.pk }}&amp;target={{ target.pk }}"
+           role="button"
+           title="Send email to both addresses to confirm they are the same person">
+            Send Email
+        </a>
+
+        <a class="btn btn-primary" href="{% url 'ietf.person.views.merge' %}" role="button">Back</a>
+
+    </form>
+{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/person/send_merge_request.html b/ietf/templates/person/send_merge_request.html
new file mode 100644
index 0000000000..f0c6272dca
--- /dev/null
+++ b/ietf/templates/person/send_merge_request.html
@@ -0,0 +1,20 @@
+{# Copyright The IETF Trust 2025, All Rights Reserved #}
+{% extends "base.html" %}
+{% load static %}
+{% load django_bootstrap5 %}
+{% block title %}Send Merge Notice{% endblock %}
+{% block content %}
+    <h1>Send Merge Notice</h1>
+    {% if form.non_field_errors %}<div class="my-3 alert alert-danger">{{ form.non_field_errors }}</div>{% endif %}
+    <form id="email-form" method="post">
+        {% csrf_token %}
+        {% bootstrap_field form.to layout='horizontal' %}
+        {% bootstrap_field form.frm layout='horizontal' %}
+        {% bootstrap_field form.reply_to layout='horizontal' %}
+        {% bootstrap_field form.subject layout='horizontal' %}
+        {% bootstrap_field form.body layout='horizontal' %}
+        <button type="submit" class="btn btn-primary">Send</button>
+        <a class="btn btn-secondary float-end"
+               href="{% url 'ietf.person.views.merge' %}">Cancel</a>
+    </form>
+{% endblock %}
diff --git a/ietf/utils/fields.py b/ietf/utils/fields.py
index ba3fecebc6..6e8765612f 100644
--- a/ietf/utils/fields.py
+++ b/ietf/utils/fields.py
@@ -1,10 +1,11 @@
-# Copyright The IETF Trust 2012-2020, All Rights Reserved
+# Copyright The IETF Trust 2012-2025, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
 import datetime
 import json
 import re
+from email.utils import parseaddr
 
 import debug                            # pyflakes:ignore
 
@@ -16,6 +17,7 @@
 from django.core.exceptions import ValidationError
 from django.utils.dateparse import parse_duration
 
+
 class MultiEmailField(forms.Field):
     def to_python(self, value):
         "Normalize data to a list of strings."
@@ -38,6 +40,25 @@ def validate(self, value):
         for email in value:
             validate_email(email)
 
+
+def validate_name_addr_email(value):
+    "Validate name-addr style email address"
+    name, addr = parseaddr(value)
+    if not addr:
+        raise ValidationError("Invalid email format.")
+    try:
+        validate_email(addr)  # validate the actual address part
+    except ValidationError:
+        raise ValidationError("Invalid email address.")
+
+
+class NameAddrEmailField(forms.CharField):
+    def validate(self, value):
+        "Check if value consists only of valid emails."
+        super().validate(value)
+        validate_name_addr_email(value)
+
+
 def yyyymmdd_to_strftime_format(fmt):
     translation_table = sorted([
         ("yyyy", "%Y"),

From 5d2790bb3cfff4e45be076d56dcc0fb209ed327e Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 22 Jan 2026 11:52:17 -0400
Subject: [PATCH 553/656] feat: additional fields for purple API (#10299)

* feat: include group, abstract in purple draft API

* feat: include ad/shepherd in purple API
---
 ietf/api/serializers_rpc.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py
index f2e735be7a..fe7f609251 100644
--- a/ietf/api/serializers_rpc.py
+++ b/ietf/api/serializers_rpc.py
@@ -114,11 +114,14 @@ class FullDraftSerializer(serializers.ModelSerializer):
     # is used for a writeable view, the validation will need to be added back.
     name = serializers.CharField(max_length=255)
     title = serializers.CharField(max_length=255)
+    group = serializers.SlugRelatedField(slug_field="acronym", read_only=True)
 
     # Other fields we need to add / adjust
     source_format = serializers.SerializerMethodField()
     authors = DocumentAuthorSerializer(many=True, source="documentauthor_set")
-    shepherd = serializers.SerializerMethodField()
+    shepherd = serializers.PrimaryKeyRelatedField(
+        source="shepherd.person", read_only=True
+    )
     consensus = serializers.SerializerMethodField()
 
     class Meta:
@@ -129,12 +132,16 @@ class Meta:
             "rev",
             "stream",
             "title",
+            "group",
+            "abstract",
             "pages",
             "source_format",
             "authors",
             "shepherd",
             "intended_std_level",
             "consensus",
+            "shepherd",
+            "ad",
         ]
 
     def get_consensus(self, doc: Document) -> Optional[bool]:
@@ -155,12 +162,6 @@ def get_source_format(
             return "txt"
         return "unknown"
 
-    @extend_schema_field(OpenApiTypes.EMAIL)
-    def get_shepherd(self, doc: Document) -> str:
-        if doc.shepherd:
-            return doc.shepherd.formatted_ascii_email()
-        return ""
-
 
 class DraftSerializer(FullDraftSerializer):
     class Meta:
@@ -171,6 +172,7 @@ class Meta:
             "rev",
             "stream",
             "title",
+            "group",
             "pages",
             "source_format",
             "authors",

From f56bfcb2f618fbaded92519797d4e4baa34ac4a1 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 23 Jan 2026 10:32:02 -0400
Subject: [PATCH 554/656] feat: more clickable message admin (#10307)

---
 ietf/message/admin.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ietf/message/admin.py b/ietf/message/admin.py
index 250e1eb596..6a876cdc70 100644
--- a/ietf/message/admin.py
+++ b/ietf/message/admin.py
@@ -27,7 +27,8 @@ def queryset(self, request, queryset):
 
 
 class MessageAdmin(admin.ModelAdmin):
-    list_display = ["sent_status", "subject", "by", "time", "groups"]
+    list_display = ["sent_status", "display_subject", "by", "time", "groups"]
+    list_display_links = ["display_subject"]
     search_fields = ["subject", "body"]
     raw_id_fields = ["by", "related_groups", "related_docs"]
     list_filter = [
@@ -37,6 +38,10 @@ class MessageAdmin(admin.ModelAdmin):
     ordering = ["-time"]
     actions = ["retry_send"]
 
+    @admin.display(description="Subject", empty_value="(no subject)")
+    def display_subject(self, instance):
+        return instance.subject or None  # None triggers the empty_value
+
     def groups(self, instance):
         return ", ".join(g.acronym for g in instance.related_groups.all())
 

From 9c6fa92b7eed8b29bea96cb73148dba130678e63 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 27 Jan 2026 11:34:17 -0400
Subject: [PATCH 555/656] fix: update RPC publish API fields (#10308)

* fix: update purple publish API fields

* fix: handle IntegrityError more cleanly

* fix: don't import RFC fields from draft

* test: update test

* chore: remove unused var/import

* fix: f-string -> string
---
 ietf/api/serializers_rpc.py | 29 ++----------------------
 ietf/api/tests_views_rpc.py | 44 ++++++++++++++++++-------------------
 ietf/api/views_rpc.py       | 16 +++++++++++++-
 3 files changed, 39 insertions(+), 50 deletions(-)

diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py
index fe7f609251..440c2a73d4 100644
--- a/ietf/api/serializers_rpc.py
+++ b/ietf/api/serializers_rpc.py
@@ -27,7 +27,7 @@
     update_rfcauthors,
 )
 from ietf.group.models import Group
-from ietf.name.models import StreamName, StdLevelName, FormalLanguageName
+from ietf.name.models import StreamName, StdLevelName
 from ietf.person.models import Person
 from ietf.utils import log
 
@@ -137,7 +137,6 @@ class Meta:
             "pages",
             "source_format",
             "authors",
-            "shepherd",
             "intended_std_level",
             "consensus",
             "shepherd",
@@ -263,15 +262,6 @@ class RfcPubSerializer(serializers.ModelSerializer):
     stream = serializers.PrimaryKeyRelatedField(
         queryset=StreamName.objects.filter(used=True)
     )
-    formal_languages = serializers.PrimaryKeyRelatedField(
-        many=True,
-        required=False,
-        queryset=FormalLanguageName.objects.filter(used=True),
-        help_text=(
-            "formal languages used in RFC (defaults to those from draft, send empty"
-            "list to override)"
-        )
-    )
     std_level = serializers.PrimaryKeyRelatedField(
         queryset=StdLevelName.objects.filter(used=True),
     )
@@ -315,11 +305,8 @@ class Meta:
             "stream",
             "abstract",
             "pages",
-            "words",
-            "formal_languages",
             "std_level",
             "ad",
-            "note",
             "obsoletes",
             "updates",
             "subseries",
@@ -353,9 +340,6 @@ def create(self, validated_data):
         # If specified, retrieve draft and extract RFC default values from it
         if draft_name is None:
             draft = None
-            defaults_from_draft = {
-                "group": Group.objects.get(acronym="none", type_id="individ"),
-            }
         else:
             # validation enforces that draft_name and draft_rev are both present
             draft = Document.objects.filter(
@@ -378,17 +362,11 @@ def create(self, validated_data):
                     },
                     code="already-published-draft",
                 )
-            defaults_from_draft = {
-                "ad": draft.ad,
-                "formal_languages": draft.formal_languages.all(),
-                "group": draft.group,
-                "note": draft.note,
-            }
 
         # Transaction to clean up if something fails
         with transaction.atomic():
             # create rfc, letting validated request data override draft defaults
-            rfc = self._create_rfc(defaults_from_draft | validated_data)
+            rfc = self._create_rfc(validated_data)
             DocEvent.objects.create(
                 doc=rfc,
                 rev=rfc.rev,
@@ -523,14 +501,11 @@ def create(self, validated_data):
 
     def _create_rfc(self, validated_data):
         authors_data = validated_data.pop("authors")
-        formal_languages = validated_data.pop("formal_languages", [])
-        # todo ad field
         rfc = Document.objects.create(
             type_id="rfc",
             name=f"rfc{validated_data['rfc_number']}",
             **validated_data,
         )
-        rfc.formal_languages.set(formal_languages)  # list of PKs is ok
         for order, author_data in enumerate(authors_data):
             rfc.rfcauthor_set.create(
                 order=order,
diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py
index ecb50ee76c..09fb40bf6e 100644
--- a/ietf/api/tests_views_rpc.py
+++ b/ietf/api/tests_views_rpc.py
@@ -80,9 +80,15 @@ def test_draftviewset_references(self):
     def test_notify_rfc_published(self):
         url = urlreverse("ietf.api.purple_api.notify_rfc_published")
         area = GroupFactory(type_id="area")
+        rfc_group = GroupFactory(type_id="wg")
         draft_ad = RoleFactory(group=area, name_id="ad").person
-        authors = PersonFactory.create_batch(2)
-        draft = WgDraftFactory(group__parent=area, authors=authors)
+        rfc_ad = PersonFactory()
+        draft_authors = PersonFactory.create_batch(2)
+        rfc_authors = PersonFactory.create_batch(3)
+        draft = WgDraftFactory(
+            group__parent=area, authors=draft_authors, ad=draft_ad, stream_id="ietf"
+        )
+        rfc_stream_id = "ise"
         assert isinstance(draft, Document), "WgDraftFactory should generate a Document"
         unused_rfc_number = (
             Document.objects.filter(rfc_number__isnull=False).aggregate(
@@ -96,7 +102,7 @@ def test_notify_rfc_published(self):
             "draft_name": draft.name,
             "draft_rev": draft.rev,
             "rfc_number": unused_rfc_number,
-            "title": draft.title,
+            "title": "RFC " + draft.title,
             "authors": [
                 {
                     "titlepage_name": f"titlepage {author.name}",
@@ -106,17 +112,14 @@ def test_notify_rfc_published(self):
                     "affiliation": "Some Affiliation",
                     "country": "CA",
                 }
-                for author in authors
+                for author in rfc_authors
             ],
-            "group": draft.group.acronym,
-            "stream": draft.stream_id,
-            "abstract": draft.abstract,
-            "pages": draft.pages,
-            "words": draft.pages * 250,
-            "formal_languages": [],
+            "group": rfc_group.acronym,
+            "stream": rfc_stream_id,
+            "abstract": "RFC version of " + draft.abstract,
+            "pages": draft.pages + 10,
             "std_level": "ps",
-            "ad": draft_ad.pk,
-            "note": "noted",
+            "ad": rfc_ad.pk,
             "obsoletes": [],
             "updates": [],
             "subseries": [],
@@ -137,7 +140,7 @@ def test_notify_rfc_published(self):
             ).count(),
             1,
         )
-        self.assertEqual(rfc.title, draft.title)
+        self.assertEqual(rfc.title, "RFC " + draft.title)
         self.assertEqual(rfc.documentauthor_set.count(), 0)
         self.assertEqual(
             list(
@@ -159,18 +162,15 @@ def test_notify_rfc_published(self):
                     "affiliation": "Some Affiliation",
                     "country": "CA",
                 }
-                for author in authors
+                for author in rfc_authors
             ],
         )
-        self.assertEqual(rfc.group, draft.group)
-        self.assertEqual(rfc.stream, draft.stream)
-        self.assertEqual(rfc.abstract, draft.abstract)
-        self.assertEqual(rfc.pages, draft.pages)
-        self.assertEqual(rfc.words, draft.pages * 250)
-        self.assertEqual(rfc.formal_languages.count(), 0)
+        self.assertEqual(rfc.group, rfc_group)
+        self.assertEqual(rfc.stream_id, rfc_stream_id)
+        self.assertEqual(rfc.abstract, "RFC version of " + draft.abstract)
+        self.assertEqual(rfc.pages, draft.pages + 10)
         self.assertEqual(rfc.std_level_id, "ps")
-        self.assertEqual(rfc.ad, draft_ad)
-        self.assertEqual(rfc.note, "noted")
+        self.assertEqual(rfc.ad, rfc_ad)
         self.assertEqual(rfc.related_that_doc("obs"), [])
         self.assertEqual(rfc.related_that_doc("updates"), [])
         self.assertEqual(rfc.part_of(), [])
diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index ea9c6348ca..6b1799f654 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -5,6 +5,7 @@
 from tempfile import TemporaryDirectory
 
 from django.conf import settings
+from django.db import IntegrityError
 from drf_spectacular.utils import OpenApiParameter
 from rest_framework import mixins, parsers, serializers, viewsets, status
 from rest_framework.decorators import action
@@ -360,7 +361,20 @@ def post(self, request):
         serializer = RfcPubSerializer(data=request.data)
         serializer.is_valid(raise_exception=True)
         # Create RFC
-        serializer.save()
+        try:
+            serializer.save()
+        except IntegrityError as err:
+            if Document.objects.filter(
+                rfc_number=serializer.validated_data["rfc_number"]
+            ):
+                raise serializers.ValidationError(
+                    "RFC with that number already exists",
+                    code="rfc-number-in-use",
+                )
+            raise serializers.ValidationError(
+                f"Unable to publish: {err}",
+                code="unknown-integrity-error",
+            )
         return Response(NotificationAckSerializer().data)
 
 

From 33fe0bcb7cafadc2096bfd1386c3d7e8a6a915f5 Mon Sep 17 00:00:00 2001
From: Rudi Matz <rudi@staff.ietf.org>
Date: Wed, 28 Jan 2026 13:47:48 -0500
Subject: [PATCH 556/656] feat: add consensus in Draft serializer (#10327)

---
 ietf/api/serializers_rpc.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py
index 440c2a73d4..34e2c791c0 100644
--- a/ietf/api/serializers_rpc.py
+++ b/ietf/api/serializers_rpc.py
@@ -175,6 +175,7 @@ class Meta:
             "pages",
             "source_format",
             "authors",
+            "consensus",
         ]
 
 

From a174f43574c5ed4f20ddcc0d600d03e5156fb351 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 30 Jan 2026 14:49:47 -0400
Subject: [PATCH 557/656] fix: use current time for bofreq revisions  (#10333)

* fix: use current time for bofreq revisions

* test: test time handling

Adjusts assertion argument order to match our usual style
---
 ietf/doc/tests_bofreq.py | 19 ++++++++++++-------
 ietf/doc/views_bofreq.py |  1 -
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/ietf/doc/tests_bofreq.py b/ietf/doc/tests_bofreq.py
index 6a7c9393ef..6b142149be 100644
--- a/ietf/doc/tests_bofreq.py
+++ b/ietf/doc/tests_bofreq.py
@@ -307,17 +307,20 @@ def test_submit(self):
         url = urlreverse('ietf.doc.views_bofreq.submit', kwargs=dict(name=doc.name))
 
         rev = doc.rev
+        doc_time = doc.time 
         r = self.client.post(url,{'bofreq_submission':'enter','bofreq_content':'# oiwefrase'})
         self.assertEqual(r.status_code, 302)
         doc = reload_db_objects(doc)
-        self.assertEqual(rev, doc.rev)
+        self.assertEqual(doc.rev, rev)
+        self.assertEqual(doc.time, doc_time)
 
         nobody = PersonFactory()
         self.client.login(username=nobody.user.username, password=nobody.user.username+'+password')
         r = self.client.post(url,{'bofreq_submission':'enter','bofreq_content':'# oiwefrase'})
         self.assertEqual(r.status_code, 403)
         doc = reload_db_objects(doc)
-        self.assertEqual(rev, doc.rev)
+        self.assertEqual(doc.rev, rev)
+        self.assertEqual(doc.time, doc_time)
         self.client.logout()
 
         editor = bofreq_editors(doc).first()
@@ -339,12 +342,14 @@ def test_submit(self):
                         r = self.client.post(url, postdict)
                         self.assertEqual(r.status_code, 302)
                         doc = reload_db_objects(doc)
-                        self.assertEqual('%02d'%(int(rev)+1) ,doc.rev)
-                        self.assertEqual(f'# {username}', doc.text())
-                        self.assertEqual(f'# {username}', retrieve_str('bofreq',doc.get_base_name()))
-                        self.assertEqual(docevent_count+1, doc.docevent_set.count())
-                        self.assertEqual(1, len(outbox))
+                        self.assertEqual(doc.rev, '%02d'%(int(rev)+1))
+                        self.assertGreater(doc.time, doc_time)
+                        self.assertEqual(doc.text(), f'# {username}')
+                        self.assertEqual(retrieve_str('bofreq', doc.get_base_name()), f'# {username}')
+                        self.assertEqual(doc.docevent_set.count(), docevent_count+1)
+                        self.assertEqual(len(outbox), 1)
                         rev = doc.rev
+                        doc_time = doc.time
             finally:
                 os.unlink(file.name)
 
diff --git a/ietf/doc/views_bofreq.py b/ietf/doc/views_bofreq.py
index 71cbe30491..94e3960dfa 100644
--- a/ietf/doc/views_bofreq.py
+++ b/ietf/doc/views_bofreq.py
@@ -91,7 +91,6 @@ def submit(request, name):
                 by=request.user.person,
                 rev=bofreq.rev,
                 desc='New revision available',
-                time=bofreq.time,
             )
             bofreq.save_with_history([e])
             bofreq_submission = form.cleaned_data['bofreq_submission']

From 2dbe61e891752245447f4c2774353151f0bb34e5 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 9 Feb 2026 14:43:58 -0400
Subject: [PATCH 558/656] feat: speed up agenda.ics; cache more agenda data
 (#10362)

* chore(dev): cprofile middleware settings

* feat: precomputed agenda.ics (wip)

* feat: precomp filtering support

* fix: separately cache versioned hrefs

Fixes https://github.com/ietf-tools/datatracker/issues/10355

* fix: versionless agenda href in agenda / ical

* fix: preserve RESCHEDULED output

* fix: fixup to agree with original output

* feat: separate agenda cache, cache old meetings

* feat: agenda refresh tasks

* chore: undo accidental commit

* chore: remove debug parameter

* fix: convert session ID to int for comparison

* test: update/fix tests, rename new task

* refactor: rename task to have _task suffix

Also changes a log msg so it won't contain "None" awkwardly

* feat: no exceptions from agenda_data_refresh_task

* test: explanatory comment

* ci: agenda cache for k8s / testcrawl
---
 ietf/doc/models.py          |  16 ++-
 ietf/meeting/tasks.py       |  92 +++++++++++---
 ietf/meeting/tests_tasks.py |  64 ++++++++--
 ietf/meeting/views.py       | 247 +++++++++++++++++++++++++++++++-----
 ietf/settings.py            |  37 +++++-
 ietf/settings_testcrawl.py  |   3 +
 k8s/settings_local.py       |  10 ++
 7 files changed, 396 insertions(+), 73 deletions(-)

diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index cce9203d09..463aa6fd97 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -239,14 +239,14 @@ def revisions_by_newrevisionevent(self):
         return revisions
 
     def get_href(self, meeting=None):
-        return self._get_ref(meeting=meeting,meeting_doc_refs=settings.MEETING_DOC_HREFS)
+        return self._get_ref(meeting=meeting, versioned=True)
 
 
     def get_versionless_href(self, meeting=None):
-        return self._get_ref(meeting=meeting,meeting_doc_refs=settings.MEETING_DOC_GREFS)
+        return self._get_ref(meeting=meeting, versioned=False)
 
 
-    def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS):
+    def _get_ref(self, meeting=None, versioned=True):
         """
         Returns an url to the document text.  This differs from .get_absolute_url(),
         which returns an url to the datatracker page for the document.   
@@ -255,12 +255,16 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS):
         # the earlier resolution order, but there's at the moment one single
         # instance which matches this (with correct results), so we won't
         # break things all over the place.
-        if not hasattr(self, '_cached_href'):
+        cache_attr = "_cached_href" if versioned else "_cached_versionless_href"
+        if not hasattr(self, cache_attr):
             validator = URLValidator()
             if self.external_url and self.external_url.split(':')[0] in validator.schemes:
                 validator(self.external_url)
                 return self.external_url
 
+            meeting_doc_refs = (
+                settings.MEETING_DOC_HREFS if versioned else settings.MEETING_DOC_GREFS
+            )
             if self.type_id in settings.DOC_HREFS and self.type_id in meeting_doc_refs:
                 if self.meeting_related():
                     self.is_meeting_related = True
@@ -312,8 +316,8 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS):
 
             if href.startswith('/'):
                 href = settings.IDTRACKER_BASE_URL + href
-            self._cached_href = href
-        return self._cached_href
+            setattr(self, cache_attr, href)
+        return getattr(self, cache_attr)
 
     def set_state(self, state):
         """Switch state type implicit in state to state. This just
diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py
index c361325f9a..a73763560b 100644
--- a/ietf/meeting/tasks.py
+++ b/ietf/meeting/tasks.py
@@ -1,11 +1,14 @@
-# Copyright The IETF Trust 2024-2025, All Rights Reserved
+# Copyright The IETF Trust 2024-2026, All Rights Reserved
 #
 # Celery task definitions
 #
 import datetime
 
-from celery import shared_task
-# from django.db.models import QuerySet
+from itertools import batched
+
+from celery import shared_task, chain
+from django.db.models import IntegerField
+from django.db.models.functions import Cast
 from django.utils import timezone
 
 from ietf.utils import log
@@ -19,9 +22,56 @@
 from .utils import fetch_attendance_from_meetings
 
 
+@shared_task
+def agenda_data_refresh_task(num=None):
+    """Refresh agenda data for one plenary meeting
+
+    If `num` is `None`, refreshes data for the current meeting.
+    """
+    log.log(
+        f"Refreshing agenda data for {f"IETF-{num}" if num else "current IETF meeting"}"
+    )
+    try:
+        generate_agenda_data(num, force_refresh=True)
+    except Exception as err:
+        # Log and swallow exceptions so failure on one meeting won't break a chain of
+        # tasks. This is used by agenda_data_refresh_all_task().
+        log.log(f"ERROR: Refreshing agenda data failed for num={num}: {err}")
+
+
 @shared_task
 def agenda_data_refresh():
-    generate_agenda_data(force_refresh=True)
+    """Deprecated. Use agenda_data_refresh_task() instead.
+    
+    TODO remove this after switching the periodic task to the new name
+    """
+    log.log("Deprecated agenda_data_refresh task called!")
+    agenda_data_refresh_task()
+
+
+@shared_task
+def agenda_data_refresh_all_task(*, batch_size=10):
+    """Refresh agenda data for all plenary meetings
+
+    Executes as a chain of tasks, each computing up to `batch_size` meetings
+    in a single task.
+    """
+    meeting_numbers = sorted(
+        Meeting.objects.annotate(
+            number_as_int=Cast("number", output_field=IntegerField())
+        )
+        .filter(type_id="ietf", number_as_int__gt=64)
+        .values_list("number_as_int", flat=True)
+    )
+    # Batch using chained maps rather than celery.chunk so we only use one worker
+    # at a time.
+    batched_task_chain = chain(
+        *(
+            agenda_data_refresh_task.map(nums)
+            for nums in batched(meeting_numbers, batch_size)
+        )
+    )
+    batched_task_chain.delay()
 
 
 @shared_task
@@ -55,7 +105,9 @@ def proceedings_content_refresh_task(*, all=False):
 @shared_task
 def fetch_meeting_attendance_task():
     # fetch most recent two meetings
-    meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by("-date")[:2]
+    meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by(
+        "-date"
+    )[:2]
     try:
         stats = fetch_attendance_from_meetings(meetings)
     except RuntimeError as err:
@@ -64,8 +116,11 @@ def fetch_meeting_attendance_task():
         for meeting, meeting_stats in zip(meetings, stats):
             log.log(
                 "Fetched data for meeting {:>3}: {:4d} created, {:4d} updated, {:4d} deleted, {:4d} processed".format(
-                    meeting.number, meeting_stats['created'], meeting_stats['updated'], meeting_stats['deleted'],
-                    meeting_stats['processed']
+                    meeting.number,
+                    meeting_stats["created"],
+                    meeting_stats["updated"],
+                    meeting_stats["deleted"],
+                    meeting_stats["processed"],
                 )
             )
 
@@ -73,7 +128,7 @@ def fetch_meeting_attendance_task():
 def _select_meetings(
     meetings: list[str] | None = None,
     meetings_since: str | None = None,
-    meetings_until: str | None = None
+    meetings_until: str | None = None,
 ):  # nyah
     """Select meetings by number or date range"""
     # IETF-1 = 1986-01-16
@@ -130,15 +185,15 @@ def _select_meetings(
 @shared_task
 def resolve_meeting_materials_task(
     *,  # only allow kw arguments
-    meetings: list[str] | None=None,
-    meetings_since: str | None=None,
-    meetings_until: str | None=None
+    meetings: list[str] | None = None,
+    meetings_since: str | None = None,
+    meetings_until: str | None = None,
 ):
     """Run materials resolver on meetings
-    
+
     Can request a set of meetings by number by passing a list in the meetings arg, or
     by range by passing an iso-format timestamps in meetings_since / meetings_until.
-    To select all meetings, set meetings_since="zero" and omit other parameters. 
+    To select all meetings, set meetings_since="zero" and omit other parameters.
     """
     meetings_qs = _select_meetings(meetings, meetings_since, meetings_until)
     for meeting in meetings_qs.order_by("date"):
@@ -155,7 +210,9 @@ def resolve_meeting_materials_task(
                 f"meeting {meeting.number}: {err}"
             )
         else:
-            log.log(f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds.")
+            log.log(
+                f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds."
+            )
 
 
 @shared_task
@@ -163,13 +220,13 @@ def store_meeting_materials_as_blobs_task(
     *,  # only allow kw arguments
     meetings: list[str] | None = None,
     meetings_since: str | None = None,
-    meetings_until: str | None = None
+    meetings_until: str | None = None,
 ):
     """Push meeting materials into the blob store
 
     Can request a set of meetings by number by passing a list in the meetings arg, or
     by range by passing an iso-format timestamps in meetings_since / meetings_until.
-    To select all meetings, set meetings_since="zero" and omit other parameters. 
+    To select all meetings, set meetings_since="zero" and omit other parameters.
     """
     meetings_qs = _select_meetings(meetings, meetings_since, meetings_until)
     for meeting in meetings_qs.order_by("date"):
@@ -187,4 +244,5 @@ def store_meeting_materials_as_blobs_task(
             )
         else:
             log.log(
-                f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds.")
+                f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds."
+            )
diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py
index a5da00ecbf..2c5120a39d 100644
--- a/ietf/meeting/tests_tasks.py
+++ b/ietf/meeting/tests_tasks.py
@@ -5,23 +5,63 @@
 from ietf.utils.test_utils import TestCase
 from ietf.utils.timezone import date_today
 from .factories import MeetingFactory
-from .tasks import proceedings_content_refresh_task, agenda_data_refresh
+from .tasks import (
+    proceedings_content_refresh_task,
+    agenda_data_refresh_task,
+    agenda_data_refresh_all_task,
+)
 from .tasks import fetch_meeting_attendance_task
 
 
 class TaskTests(TestCase):
     @patch("ietf.meeting.tasks.generate_agenda_data")
-    def test_agenda_data_refresh(self, mock_generate):
-        agenda_data_refresh()
+    def test_agenda_data_refresh_task(self, mock_generate):
+        agenda_data_refresh_task()
         self.assertTrue(mock_generate.called)
-        self.assertEqual(mock_generate.call_args, call(force_refresh=True))
+        self.assertEqual(mock_generate.call_args, call(None, force_refresh=True))
+        
+        mock_generate.reset_mock()
+        mock_generate.side_effect = RuntimeError
+        try:
+            agenda_data_refresh_task()
+        except Exception as err:
+            self.fail(
+                f"agenda_data_refresh_task should not raise exceptions (got {repr(err)})"
+            )
+
+    @patch("ietf.meeting.tasks.agenda_data_refresh_task")
+    @patch("ietf.meeting.tasks.chain")
+    def test_agenda_data_refresh_all_task(self, mock_chain, mock_agenda_data_refresh):
+        # Patch the agenda_data_refresh_task task with a mock whose `.map` attribute
+        # converts its argument, which is expected to be an iterator,  to a list
+        # and returns it. We'll use this to check that the expected task chain
+        # was set up, but we don't actually run any celery tasks.
+        mock_agenda_data_refresh.map.side_effect = lambda x: list(x)
+
+        meetings = MeetingFactory.create_batch(5, type_id="ietf")
+        numbers = sorted(int(m.number) for m in meetings)
+        agenda_data_refresh_all_task(batch_size=2)
+        self.assertTrue(mock_chain.called)
+        # The lists in the call() below are the output of the lambda we patched in
+        # via mock_agenda_data_refresh.map.side_effect above. I.e., this tests that
+        # map() was called with the correct batched data.
+        self.assertEqual(
+            mock_chain.call_args,
+            call(
+                [numbers[0], numbers[1]],
+                [numbers[2], numbers[3]],
+                [numbers[4]],
+            ),
+        )
+        self.assertEqual(mock_agenda_data_refresh.call_count, 0)
+        self.assertEqual(mock_agenda_data_refresh.map.call_count, 3)
 
     @patch("ietf.meeting.tasks.generate_proceedings_content")
     def test_proceedings_content_refresh_task(self, mock_generate):
         # Generate a couple of meetings
         meeting120 = MeetingFactory(type_id="ietf", number="120")  # 24 * 5
         meeting127 = MeetingFactory(type_id="ietf", number="127")  # 24 * 5 + 7
-        
+
         # Times to be returned
         now_utc = datetime.datetime.now(tz=datetime.UTC)
         hour_00_utc = now_utc.replace(hour=0)
@@ -34,19 +74,19 @@ def test_proceedings_content_refresh_task(self, mock_generate):
         self.assertEqual(mock_generate.call_count, 1)
         self.assertEqual(mock_generate.call_args, call(meeting120, force_refresh=True))
         mock_generate.reset_mock()
-    
+
         # hour 01 - should call no meetings
         with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc):
             proceedings_content_refresh_task()
         self.assertEqual(mock_generate.call_count, 0)
-    
+
         # hour 07 - should call meeting with number % 24 == 0
         with patch("ietf.meeting.tasks.timezone.now", return_value=hour_07_utc):
             proceedings_content_refresh_task()
         self.assertEqual(mock_generate.call_count, 1)
         self.assertEqual(mock_generate.call_args, call(meeting127, force_refresh=True))
         mock_generate.reset_mock()
-        
+
         # With all=True, all should be called regardless of time. Reuse hour_01_utc which called none before
         with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc):
             proceedings_content_refresh_task(all=True)
@@ -61,10 +101,10 @@ def test_fetch_meeting_attendance_task(self, mock_fetch_attendance):
             MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=3)),
         ]
         data = {
-            'created': 1,
-            'updated': 2,
-            'deleted': 0,
-            'processed': 3,
+            "created": 1,
+            "updated": 2,
+            "deleted": 0,
+            "processed": 3,
         }
 
         mock_fetch_attendance.return_value = [data, data]
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 903e3c7e79..8dccda9c87 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -40,7 +40,7 @@
 from django.core.exceptions import ValidationError
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.core.validators import URLValidator
-from django.urls import reverse,reverse_lazy
+from django.urls import reverse, reverse_lazy, NoReverseMatch
 from django.db.models import F, Max, Q
 from django.forms.models import modelform_factory, inlineformset_factory
 from django.template import TemplateDoesNotExist
@@ -1859,18 +1859,22 @@ def generate_agenda_data(num=None, force_refresh=False):
     :num: meeting number
     :force_refresh: True to force a refresh of the cache
     """
-    cache = caches["default"]
-    cache_timeout = 6 * 60
-
     meeting = get_ietf_meeting(num)
     if meeting is None:
         raise Http404("No such full IETF meeting")
     elif int(meeting.number) <= 64:
-        return Http404("Pre-IETF 64 meetings are not available through this API")
-    else:
-        pass
+        raise Http404("Pre-IETF 64 meetings are not available through this API")
+    is_current_meeting = meeting.number == get_current_ietf_meeting_num()
+
+    cache = caches["agenda"]
+    cache_timeout = (
+        settings.AGENDA_CACHE_TIMEOUT_CURRENT_MEETING
+        if is_current_meeting
+        else settings.AGENDA_CACHE_TIMEOUT_DEFAULT
+    )
+    cache_format = "1"  # bump this on backward-incompatible data format changes
 
-    cache_key = f"generate_agenda_data_{meeting.number}"
+    cache_key = f"generate_agenda_data:{meeting.number}:v{cache_format}"
     if not force_refresh:
         cached_value = cache.get(cache_key)
         if cached_value is not None:
@@ -1890,8 +1894,6 @@ def generate_agenda_data(num=None, force_refresh=False):
 
     filter_organizer = AgendaFilterOrganizer(assignments=filtered_assignments)
 
-    is_current_meeting = (num is None) or (num == get_current_ietf_meeting_num())
-
     # Get Floor Plans
     floors = FloorPlan.objects.filter(meeting=meeting).order_by('order')
     
@@ -1966,21 +1968,32 @@ def api_get_session_materials(request, session_id=None):
     )
 
 
-def agenda_extract_schedule (item):
+def agenda_extract_schedule(item):
+    if item.session.current_status == "resched":
+        resched_to = item.session.tombstone_for.official_timeslotassignment()
+    else:
+        resched_to = None
     return {
         "id": item.id,
+        "slug": item.slug(),
         "sessionId": item.session.id,
-        "room": item.room_name if item.timeslot.show_location else None,
+        "room": (item.timeslot.get_location() or None) if item.timeslot else None,
         "location": {
             "short": item.timeslot.location.floorplan.short,
             "name": item.timeslot.location.floorplan.name,
         } if (item.timeslot.show_location and item.timeslot.location and item.timeslot.location.floorplan) else {},
         "acronym": item.acronym,
-        "duration": item.timeslot.duration.seconds,
+        "duration": item.timeslot.duration.total_seconds(),
         "name": item.session.name,
+        "slotId": item.timeslot.id,
         "slotName": item.timeslot.name,
+        "slotModified": item.timeslot.modified.isoformat(),
         "startDateTime": item.timeslot.time.isoformat(),
         "status": item.session.current_status,
+        "rescheduledTo": {
+            "startDateTime": resched_to.timeslot.time.isoformat(),
+            "duration": resched_to.timeslot.duration.total_seconds(),
+        } if resched_to is not None else {},
         "type": item.session.type.slug,
         "purpose": item.session.purpose.slug,
         "isBoF": item.session.group_at_the_time().state_id == "bof",
@@ -1998,7 +2011,7 @@ def agenda_extract_schedule (item):
             "showAgenda": True if (item.session.agenda() is not None or item.session.remote_instructions) else False
         },
         "agenda": {
-            "url": item.session.agenda().get_href()
+            "url": item.session.agenda().get_versionless_href()
         } if item.session.agenda() is not None else {
             "url": None
         },
@@ -2290,10 +2303,131 @@ def ical_session_status(assignment):
     else:
         return "CONFIRMED"
 
+
+def render_icalendar_precomp(agenda_data):
+    ical_content = generate_agenda_ical_precomp(agenda_data)
+    return HttpResponse(ical_content, content_type="text/calendar")
+
+
 def render_icalendar(schedule, assignments):
     ical_content = generate_agenda_ical(schedule, assignments)
     return HttpResponse(ical_content, content_type="text/calendar")
 
+
+def generate_agenda_ical_precomp(agenda_data):
+    """Generate iCalendar from precomputed data using the icalendar library"""
+
+    cal = Calendar()
+    cal.add("prodid", "-//IETF//datatracker.ietf.org ical agenda//EN")
+    cal.add("version", "2.0")
+    cal.add("method", "PUBLISH")
+    
+    meeting_data = agenda_data["meeting"]
+    for item in agenda_data["schedule"]:
+        event = Event()
+        
+        uid = f"ietf-{meeting_data["number"]}-{item["slotId"]}-{item["acronym"]}"
+        event.add("uid", uid)
+
+        # add custom field with meeting's local TZ
+        event.add("x-meeting-tz", meeting_data["timezone"])
+        
+        if item["name"]:
+            summary = item["name"]
+        else:
+            summary = f"{item["groupAcronym"]} - {item["groupName"]}"
+
+        if item["note"]:
+            summary += f" ({item["note"]})"
+
+        event.add("summary", summary)
+
+        if item["room"]:
+            event.add("location", item["room"])  # room name
+
+        if item["status"] == "canceled":
+            status = "CANCELLED"
+        elif item["status"] == "resched":
+            resched_to = item["rescheduledTo"]
+            if resched_to is None:
+                status = "RESCHEDULED"
+            else:
+                resched_start = datetime.datetime.fromisoformat(
+                    resched_to["startDateTime"]
+                )
+                dur = datetime.timedelta(seconds=resched_to["duration"])
+                resched_end = resched_start + dur
+                formatted_start = resched_start.strftime("%A %H:%M").upper()
+                formatted_end = resched_end.strftime("%H:%M")
+                status = f"RESCHEDULED TO {formatted_start}-{formatted_end}"
+        else:
+            status = "CONFIRMED"
+        event.add("status", status)
+
+        event.add("class", "PUBLIC")
+
+        start_time = datetime.datetime.fromisoformat(item["startDateTime"])
+        duration = datetime.timedelta(seconds=item["duration"])
+        event.add("dtstart", start_time)
+        event.add("dtend", start_time + duration)
+
+        # DTSTAMP: when the event was created or last modified (in UTC)
+        # n.b. timeslot.modified may not be an accurate measure of this
+        event.add("dtstamp", datetime.datetime.fromisoformat(item["slotModified"]))
+
+        description_parts = [item["slotName"]]
+
+        if item["note"]:
+            description_parts.append(f"Note: {item["note"]}")
+
+        links = item["links"]
+        if links["onsiteTool"]:
+            description_parts.append(f"Onsite tool: {links["onsiteTool"]}")
+
+        if links["videoStream"]:
+            description_parts.append(f"Meetecho: {links["videoStream"]}")
+
+        if links["webex"]:
+            description_parts.append(f"Webex: {links["webex"]}")
+
+        if item["remoteInstructions"]:
+            description_parts.append(
+                f"Remote instructions: {item["remoteInstructions"]}"
+            )
+
+        try:
+            materials_url = absurl(
+                "ietf.meeting.views.session_details",
+                num=meeting_data["number"],
+                acronym=item["acronym"],
+            )
+        except NoReverseMatch:
+            pass
+        else:
+            description_parts.append(f"Session materials: {materials_url}")
+            event.add("url", materials_url)
+
+        if meeting_data["number"].isdigit():
+            try:
+                agenda_url = absurl("agenda", num=meeting_data["number"])
+            except NoReverseMatch:
+                pass
+            else:
+                description_parts.append(f"See in schedule: {agenda_url}#row-{item["slug"]}")
+
+        if item["agenda"] and item["agenda"]["url"]:
+            description_parts.append(f"Agenda {item["agenda"]["url"]}")
+
+        # Join all description parts with 2 newlines
+        description = "\n\n".join(description_parts)
+        event.add("description", description)
+
+        # Add event to calendar
+        cal.add_component(event)
+
+    return cal.to_ical().decode("utf-8")
+
+
 def generate_agenda_ical(schedule, assignments):
     """Generate iCalendar using the icalendar library"""
 
@@ -2428,10 +2562,66 @@ def parse_agenda_filter_params(querydict):
 
 def should_include_assignment(filter_params, assignment):
     """Decide whether to include an assignment"""
-    shown = len(set(filter_params['show']).intersection(assignment.filter_keywords)) > 0
-    hidden = len(set(filter_params['hide']).intersection(assignment.filter_keywords)) > 0
+    if hasattr(assignment, "filter_keywords"):
+        kw = assignment.filter_keywords
+    elif isinstance(assignment, dict):
+        kw = assignment.get("filterKeywords", [])
+    else:
+        raise ValueError("Unsupported assignment instance")
+    shown = len(set(filter_params['show']).intersection(kw)) > 0
+    hidden = len(set(filter_params['hide']).intersection(kw)) > 0
     return shown and not hidden
 
+
+def agenda_ical_ietf(meeting, filt_params, acronym=None, session_id=None):
+    agenda_data = generate_agenda_data(meeting.number, force_refresh=False)
+    if acronym:
+        agenda_data["schedule"] = [
+            item
+            for item in agenda_data["schedule"]
+            if item["groupAcronym"] == acronym
+        ]
+    elif session_id:
+        agenda_data["schedule"] = [
+            item
+            for item in agenda_data["schedule"]
+            if item["sessionId"] == session_id
+        ]
+    if filt_params is not None:
+        # Apply the filter
+        agenda_data["schedule"] = [
+            item
+            for item in agenda_data["schedule"]
+            if should_include_assignment(filt_params, item)
+        ]
+    return render_icalendar_precomp(agenda_data)
+
+
+def agenda_ical_interim(meeting, filt_params, acronym=None, session_id=None):
+    schedule = get_schedule(meeting)
+
+    if schedule is None and acronym is None and session_id is None:
+        raise Http404
+
+    assignments = SchedTimeSessAssignment.objects.filter(
+        schedule__in=[schedule, schedule.base],
+        session__on_agenda=True,
+    )
+    assignments = preprocess_assignments_for_agenda(assignments, meeting)
+    AgendaKeywordTagger(assignments=assignments).apply()
+
+    if filt_params is not None:
+        # Apply the filter
+        assignments = [a for a in assignments if should_include_assignment(filt_params, a)]
+
+    if acronym:
+        assignments = [ a for a in assignments if a.session.group_at_the_time().acronym == acronym ]
+    elif session_id:
+        assignments = [ a for a in assignments if a.session_id == int(session_id) ]
+
+    return render_icalendar(schedule, assignments)
+
+
 def agenda_ical(request, num=None, acronym=None, session_id=None):
     """Agenda ical view
 
@@ -2459,33 +2649,20 @@ def agenda_ical(request, num=None, acronym=None, session_id=None):
             raise Http404
     else:
         meeting = get_meeting(num, type_in=None)  # get requested meeting, whatever its type
-    schedule = get_schedule(meeting)
 
-    if schedule is None and acronym is None and session_id is None:
-        raise Http404
-
-    assignments = SchedTimeSessAssignment.objects.filter(
-        schedule__in=[schedule, schedule.base],
-        session__on_agenda=True,
-    )
-    assignments = preprocess_assignments_for_agenda(assignments, meeting)
-    AgendaKeywordTagger(assignments=assignments).apply()
+    if isinstance(session_id, str) and session_id.isdigit():
+        session_id = int(session_id)
 
     try:
         filt_params = parse_agenda_filter_params(request.GET)
     except ValueError as e:
         return HttpResponseBadRequest(str(e))
 
-    if filt_params is not None:
-        # Apply the filter
-        assignments = [a for a in assignments if should_include_assignment(filt_params, a)]
-
-    if acronym:
-        assignments = [ a for a in assignments if a.session.group_at_the_time().acronym == acronym ]
-    elif session_id:
-        assignments = [ a for a in assignments if a.session_id == int(session_id) ]
+    if meeting.type_id == "ietf":
+        return agenda_ical_ietf(meeting, filt_params, acronym, session_id)
+    else:
+        return agenda_ical_interim(meeting, filt_params, acronym, session_id)
 
-    return render_icalendar(schedule, assignments)
 
 @cache_page(15 * 60)
 def agenda_json(request, num=None):
diff --git a/ietf/settings.py b/ietf/settings.py
index 1cda79e21b..899a377ad7 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -228,6 +228,10 @@
 BLOBSTORAGE_CONNECT_TIMEOUT = 10  # seconds; boto3 default is 60
 BLOBSTORAGE_READ_TIMEOUT = 10  # seconds; boto3 default is 60
 
+# Caching for agenda data in seconds
+AGENDA_CACHE_TIMEOUT_DEFAULT = 8 * 24 * 60 * 60  # 8 days
+AGENDA_CACHE_TIMEOUT_CURRENT_MEETING = 6 * 60  # 6 minutes
+
 WSGI_APPLICATION = "ietf.wsgi.application"
 
 AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', )
@@ -1400,6 +1404,16 @@ def skip_unreadable_post(record):
                     f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
                 ),
             },
+            "agenda": {
+                "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt:agenda",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
+            },
             "proceedings": {
                 "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
                 "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
@@ -1453,6 +1467,17 @@ def skip_unreadable_post(record):
                 "VERSION": __version__,
                 "KEY_PREFIX": "ietf:dt",
             },
+            "agenda": {
+                "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+                # "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+                # "LOCATION": "127.0.0.1:11211",
+                # No release-specific VERSION setting.
+                "KEY_PREFIX": "ietf:dt:agenda",
+                # Key function is default except with sha384-encoded key
+                "KEY_FUNCTION": lambda key, key_prefix, version: (
+                    f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+                ),
+            },
             "proceedings": {
                 "BACKEND": "django.core.cache.backends.dummy.DummyCache",
                 # "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
@@ -1519,11 +1544,17 @@ def skip_unreadable_post(record):
         NOMCOM_APP_SECRET = b'\x9b\xdas1\xec\xd5\xa0SI~\xcb\xd4\xf5t\x99\xc4i\xd7\x9f\x0b\xa9\xe8\xfeY\x80$\x1e\x12tN:\x84'
 
     ALLOWED_HOSTS = ['*',]
-    
+
     try:
         # see https://github.com/omarish/django-cprofile-middleware
-        import django_cprofile_middleware # pyflakes:ignore
-        MIDDLEWARE = MIDDLEWARE + ['django_cprofile_middleware.middleware.ProfilerMiddleware', ]
+        import django_cprofile_middleware  # pyflakes:ignore
+
+        MIDDLEWARE = MIDDLEWARE + [
+            "django_cprofile_middleware.middleware.ProfilerMiddleware",
+        ]
+        DJANGO_CPROFILE_MIDDLEWARE_REQUIRE_STAFF = (
+            False  # Do not use this setting for a public site!
+        )
     except ImportError:
         pass
 
diff --git a/ietf/settings_testcrawl.py b/ietf/settings_testcrawl.py
index 40744a228d..edb978757a 100644
--- a/ietf/settings_testcrawl.py
+++ b/ietf/settings_testcrawl.py
@@ -27,6 +27,9 @@
             'MAX_ENTRIES': 10000,
         },
     },
+    'agenda': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+    },
     'proceedings': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
     },
diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index f8ffacc83f..0386dbbdf9 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -306,6 +306,16 @@ def _multiline_to_list(s):
             f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
         ),
     },
+    "agenda": {
+        "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
+        "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",
+        # No release-specific VERSION setting.
+        "KEY_PREFIX": "ietf:dt:agenda",
+        # Key function is default except with sha384-encoded key
+        "KEY_FUNCTION": lambda key, key_prefix, version: (
+            f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}"
+        ),
+    },
     "proceedings": {
         "BACKEND": "ietf.utils.cache.LenientMemcacheCache",
         "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}",

From f8be1436fb9570eb623c6408cbb755e39bb18b22 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 10 Feb 2026 13:33:39 -0400
Subject: [PATCH 559/656] fix: add id attr to liaisons ButtonWidget (#10389)

---
 ietf/liaisons/widgets.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ietf/liaisons/widgets.py b/ietf/liaisons/widgets.py
index 74368e83f2..48db8af0a3 100644
--- a/ietf/liaisons/widgets.py
+++ b/ietf/liaisons/widgets.py
@@ -26,7 +26,9 @@ def render(self, name, value, **kwargs):
                 html += '<span class="d-none attachRequiredField">%s</span>' % conditional_escape(i)
             required_str = 'Please fill in %s to attach a new file' % conditional_escape(self.required_label)
             html += '<span class="d-none attachDisabledLabel">%s</span>' % conditional_escape(required_str)
-        html += '<button type="button" class="addAttachmentWidget btn btn-primary btn-sm">%s</button>' % conditional_escape(self.label)
+        html += '<button type="button" id="{}" class="addAttachmentWidget btn btn-primary btn-sm">{}</button>'.format(
+            f"id_{name}", conditional_escape(self.label)
+        )
         return mark_safe(html)
 
 

From 4a024d9d64e36714523b2f9e04bdbc3005aefc03 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 10 Feb 2026 12:25:43 -0600
Subject: [PATCH 560/656] fix: identify editorial drafts that should not expire
 (#10388)

---
 ietf/doc/expire.py | 50 ++++++++++++++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 13 deletions(-)

diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py
index bf8523aa98..d42af628f8 100644
--- a/ietf/doc/expire.py
+++ b/ietf/doc/expire.py
@@ -38,22 +38,46 @@ def expirable_drafts(queryset=None):
     # Populate this first time through (but after django has been set up)
     if nonexpirable_states is None:
         # all IESG states except I-D Exists and Dead block expiry
-        nonexpirable_states = list(State.objects.filter(used=True, type="draft-iesg").exclude(slug__in=("idexists", "dead")))
+        nonexpirable_states = list(
+            State.objects.filter(used=True, type="draft-iesg").exclude(
+                slug__in=("idexists", "dead")
+            )
+        )
         # sent to RFC Editor and RFC Published block expiry (the latter
         # shouldn't be possible for an active draft, though)
-        nonexpirable_states += list(State.objects.filter(used=True, type__in=("draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"), slug__in=("rfc-edit", "pub")))
+        nonexpirable_states += list(
+            State.objects.filter(
+                used=True,
+                type__in=(
+                    "draft-stream-iab",
+                    "draft-stream-irtf",
+                    "draft-stream-ise",
+                    "draft-stream-editorial",
+                ),
+                slug__in=("rfc-edit", "pub"),
+            )
+        )
         # other IRTF states that block expiration
-        nonexpirable_states += list(State.objects.filter(used=True, type_id="draft-stream-irtf", slug__in=("irsgpoll", "iesg-rev",)))
-
-    return queryset.filter(
-        states__type="draft", states__slug="active"
-    ).exclude(
-        expires=None
-    ).exclude(
-        states__in=nonexpirable_states
-    ).exclude(
-        tags="rfc-rev"  # under review by the RFC Editor blocks expiry
-    ).distinct()
+        nonexpirable_states += list(
+            State.objects.filter(
+                used=True,
+                type_id="draft-stream-irtf",
+                slug__in=(
+                    "irsgpoll",
+                    "iesg-rev",
+                ),
+            )
+        )
+
+    return (
+        queryset.filter(states__type="draft", states__slug="active")
+        .exclude(expires=None)
+        .exclude(states__in=nonexpirable_states)
+        .exclude(
+            tags="rfc-rev"  # under review by the RFC Editor blocks expiry
+        )
+        .distinct()
+    )
 
 
 def get_soon_to_expire_drafts(days_of_warning):

From 832c62e5c1f0cba736842cbe241236bfc8cf386f Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 11 Feb 2026 11:41:58 -0400
Subject: [PATCH 561/656] feat: format+has_errata, drop see_also for red API
 (#10392)

* fix: has_errata field for red API

* chore: explanatory comment

* feat: accurate format list for red API

* refactor: specify blob names in API

* chore: remove see_also field

* fix: finish removing see_also field
---
 ietf/doc/api.py         | 27 ++++++++++++++++++++++-----
 ietf/doc/models.py      | 18 ++++++++++++++++++
 ietf/doc/serializers.py | 33 +++++++++++++++++++++++----------
 3 files changed, 63 insertions(+), 15 deletions(-)

diff --git a/ietf/doc/api.py b/ietf/doc/api.py
index 47e7e6fffd..6a4c0c9fd5 100644
--- a/ietf/doc/api.py
+++ b/ietf/doc/api.py
@@ -1,7 +1,17 @@
 # Copyright The IETF Trust 2024-2026, All Rights Reserved
 """Doc API implementations"""
 
-from django.db.models import OuterRef, Subquery, Prefetch, Value, JSONField, QuerySet
+from django.db.models import (
+    BooleanField,
+    Count,
+    JSONField,
+    OuterRef,
+    Prefetch,
+    Q,
+    QuerySet,
+    Subquery,
+    Value,
+)
 from django.db.models.functions import TruncDate
 from django_filters import rest_framework as filters
 from rest_framework import filters as drf_filters
@@ -133,11 +143,18 @@ def augment_rfc_queryset(queryset: QuerySet[Document]):
         )
         .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO))
         .annotate(
-            # TODO implement these fake fields for real
-            see_also=Value([], output_field=JSONField()),
-            formats=Value(["txt", "xml"], output_field=JSONField()),
+            # Count of "verified-errata" tags will be 1 or 0, convert to Boolean
+            has_errata=Count(
+                "tags",
+                filter=Q(
+                    tags__slug="verified-errata",
+                ),
+                output_field=BooleanField(),
+            )
+        )
+        .annotate(
+            # TODO implement this fake field for real
             keywords=Value(["keyword"], output_field=JSONField()),
-            errata=Value([], output_field=JSONField()),
         )
     )
 
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index 463aa6fd97..ec9a25add8 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -1284,6 +1284,24 @@ def action_holders_enabled(self):
         iesg_state = self.get_state('draft-iesg')
         return iesg_state and iesg_state.slug != 'idexists'
 
+    def formats(self):
+        """List of file formats available
+        
+        Only implemented for RFCs. Relies on StoredObject.
+        """
+        if self.type_id != "rfc":
+            raise RuntimeError("Only allowed for type=rfc")
+        return [
+            {
+                "fmt": Path(object_name).parts[0],
+                "name": object_name,
+            }
+            for object_name in StoredObject.objects.filter(
+                store="rfc", doc_name=self.name, doc_rev=self.rev
+            ).values_list("name", flat=True)
+        ]
+
+
 class DocumentURL(models.Model):
     doc  = ForeignKey(Document)
     tag  = ForeignKey(DocUrlTagName)
diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py
index 05647d9ce1..e8d373164b 100644
--- a/ietf/doc/serializers.py
+++ b/ietf/doc/serializers.py
@@ -16,6 +16,7 @@
 
 class RfcAuthorSerializer(serializers.ModelSerializer):
     """Serializer for an RfcAuthor / DocumentAuthor in a response"""
+
     datatracker_person_path = serializers.URLField(
         source="person.get_absolute_url",
         required=False,
@@ -36,7 +37,7 @@ class Meta:
 
     def to_representation(self, instance):
         """instance -> primitive data types
-        
+
         Translates a DocumentAuthor into an equivalent RfcAuthor we can use the same
         serializer for either type.
         """
@@ -87,7 +88,15 @@ class DocIdentifierSerializer(serializers.Serializer):
 
 
 type RfcStatusSlugT = Literal[
-    "std", "ps", "ds", "bcp", "inf", "exp", "hist", "unkn", "not-issued",
+    "std",
+    "ps",
+    "ds",
+    "bcp",
+    "inf",
+    "exp",
+    "hist",
+    "unkn",
+    "not-issued",
 ]
 
 
@@ -188,11 +197,16 @@ class ContainingSubseriesSerializer(serializers.Serializer):
     type = serializers.CharField(source="source.type_id")
 
 
+class RfcFormatSerializer(serializers.Serializer):
+    RFC_FORMATS = ("xml", "txt", "html", "pdf", "ps", "json", "notprepped")
+
+    fmt = serializers.ChoiceField(choices=RFC_FORMATS)
+    name = serializers.CharField(help_text="Name of blob in the blob store")
+
+
 class RfcMetadataSerializer(serializers.ModelSerializer):
     """Serialize metadata of an RFC"""
 
-    RFC_FORMATS = ("xml", "txt", "html", "htmlized", "pdf", "ps")
-
     number = serializers.IntegerField(source="rfc_number")
     published = serializers.DateField()
     status = RfcStatusSerializer(source="*")
@@ -207,10 +221,11 @@ class RfcMetadataSerializer(serializers.ModelSerializer):
     updates = RelatedRfcSerializer(many=True, read_only=True)
     updated_by = ReverseRelatedRfcSerializer(many=True, read_only=True)
     subseries = ContainingSubseriesSerializer(many=True, read_only=True)
-    see_also = serializers.ListField(child=serializers.CharField(), read_only=True)
-    formats = serializers.MultipleChoiceField(choices=RFC_FORMATS)
+    formats = RfcFormatSerializer(
+        many=True, read_only=True, help_text="Available formats"
+    )
     keywords = serializers.ListField(child=serializers.CharField(), read_only=True)
-    errata = serializers.ListField(child=serializers.CharField(), read_only=True)
+    has_errata = serializers.BooleanField(read_only=True)
 
     class Meta:
         model = Document
@@ -230,15 +245,13 @@ class Meta:
             "updates",
             "updated_by",
             "subseries",
-            "see_also",
             "draft",
             "abstract",
             "formats",
             "keywords",
-            "errata",
+            "has_errata",
         ]
 
-
     @extend_schema_field(RfcAuthorSerializer(many=True))
     def get_authors(self, doc: Document):
         # If doc has any RfcAuthors, use those, otherwise fall back to DocumentAuthors

From 66a1bf0a9ecae30036cf23fc07f6f26b27b1e94d Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 11 Feb 2026 15:00:57 -0400
Subject: [PATCH 562/656] chore: un-squelch bibtexparser DeprecationWarnings
 (#10395)

---
 ietf/settings.py | 2 --
 requirements.txt | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/ietf/settings.py b/ietf/settings.py
index 899a377ad7..565e8825a9 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -36,8 +36,6 @@
 warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="oic.utils.time_util")
 warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="pytz.tzinfo")
 warnings.filterwarnings("ignore", message="'instantiateVariableFont' is deprecated", module="weasyprint")
-warnings.filterwarnings("ignore", category=DeprecationWarning, module="bibtexparser")  # https://github.com/sciunto-org/python-bibtexparser/issues/502
-warnings.filterwarnings("ignore", category=DeprecationWarning, module="pyparsing")  # https://github.com/sciunto-org/python-bibtexparser/issues/502
 
 
 base_path = pathlib.Path(__file__).resolve().parent
diff --git a/requirements.txt b/requirements.txt
index 3f89f6f16c..cb583d5dc9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@ setuptools>=80.9.0  # Require this first, to prevent later errors
 aiosmtpd>=1.4.6
 argon2-cffi>=25.1.0  # For the Argon2 password hasher option
 beautifulsoup4>=4.13.4  # Only used in tests
-bibtexparser>=1.4.3  # Only used in tests
+bibtexparser>=1.4.4  # Only used in tests
 bleach>=6.2.0  # project is deprecated but supported
 types-bleach>=6.2.0
 boto3>=1.39.15

From 0b637ef4ace72f31a963b7603a8f69a253420810 Mon Sep 17 00:00:00 2001
From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com>
Date: Wed, 11 Feb 2026 19:13:44 +0000
Subject: [PATCH 563/656] ci: update base image target version to 20260211T1901

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 41ff295eec..71370fabee 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20260114T1756
+FROM ghcr.io/ietf-tools/datatracker-app-base:20260211T1901
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 3ad31c7e25..947f3790e4 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20260114T1756
+20260211T1901

From 492888b8a22113becc2cbe3900ba3294cfc6d7f6 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 12 Feb 2026 11:34:11 -0400
Subject: [PATCH 564/656] fix: handle doc_rev is None in Document.formats
 (#10401)

Likely a temporary fix, but safer for quick deployment while we
work on the bigger project.
---
 ietf/api/views_rpc.py |  2 +-
 ietf/doc/models.py    | 12 +++++++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index 6b1799f654..2bf16480f2 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -487,7 +487,7 @@ def post(self, request):
                         name=self._blob_destination(ftm),
                         file=f,
                         doc_name=rfc.name,
-                        doc_rev=rfc.rev,  # expect None, but match whatever it is
+                        doc_rev=rfc.rev,  # expect blank, but match whatever it is
                         mtime=mtime,
                     )
                 destination = self._fs_destination(ftm)
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index ec9a25add8..8f700bf496 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -12,6 +12,8 @@
 
 from io import BufferedReader
 from pathlib import Path
+
+from django.db.models import Q
 from lxml import etree
 from typing import Optional, Protocol, TYPE_CHECKING, Union
 from weasyprint import HTML as wpHTML
@@ -1291,13 +1293,21 @@ def formats(self):
         """
         if self.type_id != "rfc":
             raise RuntimeError("Only allowed for type=rfc")
+
+        # StoredObject.doc_rev can be null or "" to represent no rev. Match either
+        # of these when self.rev is "" (always expected to be the case for RFCs)
+        rev_q = Q(doc_rev=self.rev)
+        if self.rev == "":
+            rev_q |= Q(doc_rev__isnull=True)
         return [
             {
                 "fmt": Path(object_name).parts[0],
                 "name": object_name,
             }
             for object_name in StoredObject.objects.filter(
-                store="rfc", doc_name=self.name, doc_rev=self.rev
+                rev_q,
+                store="rfc",
+                doc_name=self.name,
             ).values_list("name", flat=True)
         ]
 

From 1b306eb7f3b9b45b3162101c7fc216ac7da3eab1 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 12 Feb 2026 13:49:22 -0400
Subject: [PATCH 565/656] feat: bofreq timestamp fixup task (#10402)

* feat: utility to fix up bofreq timestamps

* fix: don't fix -00 (+ logging)

* feat: task

* chore: disable test coverage for one-off task
---
 ietf/doc/tasks.py        |   8 ++-
 ietf/doc/utils_bofreq.py | 143 ++++++++++++++++++++++++++++++++++++++-
 2 files changed, 147 insertions(+), 4 deletions(-)

diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py
index 02b7c2a07d..b463b9cecf 100644
--- a/ietf/doc/tasks.py
+++ b/ietf/doc/tasks.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2024-2025, All Rights Reserved
+# Copyright The IETF Trust 2024-2026, All Rights Reserved
 #
 # Celery task definitions
 #
@@ -34,6 +34,7 @@
     ensure_draft_bibxml_path_exists,
     investigate_fragment,
 )
+from .utils_bofreq import fixup_bofreq_timestamps
 
 
 @shared_task
@@ -149,3 +150,8 @@ def rebuild_reference_relations_task(doc_names: list[str]):
             rebuild_reference_relations(doc, filenames)
         else:
             log.log(f"Found no content for {stem}")
+
+
+@shared_task
+def fixup_bofreq_timestamps_task():  # pragma: nocover
+    fixup_bofreq_timestamps()
diff --git a/ietf/doc/utils_bofreq.py b/ietf/doc/utils_bofreq.py
index aec8f60ad6..d01b039b8e 100644
--- a/ietf/doc/utils_bofreq.py
+++ b/ietf/doc/utils_bofreq.py
@@ -1,12 +1,149 @@
-# Copyright The IETF Trust 2021 All Rights Reserved
+# Copyright The IETF Trust 2021-2026 All Rights Reserved
+import datetime
+from pathlib import Path
 
-from ietf.doc.models import BofreqEditorDocEvent, BofreqResponsibleDocEvent
+from django.conf import settings
+
+from ietf.doc.models import (
+    BofreqEditorDocEvent,
+    BofreqResponsibleDocEvent,
+    DocEvent,
+    DocHistory,
+    Document,
+)
 from ietf.person.models import Person
+from ietf.utils import log
+
 
 def bofreq_editors(bofreq):
     e = bofreq.latest_event(BofreqEditorDocEvent)
     return e.editors.all() if e else Person.objects.none()
 
+
 def bofreq_responsible(bofreq):
     e = bofreq.latest_event(BofreqResponsibleDocEvent)
-    return e.responsible.all() if e else Person.objects.none()
\ No newline at end of file
+    return e.responsible.all() if e else Person.objects.none()
+
+
+def fixup_bofreq_timestamps():  # pragma: nocover
+    """Fixes bofreq event / document timestamps
+
+    Timestamp errors resulted from the bug fixed by
+    https://github.com/ietf-tools/datatracker/pull/10333
+    
+    Does not fix up -00 revs because the timestamps on these were not affected by
+    the bug. Replacing their timestamps creates a confusing event history because the 
+    filesystem timestamp is usually a fraction of a second later than other events
+    created upon the initial rev creation. This causes the "New revision available"
+    event to appear _after_ these events in the history. Better to leave them as is.  
+    """
+    FIX_DEPLOYMENT_TIME = "2026-02-03T01:16:00+00:00"  # 12.58.0 -> production
+
+    def _get_doc_time(doc_name: str, rev: str):
+        path = Path(settings.BOFREQ_PATH) / f"{doc_name}-{rev}.md"
+        return datetime.datetime.fromtimestamp(path.stat().st_mtime, datetime.UTC)
+
+    # Find affected DocEvents and DocHistories
+    new_bofreq_events = (
+        DocEvent.objects.filter(
+            doc__type="bofreq", type="new_revision", time__lt=FIX_DEPLOYMENT_TIME
+        )
+        .exclude(rev="00")  # bug did not affect rev 00 events
+        .order_by("doc__name", "rev")
+    )
+    log.log(
+        f"fixup_bofreq_timestamps: found {new_bofreq_events.count()} "
+        f"new_revision events before {FIX_DEPLOYMENT_TIME}"
+    )
+    document_fixups = {}
+    for e in new_bofreq_events:
+        name = e.doc.name
+        rev = e.rev
+        filesystem_time = _get_doc_time(name, rev)
+        assert e.time < filesystem_time, (
+            f"Rev {rev} event timestamp for {name} unexpectedly later than the "
+            "filesystem timestamp!"
+        )
+        try:
+            dochistory = DocHistory.objects.filter(
+                name=name, time__lt=filesystem_time
+            ).get(rev=rev)
+        except DocHistory.MultipleObjectsReturned as err:
+            raise RuntimeError(
+                f"Multiple DocHistories for {name} rev {rev} exist earlier than the "
+                "filesystem timestamp!"
+            ) from err
+        except DocHistory.DoesNotExist as err:
+            if rev == "00":
+                # Unreachable because we don't adjust -00 revs, but could be needed
+                # if we did, in theory. In practice it's still not reached, but
+                # keeping the case for completeness.
+                dochistory = None
+            else:
+                raise RuntimeError(
+                    f"No DocHistory for {name} rev {rev} exists earlier than the "
+                    f"filesystem timestamp!"
+                ) from err
+
+        if name not in document_fixups:
+            document_fixups[name] = []
+        document_fixups[name].append(
+            {
+                "event": e,
+                "dochistory": dochistory,
+                "filesystem_time": filesystem_time,
+            }
+        )
+
+    # Now do the actual fixup
+    system_person = Person.objects.get(name="(System)")
+    for doc_name, fixups in document_fixups.items():
+        bofreq = Document.objects.get(type="bofreq", name=doc_name)
+        log_msg_parts = []
+        adjusted_revs = []
+        for fixup in fixups:
+            event_to_fix = fixup["event"]
+            dh_to_fix = fixup["dochistory"]
+            new_time = fixup["filesystem_time"]
+            adjusted_revs.append(event_to_fix.rev)
+
+            # Fix up the event
+            event_to_fix.time = new_time
+            event_to_fix.save()
+            log_msg_parts.append(f"rev {event_to_fix.rev} DocEvent")
+
+            # Fix up the DocHistory
+            if dh_to_fix is not None:
+                dh_to_fix.time = new_time
+                dh_to_fix.save()
+                log_msg_parts.append(f"rev {dh_to_fix.rev} DocHistory")
+
+            if event_to_fix.rev == bofreq.rev and bofreq.time < new_time:
+                # Update the Document without calling save(). Only update if
+                # the time has not changed so we don't inadvertently overwrite
+                # a concurrent update.
+                Document.objects.filter(pk=bofreq.pk, time=bofreq.time).update(
+                    time=new_time
+                )
+                bofreq.refresh_from_db()
+                if bofreq.rev == event_to_fix.rev:
+                    log_msg_parts.append(f"rev {bofreq.rev} Document")
+                else:
+                    log.log(
+                        "fixup_bofreq_timestamps: WARNING: bofreq Document rev "
+                        f"changed for {bofreq.name}"
+                    )
+        log.log(f"fixup_bofreq_timestamps: {bofreq.name}: " + ", ".join(log_msg_parts))
+
+        # Fix up the Document, if necessary, and add a record of the adjustment
+        DocEvent.objects.create(
+            type="added_comment",
+            by=system_person,
+            doc=bofreq,
+            rev=bofreq.rev,
+            desc=(
+                "Corrected inaccurate document and new revision event timestamps for "
+                + ("version " if len(adjusted_revs) == 1 else "versions ")
+                + ", ".join(adjusted_revs)
+            ),
+        )

From 4945809b7804c1f15de1b0340be269dd7e200f95 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 12 Feb 2026 13:56:46 -0400
Subject: [PATCH 566/656] chore(dev): update beat in docker-compose.yml
 (#10330)

Fixes commented-out config
---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 2440faf121..ebe53cf95a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -129,7 +129,7 @@ services:
 #  but can be enabled by uncommenting the following.
 #
 #    beat:
-#        image: ghcr.io/ietf-tools/datatracker-celery:latest
+#        image: "${COMPOSE_PROJECT_NAME}-celery"
 #        init: true
 #        environment:
 #            CELERY_APP: ietf

From 8005a8baa6ffb72c47d6e35f44c0e5d78b456a2d Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 13 Feb 2026 11:36:23 -0400
Subject: [PATCH 567/656] chore(dev): update docker-compose depends_on (#10410)

* chore(dev): update docker-compose depends_on

* chore(dev): another depends_on tweak

app/celery don't actually use the blobstore container,
but the Django config refers to it so we should
probably depend on it anyway
---
 docker-compose.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index ebe53cf95a..4c3f2f6b8e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -13,9 +13,10 @@ services:
         # network_mode: service:db
 
         depends_on:
+            - blobdb
+            - blobstore
             - db
             - mq
-            - blobstore
 
         ipc: host
 
@@ -79,7 +80,10 @@ services:
         command:
             - '--loglevel=INFO'
         depends_on:
+            - blobdb
+            - blobstore
             - db
+            - mq
         restart: unless-stopped
         stop_grace_period: 1m
         volumes:
@@ -102,7 +106,10 @@ services:
             - '--concurrency=1'
                  
         depends_on:
+            - blobdb
+            - blobstore
             - db
+            - mq
         restart: unless-stopped
         stop_grace_period: 1m
         volumes:

From 8d804f3427b4d4c40aa6bfadba92a433bd468b26 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 20 Feb 2026 15:22:24 -0400
Subject: [PATCH 568/656] feat: button to push slide decks to Meetecho (#10431)

* refactor: eliminate inline script

Partially removes jQuery from this corner

* chore: indicate whether slide updates were sent

* feat: admin button to push slide decks to Meetecho

* test: new test

* test: cover interim case
---
 ietf/meeting/tests_views.py                 | 63 +++++++++++++++-
 ietf/meeting/urls.py                        |  3 +-
 ietf/meeting/views.py                       | 46 ++++++++++++
 ietf/static/js/session_details.js           | 53 ++++++++++++++
 ietf/templates/meeting/session_details.html | 81 ++++++---------------
 ietf/utils/meetecho.py                      | 48 +++++++++---
 ietf/utils/tests_meetecho.py                | 47 ++++++++++--
 package.json                                |  1 +
 8 files changed, 268 insertions(+), 74 deletions(-)
 create mode 100644 ietf/static/js/session_details.js

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index b94229d969..168999d0aa 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -4754,7 +4754,7 @@ def _approval_url(slidesub):
                 0,
                 "second session proposed slides should be linked for approval",
             )
-        
+
 
 class EditScheduleListTests(TestCase):
     def setUp(self):
@@ -7345,6 +7345,67 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls):
         fd.close()
         self.assertIn('third version', contents)
 
+    @override_settings(
+        MEETECHO_API_CONFIG="fake settings"
+    )  # enough to trigger API calls
+    @patch("ietf.meeting.views.SlidesManager")
+    def test_notify_meetecho_of_all_slides(self, mock_slides_manager_cls):
+        for meeting_type in ["ietf", "interim"]:
+            # Reset for the sake of the second iteration
+            self.client.logout()
+            mock_slides_manager_cls.reset_mock()
+
+            session = SessionFactory(meeting__type_id=meeting_type)
+            meeting = session.meeting
+    
+            # bad meeting
+            url = urlreverse(
+                "ietf.meeting.views.notify_meetecho_of_all_slides",
+                kwargs={"num": 9999, "acronym": session.group.acronym},
+            )
+            login_testing_unauthorized(self, "secretary", url)
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 404)
+            r = self.client.post(url)
+            self.assertEqual(r.status_code, 404)
+            self.assertFalse(mock_slides_manager_cls.called)
+            self.client.logout()
+    
+            # good meeting
+            url = urlreverse(
+                "ietf.meeting.views.notify_meetecho_of_all_slides",
+                kwargs={"num": meeting.number, "acronym": session.group.acronym},
+            )
+            login_testing_unauthorized(self, "secretary", url)
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 405)
+            self.assertFalse(mock_slides_manager_cls.called)
+            mock_slides_manager = mock_slides_manager_cls.return_value
+            mock_slides_manager.send_update.return_value = True
+            r = self.client.post(url)
+            self.assertEqual(r.status_code, 302)
+            self.assertEqual(mock_slides_manager.send_update.call_count, 1)
+            self.assertEqual(mock_slides_manager.send_update.call_args, call(session))
+            r = self.client.get(r["Location"])
+            messages = list(r.context["messages"])
+            self.assertEqual(len(messages), 1)
+            self.assertEqual(
+                str(messages[0]), f"Notified Meetecho about slides for {session}"
+            )
+    
+            mock_slides_manager.send_update.reset_mock()
+            mock_slides_manager.send_update.return_value = False
+            r = self.client.post(url)
+            self.assertEqual(r.status_code, 302)
+            self.assertEqual(mock_slides_manager.send_update.call_count, 1)
+            self.assertEqual(mock_slides_manager.send_update.call_args, call(session))
+            r = self.client.get(r["Location"])
+            messages = list(r.context["messages"])
+            self.assertEqual(len(messages), 1)
+            self.assertIn(
+                "No sessions were eligible for Meetecho slides update.", str(messages[0])
+            )
+
 
 @override_settings(IETF_NOTES_URL='https://notes.ietf.org/')
 class ImportNotesTests(TestCase):
diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py
index af36a6656c..a038e1cfe6 100644
--- a/ietf/meeting/urls.py
+++ b/ietf/meeting/urls.py
@@ -15,6 +15,7 @@ def get_redirect_url(self, *args, **kwargs):
 
 safe_for_all_meeting_types = [
     url(r'^session/(?P<acronym>[-a-z0-9]+)/?$',  views.session_details),
+    url(r'^session/(?P<acronym>[-a-z0-9]+)/send_slide_notifications$',  views.notify_meetecho_of_all_slides),
     url(r'^session/(?P<session_id>\d+)/drafts$',  views.add_session_drafts),
     url(r'^session/(?P<session_id>\d+)/recordings$', views.add_session_recordings),
     url(r'^session/(?P<session_id>\d+)/attendance$', views.session_attendance),
@@ -30,7 +31,7 @@ def get_redirect_url(self, *args, **kwargs):
     url(r'^session/(?P<session_id>\d+)/doc/%(name)s/remove$' % settings.URL_REGEXPS, views.remove_sessionpresentation),
     url(r'^session/(?P<session_id>\d+)\.ics$',    views.agenda_ical),
     url(r'^sessions/(?P<acronym>[-a-z0-9]+)\.ics$', views.agenda_ical),
-    url(r'^slidesubmission/(?P<slidesubmission_id>\d+)$', views.approve_proposed_slides)
+    url(r'^slidesubmission/(?P<slidesubmission_id>\d+)$', views.approve_proposed_slides),
 ]
 
 
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 8dccda9c87..731dfad88f 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -5710,6 +5710,52 @@ def approve_proposed_slides(request, slidesubmission_id, num):
                   })
 
 
+@role_required("Secretariat")
+def notify_meetecho_of_all_slides(request, num, acronym):
+    """Notify meetecho of state of all slides for the group
+
+    Respects the usual notification window around each session. Meetecho will ignore
+    notices outside that window anyway, so no sense sending them.
+    """
+    meeting = get_meeting(num=num, type_in=None)  # raises 404
+    if request.method != "POST":
+        return HttpResponseNotAllowed(
+            content="Method not allowed",
+            content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}",
+            permitted_methods=("POST",),
+        )
+    scheduled_sessions = [
+        session
+        for session in get_sessions(meeting.number, acronym)
+        if session.current_status == "sched"
+    ]
+    sm = SlidesManager(api_config=settings.MEETECHO_API_CONFIG)
+    updated = []
+    for session in scheduled_sessions:
+        if sm.send_update(session):
+            updated.append(session)
+    if len(updated) > 0:
+        messages.success(
+            request,
+            f"Notified Meetecho about slides for {','.join(str(s) for s in updated)}",
+        )
+    elif sm.slides_notify_time is not None:
+        messages.warning(
+            request,
+            "No sessions were eligible for Meetecho slides update. Updates are "
+            f"only sent within {sm.slides_notify_time} before or after the session.",
+        )
+    else:
+        messages.warning(
+            request,
+            "No sessions were eligible for Meetecho slides update. Updates are "
+            "currently disabled.",
+        )
+    return redirect(
+        "ietf.meeting.views.session_details", num=meeting.number, acronym=acronym
+    )
+
+
 def import_session_minutes(request, session_id, num):
     """Import session minutes from the ietf.notes.org site
 
diff --git a/ietf/static/js/session_details.js b/ietf/static/js/session_details.js
new file mode 100644
index 0000000000..03d1b2d3d9
--- /dev/null
+++ b/ietf/static/js/session_details.js
@@ -0,0 +1,53 @@
+// Copyright The IETF Trust 2026, All Rights Reserved
+// Relies on other scripts being loaded, see usage in session_details.html
+document.addEventListener('DOMContentLoaded', () => {
+    // Init with best guess at local timezone.
+    ietf_timezone.set_tz_change_callback(timezone_changed) // cb is in upcoming.js
+    ietf_timezone.initialize('local')
+
+    // Set up sortable elements if the user can manage materials
+    if (document.getElementById('can-manage-materials-flag')) {
+        const sortables = []
+        const options = {
+            group: 'slides',
+            animation: 150,
+            handle: '.drag-handle',
+            onAdd: function (event) {onAdd(event)},
+            onRemove: function (event) {onRemove(event)},
+            onEnd: function (event) {onEnd(event)}
+        }
+
+        function onAdd (event) {
+            const old_session = event.from.getAttribute('data-session')
+            const new_session = event.to.getAttribute('data-session')
+            $.post(event.to.getAttribute('data-add-to-session'), {
+                'order': event.newIndex + 1,
+                'name': event.item.getAttribute('name')
+            })
+            $(event.item).find('td:eq(1)').find('a').each(function () {
+                $(this).attr('href', $(this).attr('href').replace(old_session, new_session))
+            })
+        }
+
+        function onRemove (event) {
+            const old_session = event.from.getAttribute('data-session')
+            $.post(event.from.getAttribute('data-remove-from-session'), {
+                'oldIndex': event.oldIndex + 1,
+                'name': event.item.getAttribute('name')
+            })
+        }
+
+        function onEnd (event) {
+            if (event.to == event.from) {
+                $.post(event.from.getAttribute('data-reorder-in-session'), {
+                    'oldIndex': event.oldIndex + 1,
+                    'newIndex': event.newIndex + 1
+                })
+            }
+        }
+
+        for (const elt of document.querySelectorAll('.slides tbody')) {
+            sortables.push(Sortable.create(elt, options))
+        }
+    }
+})
diff --git a/ietf/templates/meeting/session_details.html b/ietf/templates/meeting/session_details.html
index 55fa3d3857..a4d9ba1090 100644
--- a/ietf/templates/meeting/session_details.html
+++ b/ietf/templates/meeting/session_details.html
@@ -1,5 +1,5 @@
 {% extends "base.html" %}
-{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{# Copyright The IETF Trust 2015-2026, All Rights Reserved #}
 {% load origin ietf_filters static %}
 {% block title %}{{ meeting }} : {{ group.acronym }}{% endblock %}
 {% block morecss %}
@@ -53,69 +53,36 @@ <h2 class="mt-3">Unscheduled Sessions</h2>
             {% endif %}
         {% if forloop.last %}</div>{% endif %}
     {% endfor %}
+    {% if user|has_role:"Secretariat" %}
+        <div class="card text-dark bg-warning mt-5 mb-3">
+            <div class="card-header">
+                Secretariat Only
+            </div>
+            <div class="card-body">
+                <form method="post"
+                      action="{% url 'ietf.meeting.views.notify_meetecho_of_all_slides' num=meeting.number acronym=group.acronym %}">
+                    {% csrf_token %}
+                    <button class="btn btn-primary" type="submit">
+                        Notify Meetecho of Slides
+                    </button>
+                </form>
+            </div>
+        </div>
+    {% endif %}
+    {% comment %}
+    The existence of an element with id canManageMaterialsFlag is checked in
+    session_details.js to determine whether it should init the sortable tables.
+    Not the most elegant approach, but it works.
+    {% endcomment %}
+    {% if can_manage_materials %} <div id="can-manage-materials-flag"></div>{% endif %}
 {% endblock %}
 {% block js %}
     <script src="{% static 'ietf/js/list.js' %}"></script>
     <script src="{% static 'ietf/js/moment.js' %}"></script>
     <script src="{% static 'ietf/js/upcoming.js' %}"></script>
     <script src="{% static 'ietf/js/timezone.js' %}"></script>
-    <script>
-    $(function () {
-            // Init with best guess at local timezone.
-            ietf_timezone.set_tz_change_callback(timezone_changed);
-            ietf_timezone.initialize('local');
-    });
-    </script>
     {% if can_manage_materials %}
         <script src="{% static 'ietf/js/sortable.js' %}"></script>
-        <script>
-            var sortables=[];
-            var options = {
-                group: "slides",
-                animation: 150,
-                handle: ".drag-handle",
-                onAdd: function(event) {onAdd(event)},
-                onRemove: function(event) {onRemove(event)},
-                onEnd: function(event) {onEnd(event)}
-            };
-
-            function onAdd(event) {
-                var old_session = event.from.getAttribute("data-session");
-                var new_session = event.to.getAttribute("data-session");
-                $.post(event.to.getAttribute("data-add-to-session"), {
-                    'order': event.newIndex + 1,
-                    'name': event.item.getAttribute("name")
-                });
-                $(event.item).find("td:eq(1)").find("a").each(function(){
-                    $(this).attr("href", $(this).attr("href").replace(old_session,new_session) );
-                });
-            }
-
-            function onRemove(event) {
-                var old_session = event.from.getAttribute("data-session");
-                $.post(event.from.getAttribute("data-remove-from-session"),{
-                    'oldIndex': event.oldIndex + 1,
-                    'name': event.item.getAttribute("name")
-                });
-            }
-
-            function onEnd(event) {
-                if (event.to == event.from) {
-                    $.post(event.from.getAttribute("data-reorder-in-session"),{
-                        'oldIndex': event.oldIndex + 1,
-                        'newIndex': event.newIndex + 1
-                    });
-                }
-            }
-
-            $(document).ready(function() {
-
-                $(".slides tbody").each(function() {
-                    sortables.push(Sortable.create(this, options));
-                });
-
-            });
-
-        </script>
     {% endif %}
+    <script src="{% static 'ietf/js/session_details.js' %}"></script>
 {% endblock %}
\ No newline at end of file
diff --git a/ietf/utils/meetecho.py b/ietf/utils/meetecho.py
index 7654f67cd1..943f3789ef 100644
--- a/ietf/utils/meetecho.py
+++ b/ietf/utils/meetecho.py
@@ -508,8 +508,13 @@ def _should_send_update(self, session):
             return (timeslot.time - self.slides_notify_time) < now < (timeslot.end_time() + self.slides_notify_time)
 
     def add(self, session: "Session", slides: "Document", order: int):
+        """Add a slide deck to the session
+        
+        Returns True if the update was sent, False if it was not sent because the
+        current time is outside the update window for the session.
+        """
         if not self._should_send_update(session):
-            return
+            return False
 
         # Would like to confirm that session.presentations includes the slides Document, but we can't
         # (same problem regarding unsaved Documents discussed in the docstring)
@@ -524,11 +529,16 @@ def add(self, session: "Session", slides: "Document", order: int):
                 "order": order,
             }
         )
+        return True
 
     def delete(self, session: "Session", slides: "Document"):
-        """Delete a slide deck from the session"""
+        """Delete a slide deck from the session
+        
+        Returns True if the update was sent, False if it was not sent because the
+        current time is outside the update window for the session.
+        """
         if not self._should_send_update(session):
-            return
+            return False
 
         if session.presentations.filter(document=slides).exists():
             # "order" problems are very likely to result if we delete slides that are actually still
@@ -543,12 +553,17 @@ def delete(self, session: "Session", slides: "Document"):
             id=slides.pk,
         )
         if session.presentations.filter(document__type_id="slides").exists():
-            self.send_update(session)  # adjust order to fill in the hole        
+            self._send_update(session)  # adjust order to fill in the hole
+        return True
     
     def revise(self, session: "Session", slides: "Document"):
-        """Replace existing deck with its current state"""
+        """Replace existing deck with its current state
+        
+        Returns True if the update was sent, False if it was not sent because the
+        current time is outside the update window for the session.
+        """
         if not self._should_send_update(session):
-            return
+            return False
 
         sp = session.presentations.filter(document=slides).first()
         if sp is None:
@@ -561,11 +576,13 @@ def revise(self, session: "Session", slides: "Document"):
             id=slides.pk,
         )
         self.add(session, slides, order)  # fill in the hole
+        return True
         
-    def send_update(self, session: "Session"):
-        if not self._should_send_update(session):
-            return
-
+    def _send_update(self, session: "Session"):
+        """Notify of the current state of the session's slides (no time window check)
+        
+        This is a private helper - use send_update() (no leading underscore) instead.
+        """
         self.api.update_slide_decks(
             wg_token=self.wg_token(session.group),
             session=str(session.pk),
@@ -580,3 +597,14 @@ def send_update(self, session: "Session"):
                 for deck in session.presentations.filter(document__type="slides")
             ]
         )
+
+    def send_update(self, session: "Session"):
+        """Notify of the current state of the session's slides
+        
+        Returns True if the update was sent, False if it was not sent because the
+        current time is outside the update window for the session.
+        """
+        if not self._should_send_update(session):
+            return False
+        self._send_update(session)
+        return True
diff --git a/ietf/utils/tests_meetecho.py b/ietf/utils/tests_meetecho.py
index 502e936483..c076a3df74 100644
--- a/ietf/utils/tests_meetecho.py
+++ b/ietf/utils/tests_meetecho.py
@@ -547,7 +547,8 @@ def test_add(self, mock_add, mock_wg_token):
         sm = SlidesManager(settings.MEETECHO_API_CONFIG)
         session = SessionFactory()
         slides_doc = DocumentFactory(type_id="slides")
-        sm.add(session, slides_doc, 13)
+        retval = sm.add(session, slides_doc, 13)
+        self.assertIs(retval, True)
         self.assertTrue(mock_wg_token.called)
         self.assertTrue(mock_add.called)
         self.assertEqual(
@@ -565,6 +566,14 @@ def test_add(self, mock_add, mock_wg_token):
             ),
         )
 
+        # Test return value when no update is sent. Really ought to do a more
+        # careful test of the _should_send_update() method.
+        sm = SlidesManager(
+            settings.MEETECHO_API_CONFIG | {"slides_notify_time": None}
+        )
+        retval = sm.add(session, slides_doc, 14)
+        self.assertIs(retval, False)
+
     @patch("ietf.utils.meetecho.MeetechoAPI.update_slide_decks")
     @patch("ietf.utils.meetecho.MeetechoAPI.delete_slide_deck")
     def test_delete(self, mock_delete, mock_update, mock_wg_token):
@@ -580,7 +589,8 @@ def test_delete(self, mock_delete, mock_update, mock_wg_token):
             sm.delete(session, slides_doc)  # can't remove slides still attached to the session
         self.assertFalse(any([mock_wg_token.called, mock_delete.called, mock_update.called]))
 
-        sm.delete(session, removed_slides_doc)
+        retval = sm.delete(session, removed_slides_doc)
+        self.assertIs(retval, True)
         self.assertTrue(mock_wg_token.called)
         self.assertTrue(mock_delete.called)
         self.assertEqual(
@@ -609,9 +619,18 @@ def test_delete(self, mock_delete, mock_update, mock_wg_token):
         
         # Delete the other session and check that we don't make the update call
         slides.delete()
-        sm.delete(session, slides_doc)
+        retval = sm.delete(session, slides_doc)
+        self.assertIs(retval, True)
         self.assertTrue(mock_delete.called)
         self.assertFalse(mock_update.called)
+        
+        # Test return value when no update is sent. Really ought to do a more
+        # careful test of the _should_send_update() method.
+        sm = SlidesManager(
+            settings.MEETECHO_API_CONFIG | {"slides_notify_time": None}
+        )
+        retval = sm.delete(session, slides_doc)
+        self.assertIs(retval, False)
 
     @patch("ietf.utils.meetecho.MeetechoAPI.delete_slide_deck")
     @patch("ietf.utils.meetecho.MeetechoAPI.add_slide_deck")
@@ -619,7 +638,8 @@ def test_revise(self, mock_add, mock_delete, mock_wg_token):
         sm = SlidesManager(settings.MEETECHO_API_CONFIG)
         slides = SessionPresentationFactory(document__type_id="slides", order=23)
         slides_doc = slides.document
-        sm.revise(slides.session, slides.document)
+        retval = sm.revise(slides.session, slides_doc)
+        self.assertIs(retval, True)
         self.assertTrue(mock_wg_token.called)
         self.assertTrue(mock_delete.called)
         self.assertEqual(
@@ -642,13 +662,22 @@ def test_revise(self, mock_add, mock_delete, mock_wg_token):
             ),
         )
 
+        # Test return value when no update is sent. Really ought to do a more
+        # careful test of the _should_send_update() method.
+        sm = SlidesManager(
+            settings.MEETECHO_API_CONFIG | {"slides_notify_time": None}
+        )
+        retval = sm.revise(slides.session, slides_doc)
+        self.assertIs(retval, False)
+
 
     @patch("ietf.utils.meetecho.MeetechoAPI.update_slide_decks")
     def test_send_update(self, mock_send_update, mock_wg_token):
         sm = SlidesManager(settings.MEETECHO_API_CONFIG)
         slides = SessionPresentationFactory(document__type_id="slides")
         SessionPresentationFactory(session=slides.session, document__type_id="agenda")
-        sm.send_update(slides.session)
+        retval = sm.send_update(slides.session)
+        self.assertIs(retval, True)
         self.assertTrue(mock_wg_token.called)
         self.assertTrue(mock_send_update.called)
         self.assertEqual(
@@ -667,3 +696,11 @@ def test_send_update(self, mock_send_update, mock_wg_token):
                 ]
             )
         )
+
+        # Test return value when no update is sent. Really ought to do a more
+        # careful test of the _should_send_update() method.
+        sm = SlidesManager(
+            settings.MEETECHO_API_CONFIG | {"slides_notify_time": None}
+        )
+        retval = sm.send_update(slides.session)
+        self.assertIs(retval, False)
diff --git a/package.json b/package.json
index e2e6fd7dab..fec29275b4 100644
--- a/package.json
+++ b/package.json
@@ -148,6 +148,7 @@
         "ietf/static/js/moment.js",
         "ietf/static/js/password_strength.js",
         "ietf/static/js/select2.js",
+        "ietf/static/js/session_details.js",
         "ietf/static/js/session_details_form.js",
         "ietf/static/js/session_form.js",
         "ietf/static/js/session_request.js",

From 619b2aee0f3b4acbc95a44f6fd3e8785163f6a93 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 20 Feb 2026 13:22:49 -0600
Subject: [PATCH 569/656] fix: adjust draft-stream-ietf state descriptions per
 IESG (#10437)

---
 ...ge_draft_stream_ietf_state_descriptions.py | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py

diff --git a/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py b/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py
new file mode 100644
index 0000000000..c664126da3
--- /dev/null
+++ b/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py
@@ -0,0 +1,57 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+    State = apps.get_model("doc", "State")
+    for name, desc in [
+        (
+            "Adopted by a WG",
+            "The individual submission document has been adopted by the Working Group (WG), but some administrative matter still needs to be completed (e.g., a WG document replacing this document with the typical naming convention of 'draft-ietf-wgname-topic-nn' has not yet been submitted).",
+        ),
+        (
+            "WG Document",
+            "The document has been identified as a Working Group (WG) document and is under development per Section 7.2 of RFC2418.",
+        ),
+        (
+            "Waiting for WG Chair Go-Ahead",
+            "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chairs are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed.",
+        ),
+        (
+            "Submitted to IESG for Publication",
+            "The Working Group (WG) document has been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication per Section 7.4 of RFC2418.  See the “IESG State” or “RFC Editor State” for further details on the state of the document.",
+        ),
+    ]:
+        State.objects.filter(name=name).update(desc=desc, type="draft-stream-ietf")
+
+
+def reverse(apps, schema_editor):
+    State = apps.get_model("doc", "State")
+    for name, desc in [
+        (
+            "Adopted by a WG",
+            "The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted.",
+        ),
+        (
+            "WG Document",
+            "The document has been adopted by the Working Group (WG) and is under development.  A document can only be adopted by one WG at a time.  However, a document may be transferred between WGs.",
+        ),
+        (
+            "Waiting for WG Chair Go-Ahead",
+            "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed",
+        ),
+        (
+            "Submitted to IESG for Publication",
+            "The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication.  See the “IESG State” or “RFC Editor State” for further details on the state of the document.",
+        ),
+    ]:
+        State.objects.filter(name=name).update(desc=desc, type="draft-stream-ietf")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("doc", "0030_alter_dochistory_title_alter_document_title"),
+    ]
+
+    operations = [migrations.RunPython(forward, reverse)]

From c4be6318f73cbf896b5cc1f3416040e12b4611f4 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 20 Feb 2026 15:25:57 -0400
Subject: [PATCH 570/656] feat: more API fields+filtering; drop RfcAuthor.email
 field (#10432)

* feat: RfcAuthorSerializer.email is current email

* refactor: RfcAuthor email field -> property

* feat: more RfcMetadataSerializer fields

* shepherd email (with a fallback to the draft)
* doc ad email
* area ad emails
* group list email

* fix: filter RFCs by any group type

* feat: filter by RFC numbers

* fix: shepherd -> draft object in response JSON

* fix: consistent filter naming

* chore: migration

* test: update test_notify_rfc_published

* fix: RfcAuthor.email() -> Email, not str

* fix: update RfcAuthorFactory

* fix: consistent blank value in email()

* fix: guard against non-prefetched queryset

* test: fix nomcom test

* refactor: name-addr -> addr for ad/shepherd

Also falls back to current primary email for ad/shepherd if the email on
record is inactive.
---
 ietf/api/tests_views_rpc.py                   | 25 +++++-----
 ietf/doc/admin.py                             |  4 +-
 ietf/doc/api.py                               | 10 +++-
 ietf/doc/factories.py                         |  1 -
 .../migrations/0031_remove_rfcauthor_email.py | 16 ++++++
 ietf/doc/models.py                            |  6 ++-
 ietf/doc/serializers.py                       | 50 +++++++++++++++----
 ietf/doc/views_doc.py                         |  2 +-
 ietf/group/serializers.py                     | 32 ++++++++++--
 ietf/nomcom/tests.py                          |  1 -
 10 files changed, 115 insertions(+), 32 deletions(-)
 create mode 100644 ietf/doc/migrations/0031_remove_rfcauthor_email.py

diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py
index 09fb40bf6e..1fbb4c3f02 100644
--- a/ietf/api/tests_views_rpc.py
+++ b/ietf/api/tests_views_rpc.py
@@ -143,22 +143,23 @@ def test_notify_rfc_published(self):
         self.assertEqual(rfc.title, "RFC " + draft.title)
         self.assertEqual(rfc.documentauthor_set.count(), 0)
         self.assertEqual(
-            list(
-                rfc.rfcauthor_set.values(
-                    "titlepage_name",
-                    "is_editor",
-                    "person",
-                    "email",
-                    "affiliation",
-                    "country",
-                )
-            ),
+            [
+                {
+                    "titlepage_name": ra.titlepage_name,
+                    "is_editor": ra.is_editor,
+                    "person": ra.person,
+                    "email": ra.email,
+                    "affiliation": ra.affiliation,
+                    "country": ra.country,
+                }
+                for ra in rfc.rfcauthor_set.all()
+            ],
             [
                 {
                     "titlepage_name": f"titlepage {author.name}",
                     "is_editor": False,
-                    "person": author.pk,
-                    "email": author.email_address(),
+                    "person": author,
+                    "email": author.email(),
                     "affiliation": "Some Affiliation",
                     "country": "CA",
                 }
diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py
index f082418935..b604d4f096 100644
--- a/ietf/doc/admin.py
+++ b/ietf/doc/admin.py
@@ -242,6 +242,6 @@ def is_deleted(self, instance):
 
 class RfcAuthorAdmin(admin.ModelAdmin):
     list_display = ['id', 'document', 'titlepage_name', 'person', 'email', 'affiliation', 'country', 'order']
-    search_fields = ['document__name', 'titlepage_name', 'person__name', 'email__address', 'affiliation', 'country']
-    raw_id_fields = ["document", "person", "email"]
+    search_fields = ['document__name', 'titlepage_name', 'person__name', 'email', 'affiliation', 'country']
+    raw_id_fields = ["document", "person"]
 admin.site.register(RfcAuthor, RfcAuthorAdmin)
diff --git a/ietf/doc/api.py b/ietf/doc/api.py
index 6a4c0c9fd5..75993f463e 100644
--- a/ietf/doc/api.py
+++ b/ietf/doc/api.py
@@ -42,13 +42,21 @@ class RfcLimitOffsetPagination(LimitOffsetPagination):
     max_limit = 500
 
 
+class NumberInFilter(filters.BaseInFilter, filters.NumberFilter):
+    """Filter against a comma-separated list of numbers"""
+    pass
+
+
 class RfcFilter(filters.FilterSet):
     published = filters.DateFromToRangeFilter()
     stream = filters.ModelMultipleChoiceFilter(
         queryset=StreamName.objects.filter(used=True)
     )
+    number = NumberInFilter(
+        field_name="rfc_number"
+    )
     group = filters.ModelMultipleChoiceFilter(
-        queryset=Group.objects.wgs(),
+        queryset=Group.objects.all(),
         field_name="group__acronym",
         to_field_name="acronym",
     )
diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py
index aad01be04f..bc38765446 100644
--- a/ietf/doc/factories.py
+++ b/ietf/doc/factories.py
@@ -391,7 +391,6 @@ class Meta:
         lambda obj: " ".join([obj.person.initials(), obj.person.last_name()])
     )
     person = factory.SubFactory('ietf.person.factories.PersonFactory')
-    email = factory.LazyAttribute(lambda obj: obj.person.email())
     affiliation = factory.Faker('company')
     order = factory.LazyAttribute(lambda o: o.document.rfcauthor_set.count() + 1)
 
diff --git a/ietf/doc/migrations/0031_remove_rfcauthor_email.py b/ietf/doc/migrations/0031_remove_rfcauthor_email.py
new file mode 100644
index 0000000000..c4c1911bfe
--- /dev/null
+++ b/ietf/doc/migrations/0031_remove_rfcauthor_email.py
@@ -0,0 +1,16 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("doc", "0030_alter_dochistory_title_alter_document_title"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="rfcauthor",
+            name="email",
+        ),
+    ]
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index 8f700bf496..cc28951be0 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -937,7 +937,6 @@ class RfcAuthor(models.Model):
     titlepage_name = models.CharField(max_length=128, blank=False)
     is_editor = models.BooleanField(default=False)
     person = ForeignKey(Person, null=True, blank=True, on_delete=models.PROTECT)
-    email = ForeignKey(Email, help_text="Email address used by author for submission", blank=True, null=True, on_delete=models.PROTECT)
     affiliation = models.CharField(max_length=100, blank=True, help_text="Organization/company used by author for submission")
     country = models.CharField(max_length=255, blank=True, help_text="Country used by author for submission")
     order = models.IntegerField(default=1)
@@ -951,6 +950,11 @@ class Meta:
             models.Index(fields=["document", "order"])
         ]
 
+    @property
+    def email(self) -> Email | None:
+        return self.person.email() if self.person else None
+
+
 class DocumentAuthorInfo(models.Model):
     person = ForeignKey(Person)
     # email should only be null for some historic documents
diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py
index e8d373164b..b054b074d7 100644
--- a/ietf/doc/serializers.py
+++ b/ietf/doc/serializers.py
@@ -9,14 +9,20 @@
 from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
 
-from ietf.group.serializers import GroupSerializer
+from ietf.group.serializers import (
+    AreaDirectorSerializer,
+    AreaSerializer,
+    GroupSerializer,
+)
 from ietf.name.serializers import StreamNameSerializer
+from ietf.utils import log
 from .models import Document, DocumentAuthor, RfcAuthor
 
 
 class RfcAuthorSerializer(serializers.ModelSerializer):
     """Serializer for an RfcAuthor / DocumentAuthor in a response"""
 
+    email = serializers.EmailField(source="email.address", read_only=True)
     datatracker_person_path = serializers.URLField(
         source="person.get_absolute_url",
         required=False,
@@ -29,7 +35,7 @@ class Meta:
             "titlepage_name",
             "is_editor",
             "person",
-            "email",  # relies on email.pk being email.address
+            "email",
             "affiliation",
             "country",
             "datatracker_person_path",
@@ -48,7 +54,6 @@ def to_representation(self, instance):
                 titlepage_name=document_author.person.plain_name(),
                 is_editor=False,
                 person=document_author.person,
-                email=document_author.email,
                 affiliation=document_author.affiliation,
                 country=document_author.country,
                 order=document_author.order,
@@ -174,10 +179,16 @@ def to_representation(self, instance: Document):
         return super().to_representation(instance=RfcStatus.from_document(instance))
 
 
+class ShepherdSerializer(serializers.Serializer):
+    email = serializers.EmailField(source="email_address")
+
+
 class RelatedDraftSerializer(serializers.Serializer):
     id = serializers.IntegerField(source="source.id")
     name = serializers.CharField(source="source.name")
     title = serializers.CharField(source="source.title")
+    shepherd = ShepherdSerializer(source="source.shepherd")
+    ad = AreaDirectorSerializer(source="source.ad")
 
 
 class RelatedRfcSerializer(serializers.Serializer):
@@ -205,15 +216,23 @@ class RfcFormatSerializer(serializers.Serializer):
 
 
 class RfcMetadataSerializer(serializers.ModelSerializer):
-    """Serialize metadata of an RFC"""
+    """Serialize metadata of an RFC
+
+    This needs to be called with a Document queryset that has been processed with
+    api.augment_rfc_queryset() or it very likely will not work. Some of the typing
+    refers to Document, but this should really be WithAnnotations[Document, ...].
+    However, have not been able to make that work yet.
+    """
 
     number = serializers.IntegerField(source="rfc_number")
     published = serializers.DateField()
     status = RfcStatusSerializer(source="*")
     authors = serializers.SerializerMethodField()
     group = GroupSerializer()
-    area = GroupSerializer(source="group.area", required=False)
+    area = AreaSerializer(source="group.area", required=False)
     stream = StreamNameSerializer()
+    ad = AreaDirectorSerializer(read_only=True)
+    group_list_email = serializers.EmailField(source="group.list_email", read_only=True)
     identifiers = serializers.SerializerMethodField()
     draft = serializers.SerializerMethodField()
     obsoletes = RelatedRfcSerializer(many=True, read_only=True)
@@ -239,6 +258,8 @@ class Meta:
             "group",
             "area",
             "stream",
+            "ad",
+            "group_list_email",
             "identifiers",
             "obsoletes",
             "obsoleted_by",
@@ -276,11 +297,20 @@ def get_identifiers(self, doc: Document):
         return DocIdentifierSerializer(instance=identifiers, many=True).data
 
     @extend_schema_field(RelatedDraftSerializer)
-    def get_draft(self, object):
-        try:
-            related_doc = object.drafts[0]
-        except IndexError:
-            return None
+    def get_draft(self, doc: Document):
+        if hasattr(doc, "drafts"):
+            # This is the expected case - drafts is added by a Prefetch in
+            # the augment_rfc_queryset() method.
+            try:
+                related_doc = doc.drafts[0]
+            except IndexError:
+                return None
+        else:
+            # Fallback in case augment_rfc_queryset() was not called
+            log.log(
+                f"Warning: {self.__class__}.get_draft() called without prefetched draft"
+            )
+            related_doc = doc.came_from_draft()
         return RelatedDraftSerializer(related_doc).data
 
 
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 0578da1b77..0ae7520681 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -1657,7 +1657,7 @@ def extract_name(s):
         doc.rfcauthor_set
         if doc.type_id == "rfc" and doc.rfcauthor_set.exists()
         else doc.documentauthor_set
-    ).select_related("person", "email").order_by("order")
+    ).select_related("person").prefetch_related("person__email_set").order_by("order")
     data["authors"] = [
         {
             "name": author.titlepage_name if hasattr(author, "titlepage_name") else author.person.name,
diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py
index 08e6bba81a..85f209019c 100644
--- a/ietf/group/serializers.py
+++ b/ietf/group/serializers.py
@@ -1,11 +1,37 @@
-# Copyright The IETF Trust 2024, All Rights Reserved
+# Copyright The IETF Trust 2024-2026, All Rights Reserved
 """django-rest-framework serializers"""
+
+from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
 
-from .models import Group
+from ietf.person.models import Email
+from .models import Group, Role
 
 
 class GroupSerializer(serializers.ModelSerializer):
     class Meta:
         model = Group
-        fields = ["acronym", "name", "type"]
+        fields = ["acronym", "name", "type", "list_email"]
+
+
+class AreaDirectorSerializer(serializers.Serializer):
+    """Serialize an area director
+
+    Works with Email or Role
+    """
+
+    email = serializers.SerializerMethodField()
+
+    @extend_schema_field(serializers.EmailField)
+    def get_email(self, instance: Email | Role):
+        if isinstance(instance, Role):
+            return instance.email.email_address()
+        return instance.email_address()
+
+
+class AreaSerializer(serializers.ModelSerializer):
+    ads = AreaDirectorSerializer(many=True, read_only=True)
+
+    class Meta:
+        model = Group
+        fields = ["acronym", "name", "type", "ads"]
diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py
index b6e8c57da7..210788ce07 100644
--- a/ietf/nomcom/tests.py
+++ b/ietf/nomcom/tests.py
@@ -2528,7 +2528,6 @@ def test_get_qualified_author_queryset(self):
             document=rfc,
             person=people[0],
             titlepage_name="P. Zero",
-            email=people[0].email_set.first(),
         )
         self.assertCountEqual(
             get_qualified_author_queryset(base_qs, now - 5 * one_year, now),

From d7319030f3d3a38bf0c048e713bc1f068ca228ed Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 20 Feb 2026 16:12:43 -0400
Subject: [PATCH 571/656] chore: renumber migrations (#10441)

---
 ...remove_rfcauthor_email.py => 0032_remove_rfcauthor_email.py} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename ietf/doc/migrations/{0031_remove_rfcauthor_email.py => 0032_remove_rfcauthor_email.py} (80%)

diff --git a/ietf/doc/migrations/0031_remove_rfcauthor_email.py b/ietf/doc/migrations/0032_remove_rfcauthor_email.py
similarity index 80%
rename from ietf/doc/migrations/0031_remove_rfcauthor_email.py
rename to ietf/doc/migrations/0032_remove_rfcauthor_email.py
index c4c1911bfe..a0e147da59 100644
--- a/ietf/doc/migrations/0031_remove_rfcauthor_email.py
+++ b/ietf/doc/migrations/0032_remove_rfcauthor_email.py
@@ -5,7 +5,7 @@
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("doc", "0030_alter_dochistory_title_alter_document_title"),
+        ("doc", "0031_change_draft_stream_ietf_state_descriptions"),
     ]
 
     operations = [

From abf5e0d97ca38ede129b218d3a895f9bb5ab441b Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 20 Feb 2026 17:45:31 -0400
Subject: [PATCH 572/656] fix: allow null for shepherd/ad (#10443)

---
 ietf/doc/serializers.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py
index b054b074d7..e42a6a0293 100644
--- a/ietf/doc/serializers.py
+++ b/ietf/doc/serializers.py
@@ -187,8 +187,8 @@ class RelatedDraftSerializer(serializers.Serializer):
     id = serializers.IntegerField(source="source.id")
     name = serializers.CharField(source="source.name")
     title = serializers.CharField(source="source.title")
-    shepherd = ShepherdSerializer(source="source.shepherd")
-    ad = AreaDirectorSerializer(source="source.ad")
+    shepherd = ShepherdSerializer(source="source.shepherd", allow_null=True)
+    ad = AreaDirectorSerializer(source="source.ad", allow_null=True)
 
 
 class RelatedRfcSerializer(serializers.Serializer):
@@ -231,7 +231,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer):
     group = GroupSerializer()
     area = AreaSerializer(source="group.area", required=False)
     stream = StreamNameSerializer()
-    ad = AreaDirectorSerializer(read_only=True)
+    ad = AreaDirectorSerializer(read_only=True, allow_null=True)
     group_list_email = serializers.EmailField(source="group.list_email", read_only=True)
     identifiers = serializers.SerializerMethodField()
     draft = serializers.SerializerMethodField()

From 18902ff1be5746cafc958561e563f00be5f05176 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 24 Feb 2026 13:10:42 -0400
Subject: [PATCH 573/656] fix: more accurate group areas (red API) (#10462)

* fix: RFC area only for ietf stream

* fix: no ADs for inactive areas
---
 ietf/doc/serializers.py   | 19 ++++++++++++++++++-
 ietf/group/serializers.py |  9 ++++++++-
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py
index e42a6a0293..36076c30be 100644
--- a/ietf/doc/serializers.py
+++ b/ietf/doc/serializers.py
@@ -229,7 +229,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer):
     status = RfcStatusSerializer(source="*")
     authors = serializers.SerializerMethodField()
     group = GroupSerializer()
-    area = AreaSerializer(source="group.area", required=False)
+    area = serializers.SerializerMethodField()
     stream = StreamNameSerializer()
     ad = AreaDirectorSerializer(read_only=True, allow_null=True)
     group_list_email = serializers.EmailField(source="group.list_email", read_only=True)
@@ -287,6 +287,23 @@ def get_authors(self, doc: Document):
             many=True,
         ).data
 
+    @extend_schema_field(AreaSerializer(required=False))
+    def get_area(self, doc: Document):
+        """Get area for the RFC
+        
+        This logic might be better moved to Document or a combination of Document
+        and Group. The current (2026-02-24) Group.area() method is not strict enough:
+        it does not limit to WG groups or IETF-stream documents.
+        """
+        if doc.stream_id != "ietf":
+            return None
+        if doc.group is None:
+            return None
+        parent = doc.group.parent
+        if parent.type_id == "area":
+            return AreaSerializer(parent).data
+        return None
+
     @extend_schema_field(DocIdentifierSerializer(many=True))
     def get_identifiers(self, doc: Document):
         identifiers = []
diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py
index 85f209019c..4ba92232c5 100644
--- a/ietf/group/serializers.py
+++ b/ietf/group/serializers.py
@@ -30,8 +30,15 @@ def get_email(self, instance: Email | Role):
 
 
 class AreaSerializer(serializers.ModelSerializer):
-    ads = AreaDirectorSerializer(many=True, read_only=True)
+    ads = serializers.SerializerMethodField()
 
     class Meta:
         model = Group
         fields = ["acronym", "name", "type", "ads"]
+
+    @extend_schema_field(AreaDirectorSerializer(many=True))
+    def get_ads(self, area: Group):
+        return AreaDirectorSerializer(
+            area.ads() if area.is_active else Role.objects.none(),
+            many=True,
+        ).data

From 07efd2b078a461da2eb7e197fc91f2ae0b45ac40 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 25 Feb 2026 12:58:57 -0400
Subject: [PATCH 574/656] fix: unbreak red API + group serializer tests
 (#10467)

* test: group serializer tests

* fix: Group.ads is a property

* fix: no need for type in AreaSerializer
---
 ietf/group/serializers.py       |  4 +-
 ietf/group/tests_serializers.py | 90 +++++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+), 2 deletions(-)
 create mode 100644 ietf/group/tests_serializers.py

diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py
index 4ba92232c5..db3b37af48 100644
--- a/ietf/group/serializers.py
+++ b/ietf/group/serializers.py
@@ -34,11 +34,11 @@ class AreaSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Group
-        fields = ["acronym", "name", "type", "ads"]
+        fields = ["acronym", "name", "ads"]
 
     @extend_schema_field(AreaDirectorSerializer(many=True))
     def get_ads(self, area: Group):
         return AreaDirectorSerializer(
-            area.ads() if area.is_active else Role.objects.none(),
+            area.ads if area.is_active else Role.objects.none(),
             many=True,
         ).data
diff --git a/ietf/group/tests_serializers.py b/ietf/group/tests_serializers.py
new file mode 100644
index 0000000000..bf29e6c8fd
--- /dev/null
+++ b/ietf/group/tests_serializers.py
@@ -0,0 +1,90 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+from ietf.group.factories import RoleFactory, GroupFactory
+from ietf.group.serializers import (
+    AreaDirectorSerializer,
+    AreaSerializer,
+    GroupSerializer,
+)
+from ietf.person.factories import EmailFactory
+from ietf.utils.test_utils import TestCase
+
+
+class GroupSerializerTests(TestCase):
+    def test_serializes(self):
+        wg = GroupFactory()
+        serialized = GroupSerializer(wg).data
+        self.assertEqual(
+            serialized,
+            {
+                "acronym": wg.acronym,
+                "name": wg.name,
+                "type": "wg",
+                "list_email": wg.list_email,
+            },
+        )
+
+
+class AreaDirectorSerializerTests(TestCase):
+    def test_serializes_role(self):
+        """Should serialize a Role correctly"""
+        role = RoleFactory(group__type_id="area", name_id="ad")
+        serialized = AreaDirectorSerializer(role).data
+        self.assertEqual(
+            serialized,
+            {"email": role.email.email_address()},
+        )
+
+    def test_serializes_email(self):
+        """Should serialize an Email correctly"""
+        email = EmailFactory()
+        serialized = AreaDirectorSerializer(email).data
+        self.assertEqual(
+            serialized,
+            {"email": email.email_address()},
+        )
+
+
+class AreaSerializerTests(TestCase):
+    def test_serializes_active_area(self):
+        """Should serialize an active area correctly"""
+        area = GroupFactory(type_id="area", state_id="active")
+        serialized = AreaSerializer(area).data
+        self.assertEqual(
+            serialized,
+            {
+                "acronym": area.acronym,
+                "name": area.name,
+                "ads": [],
+            },
+        )
+        ad_roles = RoleFactory.create_batch(2, group=area, name_id="ad")
+        serialized = AreaSerializer(area).data
+        self.assertEqual(serialized["acronym"], area.acronym)
+        self.assertEqual(serialized["name"], area.name)
+        self.assertCountEqual(
+            serialized["ads"],
+            [{"email": ad.email.email_address()} for ad in ad_roles],
+        )
+
+    def test_serializes_inactive_area(self):
+        """Should serialize an inactive area correctly"""
+        area = GroupFactory(type_id="area", state_id="conclude")
+        serialized = AreaSerializer(area).data
+        self.assertEqual(
+            serialized,
+            {
+                "acronym": area.acronym,
+                "name": area.name,
+                "ads": [],
+            },
+        )
+        RoleFactory.create_batch(2, group=area, name_id="ad")
+        serialized = AreaSerializer(area).data
+        self.assertEqual(
+            serialized,
+            {
+                "acronym": area.acronym,
+                "name": area.name,
+                "ads": [],
+            },
+        )

From b81249884877e20c6e311478fe25b472c869c555 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 25 Feb 2026 13:24:30 -0600
Subject: [PATCH 575/656] feat: notify errata system of new rfc (#10465)

* feat: notify errata system of new rfc

* chore: ruff
---
 docker/configs/settings_local.py |  4 ++++
 ietf/api/tests_views_rpc.py      | 28 +++++++++++++++++++------
 ietf/api/views_rpc.py            |  9 +++++++-
 ietf/doc/tasks.py                |  5 +++++
 ietf/doc/utils_errata.py         | 35 ++++++++++++++++++++++++++++++++
 ietf/settings.py                 |  5 +++++
 6 files changed, 79 insertions(+), 7 deletions(-)
 create mode 100644 ietf/doc/utils_errata.py

diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py
index e357ce3f73..1d4e6916b9 100644
--- a/docker/configs/settings_local.py
+++ b/docker/configs/settings_local.py
@@ -105,3 +105,7 @@
     "ietf.api.red_api" : ["devtoken", "redtoken"],  # Not a real secret
     "ietf.api.views_rpc" : ["devtoken"],  # Not a real secret
 }
+
+# Errata system api configuration
+ERRATA_METADATA_NOTIFICATION_URL = "http://host.docker.internal:8808/api/rfc_metadata_update/"
+ERRATA_METADATA_NOTIFICATION_API_KEY = "not a real secret"
diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py
index 1fbb4c3f02..6a5a5c9b88 100644
--- a/ietf/api/tests_views_rpc.py
+++ b/ietf/api/tests_views_rpc.py
@@ -9,9 +9,10 @@
 from django.db.models.functions import Coalesce
 from django.test.utils import override_settings
 from django.urls import reverse as urlreverse
+import mock
 
 from ietf.blobdb.models import Blob
-from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory
+from ietf.doc.factories import IndividualDraftFactory, RfcFactory, WgDraftFactory, WgRfcFactory
 from ietf.doc.models import RelatedDocument, Document
 from ietf.group.factories import RoleFactory, GroupFactory
 from ietf.person.factories import PersonFactory
@@ -77,7 +78,8 @@ def test_draftviewset_references(self):
         self.assertEqual(refs[0]["name"], draft_bar.name)
 
     @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
-    def test_notify_rfc_published(self):
+    @mock.patch("ietf.doc.tasks.signal_update_rfc_metadata_task.delay")
+    def test_notify_rfc_published(self, mock_task_delay):
         url = urlreverse("ietf.api.purple_api.notify_rfc_published")
         area = GroupFactory(type_id="area")
         rfc_group = GroupFactory(type_id="wg")
@@ -90,6 +92,8 @@ def test_notify_rfc_published(self):
         )
         rfc_stream_id = "ise"
         assert isinstance(draft, Document), "WgDraftFactory should generate a Document"
+        updates = RfcFactory.create_batch(2)
+        obsoletes = RfcFactory.create_batch(2)
         unused_rfc_number = (
             Document.objects.filter(rfc_number__isnull=False).aggregate(
                 unused_rfc_number=Max("rfc_number") + 1
@@ -120,8 +124,8 @@ def test_notify_rfc_published(self):
             "pages": draft.pages + 10,
             "std_level": "ps",
             "ad": rfc_ad.pk,
-            "obsoletes": [],
-            "updates": [],
+            "obsoletes": [o.rfc_number for o in obsoletes],
+            "updates": [o.rfc_number for o in updates],
             "subseries": [],
         }
         r = self.client.post(url, data=post_data, format="json")
@@ -172,13 +176,25 @@ def test_notify_rfc_published(self):
         self.assertEqual(rfc.pages, draft.pages + 10)
         self.assertEqual(rfc.std_level_id, "ps")
         self.assertEqual(rfc.ad, rfc_ad)
-        self.assertEqual(rfc.related_that_doc("obs"), [])
-        self.assertEqual(rfc.related_that_doc("updates"), [])
+        self.assertEqual(set(rfc.related_that_doc("obs")), set([o for o in obsoletes]))
+        self.assertEqual(
+            set(rfc.related_that_doc("updates")), set([o for o in updates])
+        )
         self.assertEqual(rfc.part_of(), [])
         self.assertEqual(draft.get_state().slug, "rfc")
         # todo test non-empty relationships
         # todo test references (when updating that is part of the handling)
 
+        self.assertTrue(mock_task_delay.called)
+        mock_args, mock_kwargs = mock_task_delay.call_args
+        self.assertIn("rfc_number_list", mock_kwargs)
+        expected_rfc_number_list = [rfc.rfc_number]
+        expected_rfc_number_list.extend(
+            [d.rfc_number for d in updates + obsoletes]
+        )
+        expected_rfc_number_list = sorted(set(expected_rfc_number_list))
+        self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list)
+
     @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
     def test_upload_rfc_files(self):
         def _valid_post_data():
diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index 2bf16480f2..9273590b28 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -38,6 +38,7 @@
 from ietf.doc.models import Document, DocHistory, RfcAuthor
 from ietf.doc.serializers import RfcAuthorSerializer
 from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage
+from ietf.doc.tasks import signal_update_rfc_metadata_task
 from ietf.person.models import Email, Person
 
 
@@ -362,7 +363,7 @@ def post(self, request):
         serializer.is_valid(raise_exception=True)
         # Create RFC
         try:
-            serializer.save()
+            rfc = serializer.save()
         except IntegrityError as err:
             if Document.objects.filter(
                 rfc_number=serializer.validated_data["rfc_number"]
@@ -375,6 +376,12 @@ def post(self, request):
                 f"Unable to publish: {err}",
                 code="unknown-integrity-error",
             )
+        rfc_number_list = [rfc.rfc_number]
+        rfc_number_list.extend(
+            [d.rfc_number for d in rfc.related_that_doc(("updates", "obs"))]
+        )
+        rfc_number_list = sorted(set(rfc_number_list))
+        signal_update_rfc_metadata_task.delay(rfc_number_list=rfc_number_list)
         return Response(NotificationAckSerializer().data)
 
 
diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py
index b463b9cecf..90f4c80af5 100644
--- a/ietf/doc/tasks.py
+++ b/ietf/doc/tasks.py
@@ -35,6 +35,7 @@
     investigate_fragment,
 )
 from .utils_bofreq import fixup_bofreq_timestamps
+from .utils_errata import signal_update_rfc_metadata
 
 
 @shared_task
@@ -155,3 +156,7 @@ def rebuild_reference_relations_task(doc_names: list[str]):
 @shared_task
 def fixup_bofreq_timestamps_task():  # pragma: nocover
     fixup_bofreq_timestamps()
+
+@shared_task
+def signal_update_rfc_metadata_task(rfc_number_list=()):
+    signal_update_rfc_metadata(rfc_number_list)
diff --git a/ietf/doc/utils_errata.py b/ietf/doc/utils_errata.py
new file mode 100644
index 0000000000..539262151f
--- /dev/null
+++ b/ietf/doc/utils_errata.py
@@ -0,0 +1,35 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+import requests
+
+from django.conf import settings
+
+from ietf.utils.log import log
+
+
+def signal_update_rfc_metadata(rfc_number_list=()):
+    key = getattr(settings, "ERRATA_METADATA_NOTIFICATION_API_KEY", None)
+    if key is not None:
+        headers = {"X-Api-Key": settings.ERRATA_METADATA_NOTIFICATION_API_KEY}
+        post_dict = {
+            "rfc_number_list": list(rfc_number_list),
+        }
+        try:
+            response = requests.post(
+                settings.ERRATA_METADATA_NOTIFICATION_URL,
+                headers=headers,
+                json=post_dict,
+                timeout=settings.DEFAULT_REQUESTS_TIMEOUT,
+            )
+        except requests.Timeout as e:
+            log(
+                f"POST request timed out for {settings.ERRATA_METADATA_NOTIFICATION_URL} ]: {e}"
+            )
+            # raise RuntimeError(f'POST request timed out for {settings.ERRATA_METADATA_NOTIFICATION_URL}') from e
+            return
+        if response.status_code != 200:
+            log(
+                f"POST request failed for {settings.ERRATA_METADATA_NOTIFICATION_URL} ]: {response.status_code} {response.text}"
+            )
+    else:
+        log("No API key configured for errata metadata notification, skipping")
diff --git a/ietf/settings.py b/ietf/settings.py
index 565e8825a9..71b110d762 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -1368,6 +1368,11 @@ def skip_unreadable_post(record):
 MEETECHO_AUDIO_STREAM_URL = "https://mp3.conf.meetecho.com/ietf{session.meeting.number}/{session.pk}.m3u"
 MEETECHO_SESSION_RECORDING_URL = "https://meetecho-player.ietf.org/playout/?session={session_label}"
 
+# Errata system api configuration
+# settings should provide
+# ERRATA_METADATA_NOTIFICATION_URL
+# ERRATA_METADATA_NOTIFICATION_API_KEY
+
 # Put the production SECRET_KEY in settings_local.py, and also any other
 # sensitive or site-specific changes.  DO NOT commit settings_local.py to svn.
 from ietf.settings_local import *            # pyflakes:ignore pylint: disable=wildcard-import

From da5614c4963c3dc4ff8e901c1edd888296219a0d Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 25 Feb 2026 18:14:47 -0400
Subject: [PATCH 576/656] test: avoid random fail in test_rfc_index (#10469)

---
 ietf/sync/tests.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index bcc87a43aa..21d6cb5cd5 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -301,6 +301,7 @@ def test_rfc_index(self):
             ad=Person.objects.get(user__username='ad'),
             external_url="http://my-external-url.example.com",
             note="this is a note",
+            pages=54,  # make sure this is not 42
         )
         DocumentAuthorFactory.create_batch(2, document=draft_doc)
         draft_doc.action_holders.add(draft_doc.ad)  # not normally set, but add to be sure it's cleared

From c1c24d012d23135725f0206dbe1a6be1e2a7fef4 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 27 Feb 2026 14:31:40 -0400
Subject: [PATCH 577/656] feat: RFC metadata update API (#10476)

* feat: more editable RFC fields for API (WIP)

Checkpoint commit!

* chore: avoid requiring prefetch

Makes some fields write-only to achieve this.

* refactor: replace EditableRfcSerializer

* fix: mark read-only field properly

* refactor: SubseriesNameField

* test: EditableRfcSerializer

* refactor: DocEvent adjustment

* feat: record person ids for authors

* chore: adjust history message

* fix: always save!!

* fix: better msg formatting

* fix: _almost_ always save!!

* fix: lint

* refactor: rename var
---
 ietf/api/serializers_rpc.py       | 227 ++++++++++++++++++++++++++----
 ietf/api/tests_serializers_rpc.py | 139 ++++++++++++++++++
 ietf/api/views_rpc.py             |  12 +-
 ietf/doc/serializers.py           |   1 +
 ietf/doc/utils.py                 |  16 ++-
 5 files changed, 361 insertions(+), 34 deletions(-)
 create mode 100644 ietf/api/tests_serializers_rpc.py

diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py
index 34e2c791c0..d5f5363990 100644
--- a/ietf/api/serializers_rpc.py
+++ b/ietf/api/serializers_rpc.py
@@ -216,32 +216,24 @@ class Meta:
         read_only_fields = ["id", "name"]
 
 
-class EditableRfcSerializer(serializers.ModelSerializer):
-    # Would be nice to reconcile this with ietf.doc.serializers.RfcSerializer.
-    # The purposes of that serializer (representing data for Red) and this one
-    # (accepting updates from Purple) are different enough that separate formats
-    # may be needed, but if not it'd be nice to have a single RfcSerializer that
-    # can serve both.
-    #
-    # For now, only handles authors
-    authors = RfcAuthorSerializer(many=True, min_length=1, source="rfcauthor_set")
+def _update_authors(rfc, authors_data):
+    # Construct unsaved instances from validated author data
+    new_authors = [RfcAuthor(**authdata) for authdata in authors_data]
+    # Update the RFC with the new author set
+    with transaction.atomic():
+        change_events = update_rfcauthors(rfc, new_authors)
+        for event in change_events:
+            event.save()
+    return change_events
 
-    class Meta:
-        model = Document
-        fields = ["id", "authors"]
 
-    def update(self, instance, validated_data):
-        assert isinstance(instance, Document)
-        authors_data = validated_data.pop("rfcauthor_set", None)
-        if authors_data is not None:
-            # Construct unsaved instances from validated author data
-            new_authors = [RfcAuthor(**ad) for ad in authors_data]
-            # Update the RFC with the new author set
-            with transaction.atomic():
-                change_events = update_rfcauthors(instance, new_authors)
-                for event in change_events:
-                    event.save()
-        return instance
+class SubseriesNameField(serializers.RegexField):
+
+    def __init__(self, **kwargs):
+        # pattern: no leading 0, finite length (arbitrarily set to 5 digits)
+        regex = r"^(bcp|std|fyi)[1-9][0-9]{0,4}$"
+        super().__init__(regex, **kwargs)
+
 
 
 class RfcPubSerializer(serializers.ModelSerializer):
@@ -283,13 +275,7 @@ class RfcPubSerializer(serializers.ModelSerializer):
         slug_field="rfc_number",
         queryset=Document.objects.filter(type_id="rfc"),
     )
-    subseries = serializers.ListField(
-        child=serializers.RegexField(
-            required=False,
-            # pattern: no leading 0, finite length (arbitrarily set to 5 digits)
-            regex=r"^(bcp|std|fyi)[1-9][0-9]{0,4}$", 
-        )
-    )
+    subseries = serializers.ListField(child=SubseriesNameField(required=False))
     # N.b., authors is _not_ a field on Document!
     authors = RfcAuthorSerializer(many=True)
 
@@ -327,6 +313,9 @@ def validate(self, data):
                 )
         return data
 
+    def update(self, instance, validated_data):
+        raise RuntimeError("Cannot update with this serializer")
+
     def create(self, validated_data):
         """Publish an RFC"""
         published = validated_data.pop("published")
@@ -515,6 +504,182 @@ def _create_rfc(self, validated_data):
         return rfc
 
 
+class EditableRfcSerializer(serializers.ModelSerializer):
+    # Would be nice to reconcile this with ietf.doc.serializers.RfcSerializer.
+    # The purposes of that serializer (representing data for Red) and this one
+    # (accepting updates from Purple) are different enough that separate formats
+    # may be needed, but if not it'd be nice to have a single RfcSerializer that
+    # can serve both.
+    #
+    # Should also consider whether this and RfcPubSerializer should merge.
+    #
+    # Treats published and subseries fields as write-only. This isn't quite correct,
+    # but makes it easier and we don't currently use the serialized value except for
+    # debugging.
+    published = serializers.DateTimeField(
+        default_timezone=datetime.timezone.utc,
+        write_only=True,
+    )
+    authors = RfcAuthorSerializer(many=True, min_length=1, source="rfcauthor_set")
+    subseries = serializers.ListField(
+        child=SubseriesNameField(required=False),
+        write_only=True,
+    )
+
+    class Meta:
+        model = Document
+        fields = [
+            "published",
+            "title",
+            "authors",
+            "stream",
+            "abstract",
+            "pages",
+            "std_level",
+            "subseries",
+        ]
+
+    def create(self, validated_data):
+        raise RuntimeError("Cannot create with this serializer")
+
+    def update(self, instance, validated_data):
+        assert isinstance(instance, Document)
+        assert instance.type_id == "rfc"
+        rfc = instance  # get better name
+
+        system_person = Person.objects.get(name="(System)")
+
+        # Remove data that needs special handling. Use a singleton object to detect
+        # missing values in case we ever support a value that needs None as an option.
+        omitted = object()
+        published = validated_data.pop("published", omitted)
+        subseries = validated_data.pop("subseries", omitted)
+        authors_data = validated_data.pop("rfcauthor_set", omitted)
+
+        # Transaction to clean up if something fails
+        with transaction.atomic():
+            # update the rfc Document itself
+            rfc_changes = []
+            rfc_events = []
+            
+            for attr, new_value in validated_data.items():
+                old_value = getattr(rfc, attr)
+                if new_value != old_value:
+                    rfc_changes.append(
+                        f"changed {attr} to '{new_value}' from '{old_value}'"
+                    )
+                    setattr(rfc, attr, new_value)
+            if len(rfc_changes) > 0:
+                rfc_change_summary = f"{', '.join(rfc_changes)}"
+                rfc_events.append(
+                    DocEvent.objects.create(
+                        doc=rfc,
+                        rev=rfc.rev,
+                        by=system_person,
+                        type="sync_from_rfc_editor",
+                        desc=f"Changed metadata: {rfc_change_summary}",
+                    )
+                )
+            if authors_data is not omitted:
+                rfc_events.extend(_update_authors(instance, authors_data))
+
+            if published is not omitted:
+                published_event = rfc.latest_event(type="published_rfc")
+                if published_event is None:
+                    # unexpected, but possible in theory
+                    rfc_events.append(
+                        DocEvent.objects.create(
+                            doc=rfc,
+                            rev=rfc.rev,
+                            type="published_rfc",
+                            time=published,
+                            by=system_person,
+                            desc="RFC published",
+                        )
+                    )
+                    rfc_events.append(
+                        DocEvent.objects.create(
+                            doc=rfc,
+                            rev=rfc.rev,
+                            type="sync_from_rfc_editor",
+                            by=system_person,
+                            desc=(
+                                f"Set publication timestamp to {published.isoformat()}"
+                            ),
+                        )
+                    )
+                else:
+                    original_pub_time = published_event.time
+                    if published != original_pub_time:
+                        published_event.time = published
+                        published_event.save()
+                        rfc_events.append(
+                            DocEvent.objects.create(
+                                doc=rfc,
+                                rev=rfc.rev,
+                                type="sync_from_rfc_editor",
+                                by=system_person,
+                                desc=(
+                                    f"Changed publication time to "
+                                    f"{published.isoformat()} from "
+                                    f"{original_pub_time.isoformat()}"
+                                )
+                            )
+                        )
+
+            # update subseries relations
+            if subseries is not omitted:
+                for subseries_doc_name in subseries:
+                    ss_slug = subseries_doc_name[:3]
+                    subseries_doc, ss_doc_created = Document.objects.get_or_create(
+                        type_id=ss_slug, name=subseries_doc_name
+                    )
+                    if ss_doc_created:
+                        subseries_doc.docevent_set.create(
+                            type=f"{ss_slug}_doc_created",
+                            by=system_person,
+                            desc=f"Created {subseries_doc_name} via update of {rfc.name}",
+                        )
+                    _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create(
+                        relationship_id="contains", target=rfc
+                    )
+                    if ss_rel_created:
+                        subseries_doc.docevent_set.create(
+                            type="sync_from_rfc_editor",
+                            by=system_person,
+                            desc=f"Added {rfc.name} to {subseries_doc.name}",
+                        )
+                        rfc_events.append(
+                            rfc.docevent_set.create(
+                                type="sync_from_rfc_editor",
+                                by=system_person,
+                                desc=f"Added {rfc.name} to {subseries_doc.name}",
+                            )
+                        )
+                # Delete subseries relations that are no longer current
+                stale_subseries_relations = rfc.relations_that("contains").exclude(
+                    source__name__in=subseries
+                )
+                for stale_relation in stale_subseries_relations:
+                    stale_subseries_doc = stale_relation.source
+                    rfc_events.append(
+                        rfc.docevent_set.create(
+                            type="sync_from_rfc_editor",
+                            by=system_person,
+                            desc=f"Removed {rfc.name} from {stale_subseries_doc.name}",
+                        )
+                    )
+                    stale_subseries_doc.docevent_set.create(
+                        type="sync_from_rfc_editor",
+                        by=system_person,
+                        desc=f"Removed {rfc.name} from {stale_subseries_doc.name}",
+                    )
+                stale_subseries_relations.delete()
+            if len(rfc_events) > 0:
+                rfc.save_with_history(rfc_events)
+        return rfc
+
+
 class RfcFileSerializer(serializers.Serializer):
     # The structure of this serializer is constrained by what openapi-generator-cli's
     # python generator can correctly serialize as multipart/form-data. It does not
diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py
new file mode 100644
index 0000000000..1babb4c30f
--- /dev/null
+++ b/ietf/api/tests_serializers_rpc.py
@@ -0,0 +1,139 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+from django.utils import timezone
+
+from ietf.utils.test_utils import TestCase
+from ietf.doc.models import Document
+from ietf.doc.factories import WgRfcFactory
+from .serializers_rpc import EditableRfcSerializer
+
+
+class EditableRfcSerializerTests(TestCase):
+    def test_create(self):
+        serializer = EditableRfcSerializer(
+            data={
+                "published": timezone.now(),
+                "title": "Yadda yadda yadda",
+                "authors": [
+                    {
+                        "titlepage_name": "B. Fett",
+                        "is_editor": False,
+                        "affiliation": "DBA Galactic Empire",
+                        "country": "",
+                    },
+                ],
+                "stream": "ietf",
+                "abstract": "A long time ago in a galaxy far, far away...",
+                "pages": 3,
+                "std_level": "inf",
+                "subseries": ["fyi999"],
+            }
+        )
+        self.assertTrue(serializer.is_valid())
+        with self.assertRaises(RuntimeError, msg="serializer does not allow create()"):
+            serializer.save()
+
+    def test_update(self):
+        rfc = WgRfcFactory(pages=10)
+        serializer = EditableRfcSerializer(
+            instance=rfc,
+            data={
+                "published": timezone.now(),
+                "title": "Yadda yadda yadda",
+                "authors": [
+                    {
+                        "titlepage_name": "B. Fett",
+                        "is_editor": False,
+                        "affiliation": "DBA Galactic Empire",
+                        "country": "",
+                    },
+                ],
+                "stream": "ise",
+                "abstract": "A long time ago in a galaxy far, far away...",
+                "pages": 3,
+                "std_level": "inf",
+                "subseries": ["fyi999"],
+            },
+        )
+        self.assertTrue(serializer.is_valid())
+        result = serializer.save()
+        result.refresh_from_db()
+        self.assertEqual(result.title, "Yadda yadda yadda")
+        self.assertEqual(
+            list(
+                result.rfcauthor_set.values(
+                    "titlepage_name", "is_editor", "affiliation", "country"
+                )
+            ),
+            [
+                {
+                    "titlepage_name": "B. Fett",
+                    "is_editor": False,
+                    "affiliation": "DBA Galactic Empire",
+                    "country": "",
+                },
+            ],
+        )
+        self.assertEqual(result.stream_id, "ise")
+        self.assertEqual(
+            result.abstract, "A long time ago in a galaxy far, far away..."
+        )
+        self.assertEqual(result.pages, 3)
+        self.assertEqual(result.std_level_id, "inf")
+        self.assertEqual(
+            result.part_of(),
+            [Document.objects.get(name="fyi999")],
+        )
+
+    def test_partial_update(self):
+        # We could test other permutations of fields, but authors is a partial update
+        # we know we are going to use, so verifying that one in particular.
+        rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan")
+        serializer = EditableRfcSerializer(
+            partial=True,
+            instance=rfc,
+            data={
+                "authors": [
+                    {
+                        "titlepage_name": "B. Fett",
+                        "is_editor": False,
+                        "affiliation": "DBA Galactic Empire",
+                        "country": "",
+                    },
+                ],
+            },
+        )
+        self.assertTrue(serializer.is_valid())
+        result = serializer.save()
+        result.refresh_from_db()
+        self.assertEqual(rfc.title, "padawan")
+        self.assertEqual(
+            list(
+                result.rfcauthor_set.values(
+                    "titlepage_name", "is_editor", "affiliation", "country"
+                )
+            ),
+            [
+                {
+                    "titlepage_name": "B. Fett",
+                    "is_editor": False,
+                    "affiliation": "DBA Galactic Empire",
+                    "country": "",
+                },
+            ],
+        )
+        self.assertEqual(result.stream_id, "ietf")
+        self.assertEqual(result.abstract, "do or do not")
+        self.assertEqual(result.pages, 10)
+        self.assertEqual(result.std_level_id, "ps")
+        self.assertEqual(result.part_of(), [])
+
+        # Test only a field on the Document itself to be sure that it works
+        serializer = EditableRfcSerializer(
+            partial=True,
+            instance=rfc,
+            data={"title": "jedi master"},
+        )
+        self.assertTrue(serializer.is_valid())
+        result = serializer.save()
+        result.refresh_from_db()
+        self.assertEqual(rfc.title, "jedi master")
diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index 9273590b28..8862bbf866 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -35,7 +35,7 @@
     NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer,
     EditableRfcSerializer,
 )
-from ietf.doc.models import Document, DocHistory, RfcAuthor
+from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent
 from ietf.doc.serializers import RfcAuthorSerializer
 from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage
 from ietf.doc.tasks import signal_update_rfc_metadata_task
@@ -279,6 +279,16 @@ class RfcViewSet(mixins.UpdateModelMixin, viewsets.GenericViewSet):
     lookup_field = "rfc_number"
     serializer_class = EditableRfcSerializer
 
+    def perform_update(self, serializer):
+        DocEvent.objects.create(
+            doc=serializer.instance,
+            rev=serializer.instance.rev,
+            by=Person.objects.get(name="(System)"),
+            type="sync_from_rfc_editor",
+            desc="Metadata update from RFC Editor",
+        )
+        super().perform_update(serializer)
+
     @action(detail=False, serializer_class=OriginalStreamSerializer)
     def rfc_original_stream(self, request):
         rfcs = self.get_queryset().annotate(
diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py
index 36076c30be..a7ea640be8 100644
--- a/ietf/doc/serializers.py
+++ b/ietf/doc/serializers.py
@@ -27,6 +27,7 @@ class RfcAuthorSerializer(serializers.ModelSerializer):
         source="person.get_absolute_url",
         required=False,
         help_text="URL for person link (relative to datatracker base URL)",
+        read_only=True,
     )
 
     class Meta:
diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index 42fab7d472..396b3fcfa4 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -740,14 +740,26 @@ def _rfcauthor_from_documentauthor(docauthor: DocumentAuthor) -> RfcAuthor:
             new_author.document = rfc
             new_author.order = order + 1
             new_author.save()
-            changes.append(f'Added "{new_author.titlepage_name}" as author')
+            if new_author.person_id is not None:
+                person_desc = f"Person {new_author.person_id}"
+            else:
+                person_desc = "no Person linked"
+            changes.append(
+                f'Added "{new_author.titlepage_name}" ({person_desc}) as author'
+            )
     # Any authors left in original_authors are no longer in the list, so remove them
     for removed_author in original_authors:
         # Skip actual removal of old authors if we are converting from the
         # DocumentAuthor models - the original_authors were just stand-ins anyway.
         if not converting_from_docauthors:
             removed_author.delete()
-        changes.append(f'Removed "{removed_author.titlepage_name}" as author')
+        if removed_author.person_id is not None:
+            person_desc = f"Person {removed_author.person_id}"
+        else:
+            person_desc = "no Person linked"
+        changes.append(
+            f'Removed "{removed_author.titlepage_name}" ({person_desc}) as author'
+        )
     # Create DocEvents, but leave it up to caller to save
     if by is None:
         by = Person.objects.get(name="(System)")

From 481054511b9f07a47c41f854105e00616e61d3e2 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 3 Mar 2026 11:36:45 -0400
Subject: [PATCH 578/656] feat: add area to FullDraftSerializer (#10487)

* refactor: Document.area() + serializer

* feat: add area to FullDraftSerializer
---
 ietf/api/serializers_rpc.py |  3 +++
 ietf/doc/models.py          | 16 ++++++++++++++++
 ietf/doc/serializers.py     | 19 +------------------
 3 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py
index d5f5363990..e51b917be4 100644
--- a/ietf/api/serializers_rpc.py
+++ b/ietf/api/serializers_rpc.py
@@ -27,6 +27,7 @@
     update_rfcauthors,
 )
 from ietf.group.models import Group
+from ietf.group.serializers import AreaSerializer
 from ietf.name.models import StreamName, StdLevelName
 from ietf.person.models import Person
 from ietf.utils import log
@@ -115,6 +116,7 @@ class FullDraftSerializer(serializers.ModelSerializer):
     name = serializers.CharField(max_length=255)
     title = serializers.CharField(max_length=255)
     group = serializers.SlugRelatedField(slug_field="acronym", read_only=True)
+    area = AreaSerializer(read_only=True)
 
     # Other fields we need to add / adjust
     source_format = serializers.SerializerMethodField()
@@ -133,6 +135,7 @@ class Meta:
             "stream",
             "title",
             "group",
+            "area",
             "abstract",
             "pages",
             "source_format",
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index cc28951be0..f1b319367e 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -1147,6 +1147,22 @@ def request_closed_time(self, review_req):
         e = self.latest_event(ReviewRequestDocEvent, type="closed_review_request", review_request=review_req)
         return e.time if e and e.time else None
 
+    @property
+    def area(self) -> Group | None:
+        """Get area for document, if one exists
+        
+        None for non-IETF-stream documents. N.b., this is stricter than Group.area() and
+        uses different logic from Document.area_acronym().
+        """
+        if self.stream_id != "ietf":
+            return None
+        if self.group is None:
+            return None
+        parent = self.group.parent
+        if parent.type_id == "area":
+            return parent
+        return None
+
     def area_acronym(self):
         g = self.group
         if g:
diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py
index a7ea640be8..139ae9aa7e 100644
--- a/ietf/doc/serializers.py
+++ b/ietf/doc/serializers.py
@@ -230,7 +230,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer):
     status = RfcStatusSerializer(source="*")
     authors = serializers.SerializerMethodField()
     group = GroupSerializer()
-    area = serializers.SerializerMethodField()
+    area = AreaSerializer(read_only=True)
     stream = StreamNameSerializer()
     ad = AreaDirectorSerializer(read_only=True, allow_null=True)
     group_list_email = serializers.EmailField(source="group.list_email", read_only=True)
@@ -288,23 +288,6 @@ def get_authors(self, doc: Document):
             many=True,
         ).data
 
-    @extend_schema_field(AreaSerializer(required=False))
-    def get_area(self, doc: Document):
-        """Get area for the RFC
-        
-        This logic might be better moved to Document or a combination of Document
-        and Group. The current (2026-02-24) Group.area() method is not strict enough:
-        it does not limit to WG groups or IETF-stream documents.
-        """
-        if doc.stream_id != "ietf":
-            return None
-        if doc.group is None:
-            return None
-        parent = doc.group.parent
-        if parent.type_id == "area":
-            return AreaSerializer(parent).data
-        return None
-
     @extend_schema_field(DocIdentifierSerializer(many=True))
     def get_identifiers(self, doc: Document):
         identifiers = []

From 47d3734955071d1ccc54787698e751c74ce4d303 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 4 Mar 2026 12:32:36 -0400
Subject: [PATCH 579/656] chore(deps): bump types-pytz from 2025.2.0.20250809
 to 2025.2.0.20251108 (#10424)

Bumps [types-pytz](https://github.com/typeshed-internal/stub_uploader) from 2025.2.0.20250809 to 2025.2.0.20251108.
- [Commits](https://github.com/typeshed-internal/stub_uploader/commits)

---
updated-dependencies:
- dependency-name: types-pytz
  dependency-version: 2025.2.0.20251108
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index cb583d5dc9..3d54b104ee 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -74,7 +74,7 @@ python-magic==0.4.18    # Versions beyond the yanked .19 and .20 introduce form
 pymemcache>=4.0.0  # for django.core.cache.backends.memcached.PyMemcacheCache
 python-mimeparse>=2.0.0  # from TastyPie
 pytz==2025.2  # Pinned as changes need to be vetted for their effect on Meeting fields
-types-pytz==2025.2.0.20250809  # match pytz version
+types-pytz==2025.2.0.20251108  # match pytz version
 requests>=2.32.4
 types-requests>=2.32.4
 requests-mock>=1.12.1

From 1799245dc6ce82301b0790412957ccfa19910dc1 Mon Sep 17 00:00:00 2001
From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com>
Date: Wed, 4 Mar 2026 16:45:46 +0000
Subject: [PATCH 580/656] ci: update base image target version to 20260304T1633

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 71370fabee..ce1828052e 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20260211T1901
+FROM ghcr.io/ietf-tools/datatracker-app-base:20260304T1633
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 947f3790e4..6be54fb6b0 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20260211T1901
+20260304T1633

From 7f28542c82e2c51210daf77ca10f9682c0ea709d Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 9 Mar 2026 14:02:57 -0300
Subject: [PATCH 581/656] fix: finish dropping email as RfcAuthor field
 (#10512)

* fix: fix admin / Document.author_list()

* fix: update RfcAuthorResource

email is still accessible, but read only

* fix: admin search by RfcAuthor email
---
 ietf/doc/admin.py     |  4 +++-
 ietf/doc/models.py    | 16 +++++++++++-----
 ietf/doc/resources.py |  2 +-
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py
index b604d4f096..0d04e8db3a 100644
--- a/ietf/doc/admin.py
+++ b/ietf/doc/admin.py
@@ -241,7 +241,9 @@ def is_deleted(self, instance):
 admin.site.register(StoredObject, StoredObjectAdmin)
 
 class RfcAuthorAdmin(admin.ModelAdmin):
+    # the email field in the list_display/readonly_fields works through a @property
     list_display = ['id', 'document', 'titlepage_name', 'person', 'email', 'affiliation', 'country', 'order']
-    search_fields = ['document__name', 'titlepage_name', 'person__name', 'email', 'affiliation', 'country']
+    search_fields = ['document__name', 'titlepage_name', 'person__name', 'person__email__address', 'affiliation', 'country']
     raw_id_fields = ["document", "person"]
+    readonly_fields = ["email"]
 admin.site.register(RfcAuthor, RfcAuthorAdmin)
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index f1b319367e..868bc4ac47 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -466,11 +466,12 @@ def author_persons(self):
 
     def author_list(self):
         """List of author emails"""
-        author_qs = (
-            self.rfcauthor_set
-            if self.type_id == "rfc" and self.rfcauthor_set.exists()
-            else self.documentauthor_set
-        ).select_related("email").order_by("order")
+        if self.type_id == "rfc" and self.rfcauthor_set.exists():
+            author_qs = self.rfcauthor_set.select_related("person").order_by("order")
+        else:
+            author_qs = self.documentauthor_set.select_related("email").order_by(
+                "order"
+            )
         best_addresses = []
         for author in author_qs:
             if author.email:
@@ -953,6 +954,11 @@ class Meta:
     @property
     def email(self) -> Email | None:
         return self.person.email() if self.person else None
+    
+    def format_for_titlepage(self):
+        if self.is_editor:
+            return f"{self.titlepage_name}, Ed."
+        return self.titlepage_name
 
 
 class DocumentAuthorInfo(models.Model):
diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py
index 556465a522..1d86df78d0 100644
--- a/ietf/doc/resources.py
+++ b/ietf/doc/resources.py
@@ -897,7 +897,7 @@ class Meta:
 class RfcAuthorResource(ModelResource):
     document         = ToOneField(DocumentResource, 'document')
     person           = ToOneField(PersonResource, 'person', null=True)
-    email            = ToOneField(EmailResource, 'email', null=True)
+    email            = ToOneField(EmailResource, 'email', null=True, readonly=True)
     class Meta:
         queryset = RfcAuthor.objects.all()
         serializer = api.Serializer()

From 809e7682db30279cb715f47c89ae546e320c9c76 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Wed, 11 Mar 2026 13:07:27 -0500
Subject: [PATCH 582/656] chore: remove task explorer from devcontainer
 (#10532)

---
 .devcontainer/devcontainer.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 2cfff78853..e4964e8909 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -32,7 +32,6 @@
                 "mutantdino.resourcemonitor",
                 "oderwat.indent-rainbow",
                 "redhat.vscode-yaml",
-                "spmeesseman.vscode-taskexplorer",
                 "ms-python.pylint",
                 "charliermarsh.ruff"
             ],

From d4a594ddd4a9dd0bd575465748627f7fea68aac3 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 11 Mar 2026 15:12:22 -0300
Subject: [PATCH 583/656] feat: rfc-index generation (#10526)

* chore: stand-in red_bucket STORAGE for dev

* feat: rfc index text generation (WIP)

Text generation works. Also includes XML generation that is not yet
converted. Based on Kesara's implementations in the purple repo.

* feat: rfc index XML generation (WIP)

* feat: Document.keywords + migration

* feat: keywords API

* feat: keywords in rfc-index.xml

* fix: better stream/area/wg_acronym

Still some disagreements, not sure if that's data or logic driven

* fix: NON WORKING GROUP logic

May need more attention

* fix: add rev to draft name

* fix: interleave unpublished RFC records

* fix: lint

* refactor: use lxml

* fix: multi-paragraph abstracts

* feat: RFCINDEX_MATCH_LEGACY_XML option

* fix: zero pad DOIs

* fix: better NON WORKING GROUP id

* fix: reorder elements

* refactor: extract repeated code

* refactor: unify DOI generation

* fix: modern DOI proxy URL for ATOM feed

* refactor: settings.RFC_EDITOR_ERRATA_BASE_URL

Drop unused settings.RFC_EDITOR_ERRATA_URL

* chore: real red_bucket storage cfg

* fix: handle missing json for prod/dev/test

* chore: straighten out S3 saving

* chore(dev): FileSystemStorage for red_bucket dev

(commented out)

* chore: configurable bucket path for JSON inputs

* test: tests_rfcindex.py

Not great coverage, but exercises the generators a bit.

* fix: lint + consistent var naming

* test: improve test coverage / testability

* fix: lint
---
 docker/configs/settings_local.py              |  11 +
 ietf/api/serializers_rpc.py                   |   2 +
 ietf/doc/api.py                               |   6 -
 ietf/doc/factories.py                         |   6 +
 ietf/doc/feeds.py                             |   9 +-
 ...3_dochistory_keywords_document_keywords.py |  31 ++
 ietf/doc/models.py                            |  21 +
 ietf/doc/serializers.py                       |   4 +-
 ietf/doc/views_doc.py                         |   4 +-
 ietf/settings.py                              |  10 +-
 ietf/settings_test.py                         |   8 +-
 ietf/sync/rfcindex.py                         | 480 ++++++++++++++++++
 ietf/sync/tests_rfcindex.py                   | 230 +++++++++
 ietf/templates/sync/rfc-index.txt             |  69 +++
 k8s/settings_local.py                         |  33 ++
 15 files changed, 907 insertions(+), 17 deletions(-)
 create mode 100644 ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py
 create mode 100644 ietf/sync/rfcindex.py
 create mode 100644 ietf/sync/tests_rfcindex.py
 create mode 100644 ietf/templates/sync/rfc-index.txt

diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py
index 1d4e6916b9..94adc516a4 100644
--- a/docker/configs/settings_local.py
+++ b/docker/configs/settings_local.py
@@ -101,6 +101,17 @@
         ),
     }
 
+# For dev on rfc-index generation, create a red_bucket/ directory in the project root
+# and uncomment these settings. Generated files will appear in this directory. To
+# generate an accurate index, put up-to-date copies of unusable-rfc-numbers.json,
+# april-first-rfc-numbers.json, and publication-std-levels.json in this directory
+# before generating the index.
+#
+# STORAGES["red_bucket"] = {
+#     "BACKEND": "django.core.files.storage.FileSystemStorage",
+#     "OPTIONS": {"location": "red_bucket"},
+# }
+
 APP_API_TOKENS = {
     "ietf.api.red_api" : ["devtoken", "redtoken"],  # Not a real secret
     "ietf.api.views_rpc" : ["devtoken"],  # Not a real secret
diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py
index e51b917be4..c17cbc64ce 100644
--- a/ietf/api/serializers_rpc.py
+++ b/ietf/api/serializers_rpc.py
@@ -300,6 +300,7 @@ class Meta:
             "obsoletes",
             "updates",
             "subseries",
+            "keywords",
         ]
 
     def validate(self, data):
@@ -540,6 +541,7 @@ class Meta:
             "pages",
             "std_level",
             "subseries",
+            "keywords",
         ]
 
     def create(self, validated_data):
diff --git a/ietf/doc/api.py b/ietf/doc/api.py
index 75993f463e..73fff6b27f 100644
--- a/ietf/doc/api.py
+++ b/ietf/doc/api.py
@@ -4,13 +4,11 @@
 from django.db.models import (
     BooleanField,
     Count,
-    JSONField,
     OuterRef,
     Prefetch,
     Q,
     QuerySet,
     Subquery,
-    Value,
 )
 from django.db.models.functions import TruncDate
 from django_filters import rest_framework as filters
@@ -160,10 +158,6 @@ def augment_rfc_queryset(queryset: QuerySet[Document]):
                 output_field=BooleanField(),
             )
         )
-        .annotate(
-            # TODO implement this fake field for real
-            keywords=Value(["keyword"], output_field=JSONField()),
-        )
     )
 
 
diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py
index bc38765446..1a178c6f31 100644
--- a/ietf/doc/factories.py
+++ b/ietf/doc/factories.py
@@ -311,6 +311,12 @@ class Meta:
     def desc(self):
          return 'New version available %s-%s'%(self.doc.name,self.rev)
 
+class PublishedRfcDocEventFactory(DocEventFactory):
+    class Meta:
+        model = DocEvent
+    type = "published_rfc"
+    doc = factory.SubFactory(WgRfcFactory)
+
 class StateDocEventFactory(DocEventFactory):
     class Meta:
         model = StateDocEvent
diff --git a/ietf/doc/feeds.py b/ietf/doc/feeds.py
index 500ed3cb18..afe96cf0df 100644
--- a/ietf/doc/feeds.py
+++ b/ietf/doc/feeds.py
@@ -1,5 +1,4 @@
-# Copyright The IETF Trust 2007-2020, All Rights Reserved
-# -*- coding: utf-8 -*-
+# Copyright The IETF Trust 2007-2026, All Rights Reserved
 
 import debug  # pyflakes:ignore
 
@@ -263,9 +262,11 @@ def item_extra_kwargs(self, item):
                 )
         extra.update({"media_contents": media_contents})
 
-        extra.update({"doi": "10.17487/%s" % item.name.upper()})
         extra.update(
-            {"doiuri": "http://dx.doi.org/10.17487/%s" % item.name.upper()}
+            {
+                "doi": item.doi,
+                "doiuri": f"https://doi.org/{item.doi}",
+            }
         )
 
         # R104 Publisher (Mandatory - but we need a string from them first)
diff --git a/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py b/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py
new file mode 100644
index 0000000000..5e2513e15a
--- /dev/null
+++ b/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py
@@ -0,0 +1,31 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+from django.db import migrations, models
+import ietf.doc.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("doc", "0032_remove_rfcauthor_email"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="dochistory",
+            name="keywords",
+            field=models.JSONField(
+                default=list,
+                max_length=1000,
+                validators=[ietf.doc.models.validate_doc_keywords],
+            ),
+        ),
+        migrations.AddField(
+            model_name="document",
+            name="keywords",
+            field=models.JSONField(
+                default=list,
+                max_length=1000,
+                validators=[ietf.doc.models.validate_doc_keywords],
+            ),
+        ),
+    ]
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index 868bc4ac47..7b23a62c45 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -13,6 +13,7 @@
 from io import BufferedReader
 from pathlib import Path
 
+from django.core.exceptions import ValidationError
 from django.db.models import Q
 from lxml import etree
 from typing import Optional, Protocol, TYPE_CHECKING, Union
@@ -109,6 +110,15 @@ class Meta:
 IESG_STATCHG_CONFLREV_ACTIVE_STATES = ("iesgeval", "defer")
 IESG_SUBSTATE_TAGS = ('ad-f-up', 'need-rev', 'extpty')
 
+
+def validate_doc_keywords(value):
+    if (
+        not isinstance(value, list | tuple | set) 
+        or not all(isinstance(elt, str) for elt in value)
+    ):
+        raise ValidationError("Value must be an array of strings")
+
+
 class DocumentInfo(models.Model):
     """Any kind of document.  Draft, RFC, Charter, IPR Statement, Liaison Statement"""
     time = models.DateTimeField(default=timezone.now) # should probably have auto_now=True
@@ -142,6 +152,17 @@ class DocumentInfo(models.Model):
     uploaded_filename = models.TextField(blank=True)
     note = models.TextField(blank=True)
     rfc_number = models.PositiveIntegerField(blank=True, null=True)  # only valid for type="rfc"
+    keywords = models.JSONField(
+        default=list,
+        max_length=1000,
+        validators=[validate_doc_keywords],
+    )
+
+    @property
+    def doi(self) -> str | None:
+        if self.type_id == "rfc" and self.rfc_number is not None:
+            return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number:04d}"
+        return None
 
     def file_extension(self):
         if not hasattr(self, '_cached_extension'):
diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py
index 139ae9aa7e..3651670962 100644
--- a/ietf/doc/serializers.py
+++ b/ietf/doc/serializers.py
@@ -291,9 +291,9 @@ def get_authors(self, doc: Document):
     @extend_schema_field(DocIdentifierSerializer(many=True))
     def get_identifiers(self, doc: Document):
         identifiers = []
-        if doc.rfc_number:
+        if doc.doi:
             identifiers.append(
-                DocIdentifier(type="doi", value=f"10.17487/RFC{doc.rfc_number:04d}")
+                DocIdentifier(type="doi", value=doc.doi)
             )
         return DocIdentifierSerializer(instance=identifiers, many=True).data
 
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 0ae7520681..c1f6352ac3 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -1285,9 +1285,7 @@ def document_bibtex(request, name, rev=None):
                     break
 
     elif doc.type_id == "rfc":
-        # This needs to be replaced with a lookup, as the mapping may change
-        # over time.
-        doi = f"10.17487/RFC{doc.rfc_number:04d}"
+        doi = doc.doi
 
     if doc.is_dochistory():
         latest_event = doc.latest_event(type='new_revision', rev=rev)
diff --git a/ietf/settings.py b/ietf/settings.py
index 71b110d762..e0b4f20118 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -838,6 +838,11 @@ def skip_unreadable_post(record):
     "slides",
 ]
 
+# Other storages
+STORAGES["red_bucket"] = {
+    "BACKEND": "django.core.files.storage.InMemoryStorage",
+    "OPTIONS": {"location": "red_bucket"},
+}
 
 # Override this in settings_local.py if needed
 # *_PATH variables ends with a slash/ .
@@ -932,10 +937,11 @@ def skip_unreadable_post(record):
 RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml"
 RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml"
 RFC_EDITOR_ERRATA_JSON_URL = "https://www.rfc-editor.org/errata.json"
-RFC_EDITOR_ERRATA_URL = "https://www.rfc-editor.org/errata_search.php?rfc={rfc_number}"
 RFC_EDITOR_INLINE_ERRATA_URL = "https://www.rfc-editor.org/rfc/inline-errata/rfc{rfc_number}.html"
+RFC_EDITOR_ERRATA_BASE_URL = "https://www.rfc-editor.org/errata/"
 RFC_EDITOR_INFO_BASE_URL = "https://www.rfc-editor.org/info/"
 
+
 # NomCom Tool settings
 ROLODEX_URL = ""
 NOMCOM_PUBLIC_KEYS_DIR = '/a/www/nomcom/public_keys/'
@@ -1570,3 +1576,5 @@ def skip_unreadable_post(record):
 
 
 YOUTUBE_DOMAINS = ['www.youtube.com', 'youtube.com', 'youtu.be', 'm.youtube.com', 'youtube-nocookie.com', 'www.youtube-nocookie.com']
+
+IETF_DOI_PREFIX = "10.17487"
diff --git a/ietf/settings_test.py b/ietf/settings_test.py
index 6479069db0..1f5a7e8ddc 100755
--- a/ietf/settings_test.py
+++ b/ietf/settings_test.py
@@ -14,7 +14,7 @@
 import shutil
 import tempfile
 from ietf.settings import *                                          # pyflakes:ignore
-from ietf.settings import ORIG_AUTH_PASSWORD_VALIDATORS
+from ietf.settings import ORIG_AUTH_PASSWORD_VALIDATORS, STORAGES
 import debug                            # pyflakes:ignore
 debug.debug = True
 
@@ -114,3 +114,9 @@ def tempdir_with_cleanup(**kwargs):
     AUTH_PASSWORD_VALIDATORS = ORIG_AUTH_PASSWORD_VALIDATORS
 except NameError:
     pass
+
+# Use InMemoryStorage for red bucket storage
+STORAGES["red_bucket"] = {
+    "BACKEND": "django.core.files.storage.InMemoryStorage",
+    "OPTIONS": {"location": "red_bucket"},
+}
diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py
new file mode 100644
index 0000000000..b15846094f
--- /dev/null
+++ b/ietf/sync/rfcindex.py
@@ -0,0 +1,480 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+import json
+from collections import defaultdict
+from collections.abc import Container
+from dataclasses import dataclass
+from io import StringIO, BytesIO
+from itertools import chain
+from operator import attrgetter, itemgetter
+from pathlib import Path
+from textwrap import fill
+from urllib.parse import urljoin
+
+from django.conf import settings
+from lxml import etree
+
+from django.core.files.storage import storages
+from django.db import models
+from django.db.models.functions import Substr, Cast
+from django.template.loader import render_to_string
+from django.utils import timezone
+
+from ietf.doc.models import Document
+from ietf.name.models import StdLevelName
+from ietf.utils.log import log
+
+FORMATS_FOR_INDEX = ["txt", "html", "pdf", "xml", "ps"]
+
+
+def format_rfc_number(n):
+    """Format an RFC number (or subseries doc number)
+    
+    Set settings.RFCINDEX_MATCH_LEGACY_XML=True for the legacy (leading-zero) format.
+    That is for debugging only - tests will fail.
+    """
+    if getattr(settings, "RFCINDEX_MATCH_LEGACY_XML", False):
+        return format(n, "04")
+    else:
+        return format(n)
+
+
+def errata_url(rfc: Document):
+    return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}")
+
+
+def save_to_red_bucket(filename: str, content: BytesIO | StringIO):
+    red_bucket = storages["red_bucket"]
+    bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename)
+    if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True):
+        # Django 4.2's FileSystemStorage does not support allow_overwrite.
+        red_bucket.delete(bucket_path)
+    red_bucket.save(bucket_path, content)
+    log(f"Saved {bucket_path} in red_bucket storage")
+
+
+@dataclass
+class UnusableRfcNumber:
+    rfc_number: int
+    comment: str
+
+
+def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]:
+    FILENAME = "unusable-rfc-numbers.json"
+    bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME)
+    try:
+        with storages["red_bucket"].open(bucket_path) as urn_file:
+            records = json.load(urn_file)
+    except FileNotFoundError:
+        if settings.SERVER_MODE == "development":
+            log(
+                f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev "
+                "but generated rfc-index will not agree with RFC Editor values."
+            )  # pragma: no cover
+            return []  # pragma: no cover
+        log(f"Error: unable to open {bucket_path} in red_bucket storage")
+        raise
+    except json.JSONDecodeError:
+        log(f"Error: unable to parse {bucket_path} in red_bucket storage")
+        if settings.SERVER_MODE == "development":
+            return []  # pragma: no cover  
+        raise
+    assert all(isinstance(record["number"], int) for record in records)
+    assert all(isinstance(record["comment"], str) for record in records)
+    return [
+        UnusableRfcNumber(rfc_number=record["number"], comment=record["comment"])
+        for record in sorted(records, key=itemgetter("number"))
+    ]
+
+
+def get_april1_rfc_numbers() -> Container[int]:
+    FILENAME = "april-first-rfc-numbers.json"
+    bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME)
+    try:
+        with storages["red_bucket"].open(bucket_path) as urn_file:
+            records = json.load(urn_file)
+    except FileNotFoundError:
+        if settings.SERVER_MODE == "development":
+            log(
+                f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev "
+                "but generated rfc-index will not agree with RFC Editor values."
+            )  # pragma: no cover
+            return []  # pragma: no cover
+        log(f"Error: unable to open {bucket_path} in red_bucket storage")
+        raise
+    except json.JSONDecodeError:
+        log(f"Error: unable to parse {bucket_path} in red_bucket storage")
+        if settings.SERVER_MODE == "development":
+            return []  # pragma: no cover
+        raise
+    assert all(isinstance(record, int) for record in records)
+    return records
+
+
+def get_publication_std_levels() -> dict[int, StdLevelName]:
+    FILENAME = "publication-std-levels.json"
+    bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME)
+    values: dict[int, StdLevelName] = {}
+    try:
+        with storages["red_bucket"].open(bucket_path) as urn_file:
+            records = json.load(urn_file)
+    except FileNotFoundError:
+        if settings.SERVER_MODE == "development":
+            log(
+                f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev "
+                "but generated rfc-index will not agree with RFC Editor values."
+            )  # pragma: no cover
+            # intentionally fall through instead of return here
+        else:
+            log(f"Error: unable to open {bucket_path} in red_bucket storage")
+            raise
+    except json.JSONDecodeError:
+        log(f"Error: unable to parse {bucket_path} in red_bucket storage")
+        if settings.SERVER_MODE != "development":
+            raise
+    else:
+        assert all(isinstance(record["number"], int) for record in records)
+        values = {
+            record["number"]: StdLevelName.objects.get(
+                slug=record["publication_std_level"]
+            )
+            for record in records
+        }
+    # defaultdict to return "unknown" for any missing values
+    unknown_std_level = StdLevelName.objects.get(slug="unkn")
+    return defaultdict(lambda: unknown_std_level, values)
+
+
+def format_ordering(rfc_number):
+    if rfc_number < 8650:
+        ordering = ["txt", "ps", "pdf", "html", "xml"]
+    else:
+        ordering = ["html", "txt", "ps", "pdf", "xml"]
+    return ordering.index  # return the method
+
+
+def get_rfc_text_index_entries():
+    """Returns RFC entries for rfc-index.txt"""
+    entries = []
+    april1_rfc_numbers = get_april1_rfc_numbers()
+    published_rfcs = Document.objects.filter(type_id="rfc").order_by("rfc_number")
+    rfcs = sorted(
+        chain(published_rfcs, get_unusable_rfc_numbers()), key=attrgetter("rfc_number")
+    )
+    for rfc in rfcs:
+        if isinstance(rfc, UnusableRfcNumber):
+            entries.append(f"{format_rfc_number(rfc.rfc_number)} Not Issued.")
+        else:
+            assert isinstance(rfc, Document)
+            authors = ", ".join(
+                author.format_for_titlepage() for author in rfc.rfcauthor_set.all()
+            )
+            published_at = rfc.pub_date()
+            date = (
+                published_at.strftime("1 %B %Y")
+                if rfc.rfc_number in april1_rfc_numbers
+                else published_at.strftime("%B %Y")
+            )
+
+            # formats
+            formats = ", ".join(
+                sorted(
+                    [
+                        format["fmt"]
+                        for format in rfc.formats()
+                        if format["fmt"] in FORMATS_FOR_INDEX
+                    ],
+                    key=format_ordering(rfc.rfc_number),
+                )
+            ).upper()
+
+            # obsoletes
+            obsoletes = ""
+            obsoletes_documents = sorted(
+                rfc.related_that_doc("obs"),
+                key=attrgetter("rfc_number"),
+            )
+            if len(obsoletes_documents) > 0:
+                obsoletes_names = ", ".join(
+                    f"RFC{format_rfc_number(doc.rfc_number)}"
+                    for doc in obsoletes_documents
+                )
+                obsoletes = f" (Obsoletes {obsoletes_names})"
+
+            # obsoleted by
+            obsoleted_by = ""
+            obsoleted_by_documents = sorted(
+                rfc.related_that("obs"),
+                key=attrgetter("rfc_number"),
+            )
+            if len(obsoleted_by_documents) > 0:
+                obsoleted_by_names = ", ".join(
+                    f"RFC{format_rfc_number(doc.rfc_number)}"
+                    for doc in obsoleted_by_documents
+                )
+                obsoleted_by = f" (Obsoleted by {obsoleted_by_names})"
+
+            # updates
+            updates = ""
+            updates_documents = sorted(
+                rfc.related_that_doc("updates"),
+                key=attrgetter("rfc_number"),
+            )
+            if len(updates_documents) > 0:
+                updates_names = ", ".join(
+                    f"RFC{format_rfc_number(doc.rfc_number)}"
+                    for doc in updates_documents
+                )
+                updates = f" (Updates {updates_names})"
+
+            # updated by
+            updated_by = ""
+            updated_by_documents = sorted(
+                rfc.related_that("updates"),
+                key=attrgetter("rfc_number"),
+            )
+            if len(updated_by_documents) > 0:
+                updated_by_names = ", ".join(
+                    f"RFC{format_rfc_number(doc.rfc_number)}"
+                    for doc in updated_by_documents
+                )
+                updated_by = f" (Updated by {updated_by_names})"
+
+            doc_relations = f"{obsoletes}{obsoleted_by}{updates}{updated_by} "
+
+            # subseries
+            subseries = ",".join(
+                f"{container.type.slug}{format_rfc_number(int(container.name[3:]))}"
+                for container in rfc.part_of()
+            ).upper()
+            if subseries:
+                subseries = f"(Also {subseries}) "
+
+            entry = fill(
+                (
+                    f"{format_rfc_number(rfc.rfc_number)} {rfc.title}. {authors}. {date}. "
+                    f"(Format: {formats}){doc_relations}{subseries}"
+                    f"(Status: {str(rfc.std_level).upper()}) "
+                    f"(DOI: {rfc.doi})"
+                ),
+                width=73,
+                subsequent_indent=" " * 5,
+            )
+            entries.append(entry)
+
+    return entries
+
+
+def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False):
+    """Add subseries entries for rfc-index.xml"""
+    # subseries docs annotated with numeric number
+    ss_docs = list(
+        Document.objects.filter(type_id=ss_type)
+        .annotate(
+            number=Cast(
+                Substr("name", 4, None),
+                output_field=models.IntegerField(),
+            )
+        )
+        .order_by("-number")
+    )
+    if len(ss_docs) == 0:
+        return  # very much not expected
+    highest_number = ss_docs[0].number
+    for ss_number in range(1, highest_number + 1):
+        if ss_docs[-1].number == ss_number:
+            this_ss_doc = ss_docs.pop()
+            contained_rfcs = this_ss_doc.contains()
+        else:
+            contained_rfcs = []
+        if len(contained_rfcs) == 0 and not include_all:
+            continue
+        entry = etree.SubElement(rfc_index, f"{ss_type}-entry")
+        etree.SubElement(
+            entry, "doc-id"
+        ).text = f"{ss_type.upper()}{format_rfc_number(ss_number)}"
+        if len(contained_rfcs) > 0:
+            is_also = etree.SubElement(entry, "is-also")
+            for rfc in sorted(contained_rfcs, key=attrgetter("rfc_number")):
+                etree.SubElement(
+                    is_also, "doc-id"
+                ).text = f"RFC{format_rfc_number(rfc.rfc_number)}"
+
+
+def add_related_xml_index_entries(root: etree.Element, rfc: Document, tag: str):
+    relation_getter = {
+        "obsoletes": lambda doc: doc.related_that_doc("obs"),
+        "obsoleted-by": lambda doc: doc.related_that("obs"),
+        "updates": lambda doc: doc.related_that_doc("updates"),
+        "updated-by": lambda doc: doc.related_that("updates"),
+    }
+    related_docs = sorted(
+        relation_getter[tag](rfc),
+        key=attrgetter("rfc_number"),
+    )
+    if len(related_docs) > 0:
+        element = etree.SubElement(root, tag)
+        for doc in related_docs:
+            etree.SubElement(
+                element, "doc-id"
+            ).text = f"RFC{format_rfc_number(doc.rfc_number)}"
+
+
+def add_rfc_xml_index_entries(rfc_index):
+    """Add RFC entries for rfc-index.xml"""
+    entries = []
+    april1_rfc_numbers = get_april1_rfc_numbers()
+    publication_statuses = get_publication_std_levels()
+
+    published_rfcs = Document.objects.filter(type_id="rfc").order_by("rfc_number")
+
+    # Iterators for unpublished and published, both sorted by number
+    unpublished_iter = iter(get_unusable_rfc_numbers())
+    published_iter = iter(published_rfcs)
+
+    # Prime the next_* values
+    next_unpublished = next(unpublished_iter, None)
+    next_published = next(published_iter, None)
+
+    while next_published is not None or next_unpublished is not None:
+        if next_unpublished is not None and (
+            next_published is None
+            or next_unpublished.rfc_number < next_published.rfc_number
+        ):
+            entry = etree.SubElement(rfc_index, "rfc-not-issued-entry")
+            etree.SubElement(
+                entry, "doc-id"
+            ).text = f"RFC{format_rfc_number(next_unpublished.rfc_number)}"
+            entries.append(entry)
+            next_unpublished = next(unpublished_iter, None)
+            continue
+
+        rfc = next_published  # hang on to this
+        next_published = next(published_iter, None)  # prep for next iteration
+        entry = etree.SubElement(rfc_index, "rfc-entry")
+
+        etree.SubElement(
+            entry, "doc-id"
+        ).text = f"RFC{format_rfc_number(rfc.rfc_number)}"
+        etree.SubElement(entry, "title").text = rfc.title
+
+        for author in rfc.rfcauthor_set.all():
+            author_element = etree.SubElement(entry, "author")
+            etree.SubElement(author_element, "name").text = author.titlepage_name
+            if author.is_editor:
+                etree.SubElement(author_element, "title").text = "Editor"
+
+        date = etree.SubElement(entry, "date")
+        published_at = rfc.pub_date()
+        etree.SubElement(date, "month").text = published_at.strftime("%B")
+        if rfc.rfc_number in april1_rfc_numbers:
+            etree.SubElement(date, "day").text = str(published_at.day)
+        etree.SubElement(date, "year").text = str(published_at.year)
+
+        format_ = etree.SubElement(entry, "format")
+        fmts = [ff["fmt"] for ff in rfc.formats() if ff["fmt"] in FORMATS_FOR_INDEX]
+        for fmt in sorted(fmts, key=format_ordering(rfc.rfc_number)):
+            match_legacy = getattr(settings, "RFCINDEX_MATCH_LEGACY_XML", False)
+            etree.SubElement(format_, "file-format").text = (
+                "ASCII" if match_legacy and fmt == "txt" else fmt.upper()
+            )
+
+        etree.SubElement(entry, "page-count").text = str(rfc.pages)
+
+        if len(rfc.keywords) > 0:
+            keywords = etree.SubElement(entry, "keywords")
+            for keyword in rfc.keywords:
+                etree.SubElement(keywords, "kw").text = keyword.strip()
+
+        if rfc.abstract:
+            abstract = etree.SubElement(entry, "abstract")
+            for paragraph in rfc.abstract.split("\n\n"):
+                etree.SubElement(abstract, "p").text = paragraph.strip()
+
+        draft = rfc.came_from_draft()
+        if draft is not None:
+            etree.SubElement(entry, "draft").text = f"{draft.name}-{draft.rev}"
+
+        part_of_documents = rfc.part_of()
+        if len(part_of_documents) > 0:
+            is_also = etree.SubElement(entry, "is-also")
+            for doc in part_of_documents:
+                etree.SubElement(is_also, "doc-id").text = doc.name.upper()
+
+        add_related_xml_index_entries(entry, rfc, "obsoletes")
+        add_related_xml_index_entries(entry, rfc, "obsoleted-by")
+        add_related_xml_index_entries(entry, rfc, "updates")
+        add_related_xml_index_entries(entry, rfc, "updated-by")
+
+        etree.SubElement(entry, "current-status").text = rfc.std_level.name.upper()
+        etree.SubElement(entry, "publication-status").text = publication_statuses[
+            rfc.rfc_number
+        ].name.upper()
+        etree.SubElement(entry, "stream").text = (
+            "INDEPENDENT" if rfc.stream_id == "ise" else rfc.stream.name
+        )
+
+        # Add area / wg_acronym
+        if rfc.stream_id == "ietf":
+            if rfc.group.type_id in ["individ", "area"]:
+                etree.SubElement(entry, "wg_acronym").text = "NON WORKING GROUP"
+            else:
+                if rfc.area is not None:
+                    etree.SubElement(entry, "area").text = rfc.area.acronym
+                if rfc.group:
+                    etree.SubElement(entry, "wg_acronym").text = rfc.group.acronym
+
+        if rfc.tags.filter(slug="errata").exists():
+            etree.SubElement(entry, "errata-url").text = errata_url(rfc)
+        etree.SubElement(entry, "doi").text = rfc.doi
+        entries.append(entry)
+
+
+def create_rfc_txt_index():
+    """Create text index of published documents"""
+    DATE_FMT = "%m/%d/%Y"
+    created_on = timezone.now().strftime(DATE_FMT)
+    log("Creating rfc-index.txt")
+    index = render_to_string(
+        "sync/rfc-index.txt",
+        {
+            "created_on": created_on,
+            "rfcs": get_rfc_text_index_entries(),
+        },
+    )
+    save_to_red_bucket("rfc-index.txt", StringIO(index))
+
+
+def create_rfc_xml_index():
+    """Create XML index of published documents"""
+    XSI_NAMESPACE = "http://www.w3.org/2001/XMLSchema-instance"
+    XSI = "{" + XSI_NAMESPACE + "}"
+
+    log("Creating rfc-index.xml")
+    rfc_index = etree.Element(
+        "rfc-index",
+        nsmap={
+            None: "https://www.rfc-editor.org/rfc-index",
+            "xsi": XSI_NAMESPACE,
+        },
+        attrib={
+            XSI + "schemaLocation": (
+                "https://www.rfc-editor.org/rfc-index "
+                "https://www.rfc-editor.org/rfc-index.xsd"
+            ),
+        },
+    )
+
+    # add data
+    add_subseries_xml_index_entries(rfc_index, "bcp", include_all=True)
+    add_subseries_xml_index_entries(rfc_index, "fyi")
+    add_rfc_xml_index_entries(rfc_index)
+    add_subseries_xml_index_entries(rfc_index, "std")
+
+    # make it pretty
+    pretty_index = etree.tostring(
+        rfc_index,
+        encoding="utf-8",
+        xml_declaration=True,
+        pretty_print=4,
+    )
+    save_to_red_bucket("rfc-index.xml", BytesIO(pretty_index))
diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py
new file mode 100644
index 0000000000..b0a8712fe1
--- /dev/null
+++ b/ietf/sync/tests_rfcindex.py
@@ -0,0 +1,230 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+import json
+from io import BytesIO, StringIO
+from unittest import mock
+
+from django.core.files.storage import storages
+from django.test.utils import override_settings
+from lxml import etree
+
+from ietf.doc.factories import PublishedRfcDocEventFactory, IndividualRfcFactory
+from ietf.name.models import DocTagName
+from ietf.sync.rfcindex import (
+    create_rfc_txt_index,
+    create_rfc_xml_index,
+    format_rfc_number,
+    save_to_red_bucket, get_unusable_rfc_numbers, get_april1_rfc_numbers,
+    get_publication_std_levels,
+)
+from ietf.utils.test_utils import TestCase
+
+
+class RfcIndexTests(TestCase):
+    """Tests of rfc-index generation
+
+    Tests are limited and should cover more cases. Needs:
+      * test of subseries docs
+      * test of related docs (obsoletes/updates + reverse directions)
+      * more thorough validation of index contents
+
+    Be careful when calling create_rfc_txt_index() or create_rfc_xml_index(). These
+    will save to a storage by default, which can introduce cross-talk between tests.
+    Best to patch that method with a mock.
+    """
+
+    def setUp(self):
+        super().setUp()
+        red_bucket = storages["red_bucket"]
+
+        # Create an unused RFC number
+        red_bucket.save(
+            "input/unusable-rfc-numbers.json",
+            StringIO(json.dumps([{"number": 123, "comment": ""}])),
+        )
+
+        # actual April 1 RFC
+        self.april_fools_rfc = PublishedRfcDocEventFactory(
+            time="2020-04-01T12:00:00Z",
+            doc=IndividualRfcFactory(
+                name="rfc4560",
+                rfc_number=4560,
+                stream_id="ise",
+                std_level_id="inf",
+            ),
+        ).doc
+        # Set up a JSON file to flag the April 1 RFC
+        red_bucket.save(
+            "input/april-first-rfc-numbers.json",
+            StringIO(json.dumps([self.april_fools_rfc.rfc_number])),
+        )
+
+        # non-April Fools RFC that happens to have been published on April 1
+        self.rfc = PublishedRfcDocEventFactory(
+            time="2021-04-01T12:00:00Z",
+            doc__name="rfc10000",
+            doc__rfc_number=10000,
+            doc__std_level_id="std",
+        ).doc
+        self.rfc.tags.add(DocTagName.objects.get(slug="errata"))
+
+        # Set up a publication-std-levels.json file to indicate the publication
+        # standard of self.rfc as different from its current value
+        red_bucket.save(
+            "input/publication-std-levels.json",
+            StringIO(
+                json.dumps(
+                    [{"number": self.rfc.rfc_number, "publication_std_level": "ps"}]
+                )
+            ),
+        )
+
+    def tearDown(self):
+        red_bucket = storages["red_bucket"]
+        red_bucket.delete("input/unusable-rfc-numbers.json")
+        red_bucket.delete("input/april-first-rfc-numbers.json")
+        red_bucket.delete("input/publication-std-levels.json")
+        super().tearDown()
+
+    @override_settings(RFCINDEX_INPUT_PATH="input/")
+    @mock.patch("ietf.sync.rfcindex.save_to_red_bucket")
+    def test_create_rfc_txt_index(self, mock_save):
+        create_rfc_txt_index()
+        self.assertEqual(mock_save.call_count, 1)
+        self.assertEqual(mock_save.call_args[0][0], "rfc-index.txt")
+        contents = mock_save.call_args[0][1].read()
+        self.assertIn(
+            "123 Not Issued.",
+            contents,
+        )
+        # No zero prefix!
+        self.assertNotIn(
+            "0123 Not Issued.",
+            contents,
+        )
+        self.assertIn(
+            f"{self.april_fools_rfc.rfc_number} {self.april_fools_rfc.title}",
+            contents,
+        )
+        self.assertIn("1 April 2020", contents)  # from the April 1 RFC
+        self.assertIn(
+            f"{self.rfc.rfc_number} {self.rfc.title}",
+            contents,
+        )
+        self.assertIn("April 2021", contents)  # from the non-April 1 RFC
+        self.assertNotIn("1 April 2021", contents)
+
+    @override_settings(RFCINDEX_INPUT_PATH="input/")
+    @mock.patch("ietf.sync.rfcindex.save_to_red_bucket")
+    def test_create_rfc_xml_index(self, mock_save):
+        create_rfc_xml_index()
+        self.assertEqual(mock_save.call_count, 1)
+        self.assertEqual(mock_save.call_args[0][0], "rfc-index.xml")
+        contents = mock_save.call_args[0][1].read()
+        ns = "{https://www.rfc-editor.org/rfc-index}"  # NOT an f-string
+        index = etree.fromstring(contents)
+
+        # We can aspire to validating the schema - currently does not conform because
+        # XSD expects 4-digit RFC numbers (etc).
+        #
+        # xmlschema = etree.XMLSchema(etree.fromstring(
+        #     Path(__file__).with_name("rfc-index.xsd").read_bytes())
+        # )
+        # xmlschema.assertValid(index)
+
+        children = list(index)  # elements as list
+        # Should be one rfc-not-issued-entry
+        self.assertEqual(len(children), 3)
+        self.assertEqual(
+            [
+                c.find(f"{ns}doc-id").text
+                for c in children
+                if c.tag == f"{ns}rfc-not-issued-entry"
+            ],
+            ["RFC123"],
+        )
+        # Should be two rfc-entries
+        rfc_entries = {
+            c.find(f"{ns}doc-id").text: c for c in children if c.tag == f"{ns}rfc-entry"
+        }
+
+        # Check the April Fool's entry
+        april_fools_entry = rfc_entries[self.april_fools_rfc.name.upper()]
+        self.assertEqual(
+            april_fools_entry.find(f"{ns}title").text,
+            self.april_fools_rfc.title,
+        )
+        self.assertEqual(
+            [(c.tag, c.text) for c in april_fools_entry.find(f"{ns}date")],
+            [(f"{ns}month", "April"), (f"{ns}day", "1"), (f"{ns}year", "2020")],
+        )
+        self.assertEqual(
+            april_fools_entry.find(f"{ns}current-status").text,
+            "INFORMATIONAL",
+        )
+        self.assertEqual(
+            april_fools_entry.find(f"{ns}publication-status").text,
+            "UNKNOWN",
+        )
+
+        # Check the Regular entry
+        rfc_entry = rfc_entries[self.rfc.name.upper()]
+        self.assertEqual(rfc_entry.find(f"{ns}title").text, self.rfc.title)
+        self.assertEqual(
+            rfc_entry.find(f"{ns}current-status").text, "INTERNET STANDARD"
+        )
+        self.assertEqual(
+            rfc_entry.find(f"{ns}publication-status").text, "PROPOSED STANDARD"
+        )
+        self.assertEqual(
+            [(c.tag, c.text) for c in rfc_entry.find(f"{ns}date")],
+            [(f"{ns}month", "April"), (f"{ns}year", "2021")],
+        )
+
+
+class HelperTests(TestCase):
+    def test_format_rfc_number(self):
+        self.assertEqual(format_rfc_number(10), "10")
+        with override_settings(RFCINDEX_MATCH_LEGACY_XML=True):
+            self.assertEqual(format_rfc_number(10), "0010")
+
+    def test_save_to_red_bucket(self):
+        red_bucket = storages["red_bucket"]
+        with override_settings(RFCINDEX_DELETE_THEN_WRITE=False):
+            save_to_red_bucket("test", StringIO("contents"))
+        with red_bucket.open("test", "r") as f:
+            self.assertEqual(f.read(), "contents")
+        with override_settings(RFCINDEX_DELETE_THEN_WRITE=True):
+            save_to_red_bucket("test", BytesIO(b"new contents"))
+        with red_bucket.open("test", "r") as f:
+            self.assertEqual(f.read(), "new contents")
+        red_bucket.delete("test")  # clean up like a good child
+
+    def test_get_unusable_rfc_numbers_raises(self):
+        """get_unusable_rfc_numbers should bail on errors"""
+        with self.assertRaises(FileNotFoundError):
+            get_unusable_rfc_numbers()
+        red_bucket = storages["red_bucket"]
+        red_bucket.save("unusable-rfc-numbers.json", StringIO("not json"))
+        with self.assertRaises(json.JSONDecodeError):
+            get_unusable_rfc_numbers()
+        red_bucket.delete("unusable-rfc-numbers.json")
+
+    def test_get_april1_rfc_numbers_raises(self):
+        """get_april1_rfc_numbers should bail on errors"""
+        with self.assertRaises(FileNotFoundError):
+            get_april1_rfc_numbers()
+        red_bucket = storages["red_bucket"]
+        red_bucket.save("april-first-rfc-numbers.json", StringIO("not json"))
+        with self.assertRaises(json.JSONDecodeError):
+            get_april1_rfc_numbers()
+        red_bucket.delete("april-first-rfc-numbers.json")
+
+    def test_get_publication_std_levels_raises(self):
+        """get_publication_std_levels should bail on errors"""
+        with self.assertRaises(FileNotFoundError):
+            get_publication_std_levels()
+        red_bucket = storages["red_bucket"]
+        red_bucket.save("publication-std-levels.json", StringIO("not json"))
+        with self.assertRaises(json.JSONDecodeError):
+            get_publication_std_levels()
+        red_bucket.delete("publication-std-levels.json")
diff --git a/ietf/templates/sync/rfc-index.txt b/ietf/templates/sync/rfc-index.txt
new file mode 100644
index 0000000000..0f01ddfa90
--- /dev/null
+++ b/ietf/templates/sync/rfc-index.txt
@@ -0,0 +1,69 @@
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+                             RFC INDEX
+                           -------------
+
+(CREATED ON: {{created_on}}.)
+
+This file contains citations for all RFCs in numeric order.
+
+RFC citations appear in this format:
+
+  ####  Title of RFC.  Author 1, Author 2, Author 3.  Issue date.
+        (Format: ASCII) (Obsoletes xxx) (Obsoleted by xxx) (Updates xxx)
+        (Updated by xxx) (Also FYI ####) (Status: ssssss) (DOI: ddd)
+
+or
+
+  ####  Not Issued.
+
+For example:
+
+  1129 Internet Time Synchronization: The Network Time Protocol. D.L.
+       Mills. October 1989. (Format: TXT, PS, PDF, HTML) (Also RFC1119)
+       (Status: INFORMATIONAL) (DOI: 10.17487/RFC1129)
+
+Key to citations:
+
+#### is the RFC number.
+
+Following the RFC number are the title, the author(s), and the
+publication date of the RFC.  Each of these is terminated by a period.
+
+Following the number are the title (terminated with a period), the
+author, or list of authors (terminated with a period), and the date
+(terminated with a period).
+
+The format follows in parentheses. One or more of the following formats
+are listed:  text (TXT), PostScript (PS), Portable Document Format
+(PDF), HTML, XML.
+
+Obsoletes xxxx refers to other RFCs that this one replaces;
+Obsoleted by xxxx refers to RFCs that have replaced this one.
+Updates xxxx refers to other RFCs that this one merely updates (but
+does not replace); Updated by xxxx refers to RFCs that have updated
+(but not replaced) this one.  Generally, only immediately succeeding
+and/or preceding RFCs are indicated, not the entire history of each
+related earlier or later RFC in a related series.
+
+The (Also FYI ##) or (Also STD ##) or (Also BCP ##) phrase gives the
+equivalent FYI, STD, or BCP number if the RFC is also in those
+document sub-series.  The Status field gives the document's
+current status (see RFC 2026).  The (DOI ddd) field gives the
+Digital Object Identifier.
+
+RFCs may be obtained in a number of ways, using HTTP, FTP, or email.
+See the RFC Editor Web page http://www.rfc-editor.org
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+                                RFC INDEX
+                                ---------
+
+
+
+{% for rfc in rfcs %}{{rfc|safe}}
+
+{% endfor %}
diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 0386dbbdf9..5ca4ba5cd9 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -417,6 +417,39 @@ def _multiline_to_list(s):
         ),
     }
 
+# Configure storage for the red bucket - assume it uses the same credentials as
+# other blobs
+_red_bucket_name = os.environ.get("DATATRACKER_BLOB_STORE_RED_BUCKET_NAME", "").strip()
+if _red_bucket_name == "":
+    raise RuntimeError("DATATRACKER_BLOB_STORE_RED_BUCKET_NAME must be set")
+
+STORAGES["red_bucket"] = {
+    "BACKEND": "storages.backends.s3.S3Storage",
+    "OPTIONS": dict(
+        endpoint_url=_blob_store_endpoint_url,
+        access_key=_blob_store_access_key,
+        secret_key=_blob_store_secret_key,
+        security_token=None,
+        client_config=botocore.config.Config(
+            request_checksum_calculation="when_required",
+            response_checksum_validation="when_required",
+            signature_version="s3v4",
+            connect_timeout=_blob_store_connect_timeout,
+            read_timeout=_blob_store_read_timeout,
+            retries={"total_max_attempts": _blob_store_max_attempts},
+        ),
+        verify=False,
+        bucket_name=_red_bucket_name,
+    ),
+}
+RFCINDEX_DELETE_THEN_WRITE = False  # S3Storage allows file_overwrite by default
+RFCINDEX_OUTPUT_PATH = os.environ.get(
+    "DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/"
+)
+RFCINDEX_INPUT_PATH = os.environ.get(
+    "DATATRACKR_RFCINDEX_INPUT_PATH", ""
+)
+
 # Configure the blobdb app for artifact storage
 _blobdb_replication_enabled = (
     os.environ.get("DATATRACKER_BLOBDB_REPLICATION_ENABLED", "true").lower() == "true"

From c226749c301fbecfd5503ebd66ba3692187d2946 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 12 Mar 2026 16:31:56 -0300
Subject: [PATCH 584/656] feat: task + API for rfc-index creation (#10537)

* chore: fix typo in k8s/settings_local.py

* feat: refresh_rfc_index() API

* fix: use ContentFile, manually encode str

Works better with S3Storage

* chore(dev): expose blobstore on fixed ports

Simplifies connecting purple to the blob store

* chore(dev): typo

* test: fix + test encoding more carefully

* test: cover the new url
---
 .devcontainer/docker-compose.extend.yml |  4 +--
 docker/docker-compose.extend.yml        |  4 +--
 ietf/api/tests_views_rpc.py             | 15 ++++++++++
 ietf/api/urls_rpc.py                    |  5 ++++
 ietf/api/views_rpc.py                   | 16 +++++++++++
 ietf/sync/rfcindex.py                   | 17 ++++++-----
 ietf/sync/tasks.py                      | 10 ++++++-
 ietf/sync/tests_rfcindex.py             | 38 ++++++++++++++-----------
 k8s/settings_local.py                   |  2 +-
 9 files changed, 82 insertions(+), 29 deletions(-)

diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml
index a92f42bc6d..ce1ce259fd 100644
--- a/.devcontainer/docker-compose.extend.yml
+++ b/.devcontainer/docker-compose.extend.yml
@@ -14,8 +14,8 @@ services:
         network_mode: service:db
     blobstore:
         ports:
-            - '9000'
-            - '9001'
+            - '9000:9000'
+            - '9001:9001'
 
 volumes:
     datatracker-vscode-ext:
diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml
index a69a453110..12ebe447d5 100644
--- a/docker/docker-compose.extend.yml
+++ b/docker/docker-compose.extend.yml
@@ -18,8 +18,8 @@ services:
             - '5433'
     blobstore:
         ports:
-            - '9000'
-            - '9001'
+            - '9000:9000'
+            - '9001:9001'
     celery:
         volumes:
             - .:/workspace
diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py
index 6a5a5c9b88..7ab8778d28 100644
--- a/ietf/api/tests_views_rpc.py
+++ b/ietf/api/tests_views_rpc.py
@@ -363,3 +363,18 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 200)  # conflict
+
+    @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
+    @mock.patch("ietf.api.views_rpc.create_rfc_index_task")
+    def test_refresh_rfc_index(self, mock_task):
+        url = urlreverse("ietf.api.purple_api.refresh_rfc_index")
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 403)
+        response = self.client.get(url, headers={"X-Api-Key": "invalid-token"})
+        self.assertEqual(response.status_code, 403)
+        response = self.client.get(url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(response.status_code, 405)
+        self.assertFalse(mock_task.delay.called)
+        response = self.client.post(url, headers={"X-Api-Key": "valid-token"})
+        self.assertEqual(response.status_code, 202)
+        self.assertTrue(mock_task.delay.called)
diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py
index 9d41ac137f..8555610dc3 100644
--- a/ietf/api/urls_rpc.py
+++ b/ietf/api/urls_rpc.py
@@ -30,6 +30,11 @@
         views_rpc.RfcPubFilesView.as_view(),
         name="ietf.api.purple_api.upload_rfc_files",
     ),
+    path(
+        r"rfc_index/refresh/",
+        views_rpc.RfcIndexView.as_view(),
+        name="ietf.api.purple_api.refresh_rfc_index",
+    ),
     path(r"subject/<str:subject_id>/person/", views_rpc.SubjectPersonView.as_view()),
 ]
 
diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index 8862bbf866..c7ae699005 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -40,6 +40,7 @@
 from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage
 from ietf.doc.tasks import signal_update_rfc_metadata_task
 from ietf.person.models import Email, Person
+from ietf.sync.tasks import create_rfc_index_task
 
 
 class Conflict(APIException):
@@ -516,3 +517,18 @@ def post(self, request):
                 shutil.move(ftm, destination)
 
         return Response(NotificationAckSerializer().data)
+
+
+class RfcIndexView(APIView):
+    api_key_endpoint = "ietf.api.views_rpc"
+
+    @extend_schema(
+        operation_id="refresh_rfc_index",
+        summary="Refresh rfc-index files",
+        description="Requests creation of rfc-index.xml and rfc-index.txt files",
+        responses={202: None},
+        request=None,
+    )
+    def post(self, request):
+        create_rfc_index_task.delay()
+        return Response(status=202)
diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py
index b15846094f..63c2044931 100644
--- a/ietf/sync/rfcindex.py
+++ b/ietf/sync/rfcindex.py
@@ -3,7 +3,6 @@
 from collections import defaultdict
 from collections.abc import Container
 from dataclasses import dataclass
-from io import StringIO, BytesIO
 from itertools import chain
 from operator import attrgetter, itemgetter
 from pathlib import Path
@@ -11,6 +10,7 @@
 from urllib.parse import urljoin
 
 from django.conf import settings
+from django.core.files.base import ContentFile
 from lxml import etree
 
 from django.core.files.storage import storages
@@ -28,7 +28,7 @@
 
 def format_rfc_number(n):
     """Format an RFC number (or subseries doc number)
-    
+
     Set settings.RFCINDEX_MATCH_LEGACY_XML=True for the legacy (leading-zero) format.
     That is for debugging only - tests will fail.
     """
@@ -42,13 +42,16 @@ def errata_url(rfc: Document):
     return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}")
 
 
-def save_to_red_bucket(filename: str, content: BytesIO | StringIO):
+def save_to_red_bucket(filename: str, content: str | bytes):
     red_bucket = storages["red_bucket"]
     bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename)
     if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True):
         # Django 4.2's FileSystemStorage does not support allow_overwrite.
         red_bucket.delete(bucket_path)
-    red_bucket.save(bucket_path, content)
+    red_bucket.save(
+        bucket_path,
+        ContentFile(content if isinstance(content, bytes) else content.encode("utf-8")),
+    )
     log(f"Saved {bucket_path} in red_bucket storage")
 
 
@@ -76,7 +79,7 @@ def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]:
     except json.JSONDecodeError:
         log(f"Error: unable to parse {bucket_path} in red_bucket storage")
         if settings.SERVER_MODE == "development":
-            return []  # pragma: no cover  
+            return []  # pragma: no cover
         raise
     assert all(isinstance(record["number"], int) for record in records)
     assert all(isinstance(record["comment"], str) for record in records)
@@ -441,7 +444,7 @@ def create_rfc_txt_index():
             "rfcs": get_rfc_text_index_entries(),
         },
     )
-    save_to_red_bucket("rfc-index.txt", StringIO(index))
+    save_to_red_bucket("rfc-index.txt", index)
 
 
 def create_rfc_xml_index():
@@ -477,4 +480,4 @@ def create_rfc_xml_index():
         xml_declaration=True,
         pretty_print=4,
     )
-    save_to_red_bucket("rfc-index.xml", BytesIO(pretty_index))
+    save_to_red_bucket("rfc-index.xml", pretty_index)
diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py
index fc75a056ed..4c84dc581e 100644
--- a/ietf/sync/tasks.py
+++ b/ietf/sync/tasks.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2024, All Rights Reserved
+# Copyright The IETF Trust 2024-2026, All Rights Reserved
 #
 # Celery task definitions
 #
@@ -18,6 +18,7 @@
 from ietf.sync import iana
 from ietf.sync import rfceditor
 from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue
+from ietf.sync.rfcindex import create_rfc_txt_index, create_rfc_xml_index
 from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper
 from ietf.utils import log
 from ietf.utils.timezone import date_today
@@ -272,3 +273,10 @@ def load_rfcs_into_blobdb_task(start: int, end: int):
     if end > 11000:  # Arbitrarily chosen
         end = 11000
     load_rfcs_into_blobdb(list(range(start, end + 1)))
+
+
+@shared_task
+def create_rfc_index_task():
+    create_rfc_txt_index()
+    create_rfc_xml_index()
+
diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py
index b0a8712fe1..e682c016f5 100644
--- a/ietf/sync/tests_rfcindex.py
+++ b/ietf/sync/tests_rfcindex.py
@@ -1,8 +1,8 @@
 # Copyright The IETF Trust 2026, All Rights Reserved
 import json
-from io import BytesIO, StringIO
 from unittest import mock
 
+from django.core.files.base import ContentFile
 from django.core.files.storage import storages
 from django.test.utils import override_settings
 from lxml import etree
@@ -13,7 +13,9 @@
     create_rfc_txt_index,
     create_rfc_xml_index,
     format_rfc_number,
-    save_to_red_bucket, get_unusable_rfc_numbers, get_april1_rfc_numbers,
+    save_to_red_bucket,
+    get_unusable_rfc_numbers,
+    get_april1_rfc_numbers,
     get_publication_std_levels,
 )
 from ietf.utils.test_utils import TestCase
@@ -39,7 +41,7 @@ def setUp(self):
         # Create an unused RFC number
         red_bucket.save(
             "input/unusable-rfc-numbers.json",
-            StringIO(json.dumps([{"number": 123, "comment": ""}])),
+            ContentFile(json.dumps([{"number": 123, "comment": ""}])),
         )
 
         # actual April 1 RFC
@@ -55,7 +57,7 @@ def setUp(self):
         # Set up a JSON file to flag the April 1 RFC
         red_bucket.save(
             "input/april-first-rfc-numbers.json",
-            StringIO(json.dumps([self.april_fools_rfc.rfc_number])),
+            ContentFile(json.dumps([self.april_fools_rfc.rfc_number])),
         )
 
         # non-April Fools RFC that happens to have been published on April 1
@@ -71,7 +73,7 @@ def setUp(self):
         # standard of self.rfc as different from its current value
         red_bucket.save(
             "input/publication-std-levels.json",
-            StringIO(
+            ContentFile(
                 json.dumps(
                     [{"number": self.rfc.rfc_number, "publication_std_level": "ps"}]
                 )
@@ -91,7 +93,8 @@ def test_create_rfc_txt_index(self, mock_save):
         create_rfc_txt_index()
         self.assertEqual(mock_save.call_count, 1)
         self.assertEqual(mock_save.call_args[0][0], "rfc-index.txt")
-        contents = mock_save.call_args[0][1].read()
+        contents = mock_save.call_args[0][1]
+        self.assertTrue(isinstance(contents, str))
         self.assertIn(
             "123 Not Issued.",
             contents,
@@ -119,7 +122,8 @@ def test_create_rfc_xml_index(self, mock_save):
         create_rfc_xml_index()
         self.assertEqual(mock_save.call_count, 1)
         self.assertEqual(mock_save.call_args[0][0], "rfc-index.xml")
-        contents = mock_save.call_args[0][1].read()
+        contents = mock_save.call_args[0][1]
+        self.assertTrue(isinstance(contents, bytes))
         ns = "{https://www.rfc-editor.org/rfc-index}"  # NOT an f-string
         index = etree.fromstring(contents)
 
@@ -190,13 +194,15 @@ def test_format_rfc_number(self):
     def test_save_to_red_bucket(self):
         red_bucket = storages["red_bucket"]
         with override_settings(RFCINDEX_DELETE_THEN_WRITE=False):
-            save_to_red_bucket("test", StringIO("contents"))
-        with red_bucket.open("test", "r") as f:
-            self.assertEqual(f.read(), "contents")
+            save_to_red_bucket("test", "contents \U0001f600")
+        # Read as binary and explicitly decode to confirm encoding
+        with red_bucket.open("test", "rb") as f:
+            self.assertEqual(f.read().decode("utf-8"), "contents \U0001f600")
         with override_settings(RFCINDEX_DELETE_THEN_WRITE=True):
-            save_to_red_bucket("test", BytesIO(b"new contents"))
-        with red_bucket.open("test", "r") as f:
-            self.assertEqual(f.read(), "new contents")
+            save_to_red_bucket("test", "new contents \U0001fae0".encode("utf-8"))
+        # Read as binary and explicitly decode to confirm encoding
+        with red_bucket.open("test", "rb") as f:
+            self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0")
         red_bucket.delete("test")  # clean up like a good child
 
     def test_get_unusable_rfc_numbers_raises(self):
@@ -204,7 +210,7 @@ def test_get_unusable_rfc_numbers_raises(self):
         with self.assertRaises(FileNotFoundError):
             get_unusable_rfc_numbers()
         red_bucket = storages["red_bucket"]
-        red_bucket.save("unusable-rfc-numbers.json", StringIO("not json"))
+        red_bucket.save("unusable-rfc-numbers.json", ContentFile("not json"))
         with self.assertRaises(json.JSONDecodeError):
             get_unusable_rfc_numbers()
         red_bucket.delete("unusable-rfc-numbers.json")
@@ -214,7 +220,7 @@ def test_get_april1_rfc_numbers_raises(self):
         with self.assertRaises(FileNotFoundError):
             get_april1_rfc_numbers()
         red_bucket = storages["red_bucket"]
-        red_bucket.save("april-first-rfc-numbers.json", StringIO("not json"))
+        red_bucket.save("april-first-rfc-numbers.json", ContentFile("not json"))
         with self.assertRaises(json.JSONDecodeError):
             get_april1_rfc_numbers()
         red_bucket.delete("april-first-rfc-numbers.json")
@@ -224,7 +230,7 @@ def test_get_publication_std_levels_raises(self):
         with self.assertRaises(FileNotFoundError):
             get_publication_std_levels()
         red_bucket = storages["red_bucket"]
-        red_bucket.save("publication-std-levels.json", StringIO("not json"))
+        red_bucket.save("publication-std-levels.json", ContentFile("not json"))
         with self.assertRaises(json.JSONDecodeError):
             get_publication_std_levels()
         red_bucket.delete("publication-std-levels.json")
diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 5ca4ba5cd9..56e395c5ac 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -447,7 +447,7 @@ def _multiline_to_list(s):
     "DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/"
 )
 RFCINDEX_INPUT_PATH = os.environ.get(
-    "DATATRACKR_RFCINDEX_INPUT_PATH", ""
+    "DATATRACKER_RFCINDEX_INPUT_PATH", ""
 )
 
 # Configure the blobdb app for artifact storage

From 2c59afe783216285b9695e99ee64547fe4e66469 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 13 Mar 2026 16:14:38 -0300
Subject: [PATCH 585/656] fix: drop stale obs/updates in rfced sync (#10543)

* fix: drop stale obs/updates in rfced sync

* refactor: partial revert, orig was safer
---
 ietf/sync/rfceditor.py | 73 +++++++++++++++++++++++++++++-------------
 1 file changed, 50 insertions(+), 23 deletions(-)

diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index cdcdeb5989..aa0e643b20 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -636,43 +636,70 @@ def update_docs_from_rfc_index(
             )
             rfc_published = True
 
-        def parse_relation_list(l):
-            res = []
-            for x in l:
-                for a in Document.objects.filter(name=x.lower(), type_id="rfc"):
-                    if a not in res:
-                        res.append(a)
-            return res
-
-        for x in parse_relation_list(obsoletes):
-            if not RelatedDocument.objects.filter(
-                source=doc, target=x, relationship=relationship_obsoletes
+        def parse_relation_list(rel_list: list[str]) -> list[Document]:
+            return list(
+                Document.objects.filter(
+                    name__in=[name.strip().lower() for name in rel_list],
+                    type_id="rfc"
+                )
+            )
+
+        # Create missing obsoletes relations
+        docs_this_obsoletes = parse_relation_list(obsoletes)
+        for obs_doc in docs_this_obsoletes:
+            if not doc.relateddocument_set.filter(
+                target=obs_doc, relationship=relationship_obsoletes
             ):
-                r = RelatedDocument.objects.create(
-                    source=doc, target=x, relationship=relationship_obsoletes
+                r = doc.relateddocument_set.create(
+                    target=obs_doc, relationship=relationship_obsoletes
                 )
                 rfc_changes.append(
-                    "created {rel_name} relation between {src_name} and {tgt_name}".format(
+                    "created {rel_name} relation between {src} and {tgt}".format(
                         rel_name=r.relationship.name.lower(),
-                        src_name=prettify_std_name(r.source.name),
-                        tgt_name=prettify_std_name(r.target.name),
+                        src=prettify_std_name(r.source.name),
+                        tgt=prettify_std_name(r.target.name),
                     )
                 )
+        # Remove stale obsoletes relations
+        for r in doc.relateddocument_set.filter(
+            relationship=relationship_obsoletes
+        ).exclude(target_id__in=[d.pk for d in docs_this_obsoletes]):
+            r.delete()
+            rfc_changes.append(
+                "removed {rel_name} relation between {src} and {tgt}".format(
+                    rel_name=r.relationship.name.lower(),
+                    src=prettify_std_name(r.source.name),
+                    tgt=prettify_std_name(r.target.name),
+                )
+            )
 
-        for x in parse_relation_list(updates):
+        docs_this_updates = parse_relation_list(updates)
+        for upd_doc in docs_this_updates:
             if not RelatedDocument.objects.filter(
-                source=doc, target=x, relationship=relationship_updates
+                source=doc, target=upd_doc, relationship=relationship_updates
             ):
-                r = RelatedDocument.objects.create(
-                    source=doc, target=x, relationship=relationship_updates
+                r = doc.relateddocument_set.create(
+                    target=upd_doc, relationship=relationship_updates
                 )
                 rfc_changes.append(
-                    "created {rel_name} relation between {src_name} and {tgt_name}".format(
+                    "created {rel_name} relation between {src} and {tgt}".format(
                         rel_name=r.relationship.name.lower(),
-                        src_name=prettify_std_name(r.source.name),
-                        tgt_name=prettify_std_name(r.target.name),
+                        src=prettify_std_name(r.source.name),
+                        tgt=prettify_std_name(r.target.name),
                     )
                 )
+        # Remove stale updates relations
+        for r in doc.relateddocument_set.filter(
+            relationship=relationship_updates
+        ).exclude(target_id__in=[d.pk for d in docs_this_updates]):
+            r.delete()
+            rfc_changes.append(
+                "removed {rel_name} relation between {src} and {tgt}".format(
+                    rel_name=r.relationship.name.lower(),
+                    src=prettify_std_name(r.source.name),
+                    tgt=prettify_std_name(r.target.name),
+                )
+            )
 
         if also:
             # recondition also to have proper subseries document names:

From 76fd25a1f39093a214be8ac2e0a9ed452beb7a47 Mon Sep 17 00:00:00 2001
From: Tianyi Gao <tianyi.gao@ed.ac.uk>
Date: Sat, 14 Mar 2026 12:19:51 +0800
Subject: [PATCH 586/656] fix: wording in id_expired_email template (#10154)

---
 ietf/templates/doc/draft/id_expired_email.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/templates/doc/draft/id_expired_email.txt b/ietf/templates/doc/draft/id_expired_email.txt
index afbf253ee2..161146a301 100644
--- a/ietf/templates/doc/draft/id_expired_email.txt
+++ b/ietf/templates/doc/draft/id_expired_email.txt
@@ -1,4 +1,4 @@
-{% autoescape off %}{{ doc.file_tag|safe }} was just expired.
+{% autoescape off %}{{ doc.file_tag|safe }} just expired.
 This Internet-Draft is in the state "{{ state }}" in the Datatracker.
 
 

From 9646edc20378e101ab48ff24253861fc5ea78fe9 Mon Sep 17 00:00:00 2001
From: Rudi Matz <rudi@staff.ietf.org>
Date: Sun, 15 Mar 2026 02:17:40 +0800
Subject: [PATCH 587/656] feat: add author affiliation in serializer (#10549)

---
 ietf/api/serializers_rpc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py
index c17cbc64ce..a18dc588c4 100644
--- a/ietf/api/serializers_rpc.py
+++ b/ietf/api/serializers_rpc.py
@@ -103,7 +103,7 @@ class DocumentAuthorSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = DocumentAuthor
-        fields = ["person", "plain_name"]
+        fields = ["person", "plain_name", "affiliation"]
 
     def get_plain_name(self, document_author: DocumentAuthor) -> str:
         return document_author.person.plain_name()

From 36fa518ec387f425b1f11f6f9040a73e8f61df30 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Mar 2026 10:02:47 -0400
Subject: [PATCH 588/656] chore(deps): bump the npm group across
 /dev/deploy-to-container with 5 updates (#10560)

Bumps the npm group with 5 updates in the /dev/deploy-to-container directory:

| Package | From | To |
| --- | --- | --- |
| [dockerode](https://github.com/apocas/dockerode) | `4.0.6` | `4.0.9` |
| [fs-extra](https://github.com/jprichardson/node-fs-extra) | `11.3.0` | `11.3.4` |
| [nanoid](https://github.com/ai/nanoid) | `5.1.5` | `5.1.7` |
| [slugify](https://github.com/simov/slugify) | `1.6.6` | `1.6.8` |
| [tar](https://github.com/isaacs/node-tar) | `7.4.3` | `7.5.11` |



Updates `dockerode` from 4.0.6 to 4.0.9
- [Release notes](https://github.com/apocas/dockerode/releases)
- [Commits](https://github.com/apocas/dockerode/compare/v4.0.6...v4.0.9)

Updates `fs-extra` from 11.3.0 to 11.3.4
- [Changelog](https://github.com/jprichardson/node-fs-extra/blob/master/CHANGELOG.md)
- [Commits](https://github.com/jprichardson/node-fs-extra/compare/11.3.0...11.3.4)

Updates `nanoid` from 5.1.5 to 5.1.7
- [Release notes](https://github.com/ai/nanoid/releases)
- [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md)
- [Commits](https://github.com/ai/nanoid/compare/5.1.5...5.1.7)

Updates `slugify` from 1.6.6 to 1.6.8
- [Changelog](https://github.com/simov/slugify/blob/master/CHANGELOG.md)
- [Commits](https://github.com/simov/slugify/compare/v1.6.6...v1.6.8)

Updates `tar` from 7.4.3 to 7.5.11
- [Release notes](https://github.com/isaacs/node-tar/releases)
- [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md)
- [Commits](https://github.com/isaacs/node-tar/compare/v7.4.3...v7.5.11)

---
updated-dependencies:
- dependency-name: dockerode
  dependency-version: 4.0.9
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
- dependency-name: fs-extra
  dependency-version: 11.3.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
- dependency-name: nanoid
  dependency-version: 5.1.7
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
- dependency-name: slugify
  dependency-version: 1.6.8
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
- dependency-name: tar
  dependency-version: 7.5.11
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/deploy-to-container/package-lock.json | 733 +++-------------------
 dev/deploy-to-container/package.json      |  10 +-
 2 files changed, 76 insertions(+), 667 deletions(-)

diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json
index 0954ec9af4..b62109f0e2 100644
--- a/dev/deploy-to-container/package-lock.json
+++ b/dev/deploy-to-container/package-lock.json
@@ -6,12 +6,12 @@
     "": {
       "name": "deploy-to-container",
       "dependencies": {
-        "dockerode": "^4.0.6",
-        "fs-extra": "^11.3.0",
-        "nanoid": "5.1.5",
+        "dockerode": "^4.0.9",
+        "fs-extra": "^11.3.4",
+        "nanoid": "5.1.7",
         "nanoid-dictionary": "5.0.0",
-        "slugify": "1.6.6",
-        "tar": "^7.4.3",
+        "slugify": "1.6.8",
+        "tar": "^7.5.11",
         "yargs": "^17.7.2"
       },
       "engines": {
@@ -52,95 +52,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/@isaacs/cliui": {
-      "version": "8.0.2",
-      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
-      "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
-      "dependencies": {
-        "string-width": "^5.1.2",
-        "string-width-cjs": "npm:string-width@^4.2.0",
-        "strip-ansi": "^7.0.1",
-        "strip-ansi-cjs": "npm:strip-ansi@^6.0.1",
-        "wrap-ansi": "^8.1.0",
-        "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0"
-      },
-      "engines": {
-        "node": ">=12"
-      }
-    },
-    "node_modules/@isaacs/cliui/node_modules/ansi-regex": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
-      "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==",
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-regex?sponsor=1"
-      }
-    },
-    "node_modules/@isaacs/cliui/node_modules/ansi-styles": {
-      "version": "6.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
-      "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==",
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
-      }
-    },
-    "node_modules/@isaacs/cliui/node_modules/emoji-regex": {
-      "version": "9.2.2",
-      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
-      "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
-    },
-    "node_modules/@isaacs/cliui/node_modules/string-width": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
-      "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
-      "dependencies": {
-        "eastasianwidth": "^0.2.0",
-        "emoji-regex": "^9.2.2",
-        "strip-ansi": "^7.0.1"
-      },
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/@isaacs/cliui/node_modules/strip-ansi": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
-      "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
-      "dependencies": {
-        "ansi-regex": "^6.0.1"
-      },
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/strip-ansi?sponsor=1"
-      }
-    },
-    "node_modules/@isaacs/cliui/node_modules/wrap-ansi": {
-      "version": "8.1.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
-      "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
-      "dependencies": {
-        "ansi-styles": "^6.1.0",
-        "string-width": "^5.0.1",
-        "strip-ansi": "^7.0.1"
-      },
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
-      }
-    },
     "node_modules/@isaacs/fs-minipass": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
@@ -161,15 +72,6 @@
         "url": "https://opencollective.com/js-sdsl"
       }
     },
-    "node_modules/@pkgjs/parseargs": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
-      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
-      "optional": true,
-      "engines": {
-        "node": ">=14"
-      }
-    },
     "node_modules/@protobufjs/aspromise": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
@@ -263,11 +165,6 @@
         "safer-buffer": "~2.1.0"
       }
     },
-    "node_modules/balanced-match": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
-      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
-    },
     "node_modules/base64-js": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -285,8 +182,7 @@
           "type": "consulting",
           "url": "https://feross.org/support"
         }
-      ],
-      "license": "MIT"
+      ]
     },
     "node_modules/bcrypt-pbkdf": {
       "version": "1.0.2",
@@ -301,21 +197,12 @@
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
       "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
-      "license": "MIT",
       "dependencies": {
         "buffer": "^5.5.0",
         "inherits": "^2.0.4",
         "readable-stream": "^3.4.0"
       }
     },
-    "node_modules/brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
-      "dependencies": {
-        "balanced-match": "^1.0.0"
-      }
-    },
     "node_modules/buffer": {
       "version": "5.7.1",
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
@@ -334,7 +221,6 @@
           "url": "https://feross.org/support"
         }
       ],
-      "license": "MIT",
       "dependencies": {
         "base64-js": "^1.3.1",
         "ieee754": "^1.1.13"
@@ -352,8 +238,7 @@
     "node_modules/chownr": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
-      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
-      "license": "ISC"
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
     },
     "node_modules/cliui": {
       "version": "8.0.1",
@@ -398,19 +283,6 @@
         "node": ">=10.0.0"
       }
     },
-    "node_modules/cross-spawn": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
-      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
-      "dependencies": {
-        "path-key": "^3.1.0",
-        "shebang-command": "^2.0.0",
-        "which": "^2.0.1"
-      },
-      "engines": {
-        "node": ">= 8"
-      }
-    },
     "node_modules/debug": {
       "version": "4.4.0",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
@@ -444,38 +316,31 @@
       }
     },
     "node_modules/dockerode": {
-      "version": "4.0.6",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz",
-      "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==",
-      "license": "Apache-2.0",
+      "version": "4.0.9",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz",
+      "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==",
       "dependencies": {
         "@balena/dockerignore": "^1.0.2",
         "@grpc/grpc-js": "^1.11.1",
         "@grpc/proto-loader": "^0.7.13",
         "docker-modem": "^5.0.6",
         "protobufjs": "^7.3.2",
-        "tar-fs": "~2.1.2",
+        "tar-fs": "^2.1.4",
         "uuid": "^10.0.0"
       },
       "engines": {
         "node": ">= 8.0"
       }
     },
-    "node_modules/eastasianwidth": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
-      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
-    },
     "node_modules/emoji-regex": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
       "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
     },
     "node_modules/end-of-stream": {
-      "version": "1.4.4",
-      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
-      "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
-      "license": "MIT",
+      "version": "1.4.5",
+      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",
+      "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==",
       "dependencies": {
         "once": "^1.4.0"
       }
@@ -488,32 +353,15 @@
         "node": ">=6"
       }
     },
-    "node_modules/foreground-child": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
-      "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
-      "dependencies": {
-        "cross-spawn": "^7.0.0",
-        "signal-exit": "^4.0.1"
-      },
-      "engines": {
-        "node": ">=14"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/fs-constants": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
-      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
-      "license": "MIT"
+      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
     },
     "node_modules/fs-extra": {
-      "version": "11.3.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz",
-      "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==",
-      "license": "MIT",
+      "version": "11.3.4",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz",
+      "integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==",
       "dependencies": {
         "graceful-fs": "^4.2.0",
         "jsonfile": "^6.0.1",
@@ -531,27 +379,6 @@
         "node": "6.* || 8.* || >= 10.*"
       }
     },
-    "node_modules/glob": {
-      "version": "10.3.12",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz",
-      "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==",
-      "dependencies": {
-        "foreground-child": "^3.1.0",
-        "jackspeak": "^2.3.6",
-        "minimatch": "^9.0.1",
-        "minipass": "^7.0.4",
-        "path-scurry": "^1.10.2"
-      },
-      "bin": {
-        "glob": "dist/esm/bin.mjs"
-      },
-      "engines": {
-        "node": ">=16 || 14 >=14.17"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/graceful-fs": {
       "version": "4.2.10",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz",
@@ -574,8 +401,7 @@
           "type": "consulting",
           "url": "https://feross.org/support"
         }
-      ],
-      "license": "BSD-3-Clause"
+      ]
     },
     "node_modules/inherits": {
       "version": "2.0.4",
@@ -590,28 +416,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/isexe": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="
-    },
-    "node_modules/jackspeak": {
-      "version": "2.3.6",
-      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz",
-      "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==",
-      "dependencies": {
-        "@isaacs/cliui": "^8.0.2"
-      },
-      "engines": {
-        "node": ">=14"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      },
-      "optionalDependencies": {
-        "@pkgjs/parseargs": "^0.11.0"
-      }
-    },
     "node_modules/jsonfile": {
       "version": "6.1.0",
       "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
@@ -633,28 +437,6 @@
       "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz",
       "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg=="
     },
-    "node_modules/lru-cache": {
-      "version": "10.2.2",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz",
-      "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==",
-      "engines": {
-        "node": "14 || >=16.14"
-      }
-    },
-    "node_modules/minimatch": {
-      "version": "9.0.4",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz",
-      "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==",
-      "dependencies": {
-        "brace-expansion": "^2.0.1"
-      },
-      "engines": {
-        "node": ">=16 || 14 >=14.17"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/minipass": {
       "version": "7.1.2",
       "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
@@ -664,36 +446,20 @@
       }
     },
     "node_modules/minizlib": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz",
-      "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==",
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz",
+      "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==",
       "dependencies": {
-        "minipass": "^7.0.4",
-        "rimraf": "^5.0.5"
+        "minipass": "^7.1.2"
       },
       "engines": {
         "node": ">= 18"
       }
     },
-    "node_modules/mkdirp": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz",
-      "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==",
-      "bin": {
-        "mkdirp": "dist/cjs/src/bin.js"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/mkdirp-classic": {
       "version": "0.5.3",
       "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
-      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
-      "license": "MIT"
+      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
     },
     "node_modules/ms": {
       "version": "2.1.3",
@@ -709,16 +475,15 @@
       "optional": true
     },
     "node_modules/nanoid": {
-      "version": "5.1.5",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz",
-      "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw==",
+      "version": "5.1.7",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.7.tgz",
+      "integrity": "sha512-ua3NDgISf6jdwezAheMOk4mbE1LXjm1DfMUDMuJf4AqxLFK3ccGpgWizwa5YV7Yz9EpXwEaWoRXSb/BnV0t5dQ==",
       "funding": [
         {
           "type": "github",
           "url": "https://github.com/sponsors/ai"
         }
       ],
-      "license": "MIT",
       "bin": {
         "nanoid": "bin/nanoid.js"
       },
@@ -736,34 +501,10 @@
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
       "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
-      "license": "ISC",
       "dependencies": {
         "wrappy": "1"
       }
     },
-    "node_modules/path-key": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
-      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/path-scurry": {
-      "version": "1.10.2",
-      "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz",
-      "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==",
-      "dependencies": {
-        "lru-cache": "^10.2.0",
-        "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0"
-      },
-      "engines": {
-        "node": ">=16 || 14 >=14.17"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/protobufjs": {
       "version": "7.4.0",
       "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz",
@@ -788,10 +529,9 @@
       }
     },
     "node_modules/pump": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz",
-      "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==",
-      "license": "MIT",
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz",
+      "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==",
       "dependencies": {
         "end-of-stream": "^1.1.0",
         "once": "^1.3.1"
@@ -818,23 +558,6 @@
         "node": ">=0.10.0"
       }
     },
-    "node_modules/rimraf": {
-      "version": "5.0.5",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz",
-      "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==",
-      "dependencies": {
-        "glob": "^10.3.7"
-      },
-      "bin": {
-        "rimraf": "dist/esm/bin.mjs"
-      },
-      "engines": {
-        "node": ">=14"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -860,40 +583,10 @@
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
       "license": "MIT"
     },
-    "node_modules/shebang-command": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
-      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
-      "dependencies": {
-        "shebang-regex": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/shebang-regex": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
-      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
-      "engines": {
-        "node": ">=8"
-      }
-    },
-    "node_modules/signal-exit": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
-      "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
-      "engines": {
-        "node": ">=14"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
-      }
-    },
     "node_modules/slugify": {
-      "version": "1.6.6",
-      "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz",
-      "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==",
+      "version": "1.6.8",
+      "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.8.tgz",
+      "integrity": "sha512-HVk9X1E0gz3mSpoi60h/saazLKXKaZThMLU3u/aNwoYn8/xQyX2MGxL0ui2eaokkD7tF+Zo+cKTHUbe1mmmGzA==",
       "engines": {
         "node": ">=8.0.0"
       }
@@ -942,20 +635,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/string-width-cjs": {
-      "name": "string-width",
-      "version": "4.2.3",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
-      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
-      "dependencies": {
-        "emoji-regex": "^8.0.0",
-        "is-fullwidth-code-point": "^3.0.0",
-        "strip-ansi": "^6.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/strip-ansi": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
@@ -967,28 +646,15 @@
         "node": ">=8"
       }
     },
-    "node_modules/strip-ansi-cjs": {
-      "name": "strip-ansi",
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-      "dependencies": {
-        "ansi-regex": "^5.0.1"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/tar": {
-      "version": "7.4.3",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz",
-      "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==",
+      "version": "7.5.11",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz",
+      "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==",
       "dependencies": {
         "@isaacs/fs-minipass": "^4.0.0",
         "chownr": "^3.0.0",
         "minipass": "^7.1.2",
-        "minizlib": "^3.0.1",
-        "mkdirp": "^3.0.1",
+        "minizlib": "^3.1.0",
         "yallist": "^5.0.0"
       },
       "engines": {
@@ -996,10 +662,9 @@
       }
     },
     "node_modules/tar-fs": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz",
-      "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==",
-      "license": "MIT",
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz",
+      "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==",
       "dependencies": {
         "chownr": "^1.1.1",
         "mkdirp-classic": "^0.5.2",
@@ -1011,7 +676,6 @@
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
       "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
-      "license": "MIT",
       "dependencies": {
         "bl": "^4.0.3",
         "end-of-stream": "^1.4.1",
@@ -1067,20 +731,6 @@
         "uuid": "dist/bin/uuid"
       }
     },
-    "node_modules/which": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
-      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
-      "dependencies": {
-        "isexe": "^2.0.0"
-      },
-      "bin": {
-        "node-which": "bin/node-which"
-      },
-      "engines": {
-        "node": ">= 8"
-      }
-    },
     "node_modules/wrap-ansi": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
@@ -1097,28 +747,10 @@
         "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
       }
     },
-    "node_modules/wrap-ansi-cjs": {
-      "name": "wrap-ansi",
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
-      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
-      "dependencies": {
-        "ansi-styles": "^4.0.0",
-        "string-width": "^4.1.0",
-        "strip-ansi": "^6.0.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
-      }
-    },
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
-      "license": "ISC"
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
     },
     "node_modules/y18n": {
       "version": "5.0.8",
@@ -1188,64 +820,6 @@
         "yargs": "^17.7.2"
       }
     },
-    "@isaacs/cliui": {
-      "version": "8.0.2",
-      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
-      "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
-      "requires": {
-        "string-width": "^5.1.2",
-        "string-width-cjs": "npm:string-width@^4.2.0",
-        "strip-ansi": "^7.0.1",
-        "strip-ansi-cjs": "npm:strip-ansi@^6.0.1",
-        "wrap-ansi": "^8.1.0",
-        "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "6.0.1",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
-          "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA=="
-        },
-        "ansi-styles": {
-          "version": "6.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
-          "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug=="
-        },
-        "emoji-regex": {
-          "version": "9.2.2",
-          "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
-          "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
-        },
-        "string-width": {
-          "version": "5.1.2",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
-          "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
-          "requires": {
-            "eastasianwidth": "^0.2.0",
-            "emoji-regex": "^9.2.2",
-            "strip-ansi": "^7.0.1"
-          }
-        },
-        "strip-ansi": {
-          "version": "7.1.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
-          "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
-          "requires": {
-            "ansi-regex": "^6.0.1"
-          }
-        },
-        "wrap-ansi": {
-          "version": "8.1.0",
-          "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
-          "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
-          "requires": {
-            "ansi-styles": "^6.1.0",
-            "string-width": "^5.0.1",
-            "strip-ansi": "^7.0.1"
-          }
-        }
-      }
-    },
     "@isaacs/fs-minipass": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
@@ -1259,12 +833,6 @@
       "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz",
       "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw=="
     },
-    "@pkgjs/parseargs": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
-      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
-      "optional": true
-    },
     "@protobufjs/aspromise": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
@@ -1348,11 +916,6 @@
         "safer-buffer": "~2.1.0"
       }
     },
-    "balanced-match": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
-      "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
-    },
     "base64-js": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -1376,14 +939,6 @@
         "readable-stream": "^3.4.0"
       }
     },
-    "brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
-      "requires": {
-        "balanced-match": "^1.0.0"
-      }
-    },
     "buffer": {
       "version": "5.7.1",
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
@@ -1437,16 +992,6 @@
         "nan": "^2.19.0"
       }
     },
-    "cross-spawn": {
-      "version": "7.0.3",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
-      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
-      "requires": {
-        "path-key": "^3.1.0",
-        "shebang-command": "^2.0.0",
-        "which": "^2.0.1"
-      }
-    },
     "debug": {
       "version": "4.4.0",
       "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
@@ -1467,33 +1012,28 @@
       }
     },
     "dockerode": {
-      "version": "4.0.6",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz",
-      "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==",
+      "version": "4.0.9",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz",
+      "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==",
       "requires": {
         "@balena/dockerignore": "^1.0.2",
         "@grpc/grpc-js": "^1.11.1",
         "@grpc/proto-loader": "^0.7.13",
         "docker-modem": "^5.0.6",
         "protobufjs": "^7.3.2",
-        "tar-fs": "~2.1.2",
+        "tar-fs": "^2.1.4",
         "uuid": "^10.0.0"
       }
     },
-    "eastasianwidth": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
-      "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
-    },
     "emoji-regex": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
       "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
     },
     "end-of-stream": {
-      "version": "1.4.4",
-      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
-      "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
+      "version": "1.4.5",
+      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",
+      "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==",
       "requires": {
         "once": "^1.4.0"
       }
@@ -1503,24 +1043,15 @@
       "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
       "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw=="
     },
-    "foreground-child": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
-      "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
-      "requires": {
-        "cross-spawn": "^7.0.0",
-        "signal-exit": "^4.0.1"
-      }
-    },
     "fs-constants": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
       "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
     },
     "fs-extra": {
-      "version": "11.3.0",
-      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz",
-      "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==",
+      "version": "11.3.4",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz",
+      "integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==",
       "requires": {
         "graceful-fs": "^4.2.0",
         "jsonfile": "^6.0.1",
@@ -1532,18 +1063,6 @@
       "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
       "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
     },
-    "glob": {
-      "version": "10.3.12",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz",
-      "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==",
-      "requires": {
-        "foreground-child": "^3.1.0",
-        "jackspeak": "^2.3.6",
-        "minimatch": "^9.0.1",
-        "minipass": "^7.0.4",
-        "path-scurry": "^1.10.2"
-      }
-    },
     "graceful-fs": {
       "version": "4.2.10",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz",
@@ -1564,20 +1083,6 @@
       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
       "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
     },
-    "isexe": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="
-    },
-    "jackspeak": {
-      "version": "2.3.6",
-      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz",
-      "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==",
-      "requires": {
-        "@isaacs/cliui": "^8.0.2",
-        "@pkgjs/parseargs": "^0.11.0"
-      }
-    },
     "jsonfile": {
       "version": "6.1.0",
       "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
@@ -1597,38 +1102,19 @@
       "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz",
       "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg=="
     },
-    "lru-cache": {
-      "version": "10.2.2",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz",
-      "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ=="
-    },
-    "minimatch": {
-      "version": "9.0.4",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz",
-      "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==",
-      "requires": {
-        "brace-expansion": "^2.0.1"
-      }
-    },
     "minipass": {
       "version": "7.1.2",
       "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
       "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw=="
     },
     "minizlib": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz",
-      "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==",
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz",
+      "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==",
       "requires": {
-        "minipass": "^7.0.4",
-        "rimraf": "^5.0.5"
+        "minipass": "^7.1.2"
       }
     },
-    "mkdirp": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz",
-      "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg=="
-    },
     "mkdirp-classic": {
       "version": "0.5.3",
       "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
@@ -1646,9 +1132,9 @@
       "optional": true
     },
     "nanoid": {
-      "version": "5.1.5",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.5.tgz",
-      "integrity": "sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw=="
+      "version": "5.1.7",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.7.tgz",
+      "integrity": "sha512-ua3NDgISf6jdwezAheMOk4mbE1LXjm1DfMUDMuJf4AqxLFK3ccGpgWizwa5YV7Yz9EpXwEaWoRXSb/BnV0t5dQ=="
     },
     "nanoid-dictionary": {
       "version": "5.0.0",
@@ -1663,20 +1149,6 @@
         "wrappy": "1"
       }
     },
-    "path-key": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
-      "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="
-    },
-    "path-scurry": {
-      "version": "1.10.2",
-      "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz",
-      "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==",
-      "requires": {
-        "lru-cache": "^10.2.0",
-        "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0"
-      }
-    },
     "protobufjs": {
       "version": "7.4.0",
       "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz",
@@ -1697,9 +1169,9 @@
       }
     },
     "pump": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz",
-      "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==",
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz",
+      "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==",
       "requires": {
         "end-of-stream": "^1.1.0",
         "once": "^1.3.1"
@@ -1720,14 +1192,6 @@
       "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
       "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="
     },
-    "rimraf": {
-      "version": "5.0.5",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz",
-      "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==",
-      "requires": {
-        "glob": "^10.3.7"
-      }
-    },
     "safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -1738,28 +1202,10 @@
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
-    "shebang-command": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
-      "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
-      "requires": {
-        "shebang-regex": "^3.0.0"
-      }
-    },
-    "shebang-regex": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
-      "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="
-    },
-    "signal-exit": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
-      "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="
-    },
     "slugify": {
-      "version": "1.6.6",
-      "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz",
-      "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw=="
+      "version": "1.6.8",
+      "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.8.tgz",
+      "integrity": "sha512-HVk9X1E0gz3mSpoi60h/saazLKXKaZThMLU3u/aNwoYn8/xQyX2MGxL0ui2eaokkD7tF+Zo+cKTHUbe1mmmGzA=="
     },
     "split-ca": {
       "version": "1.0.1",
@@ -1795,16 +1241,6 @@
         "strip-ansi": "^6.0.1"
       }
     },
-    "string-width-cjs": {
-      "version": "npm:string-width@4.2.3",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
-      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
-      "requires": {
-        "emoji-regex": "^8.0.0",
-        "is-fullwidth-code-point": "^3.0.0",
-        "strip-ansi": "^6.0.1"
-      }
-    },
     "strip-ansi": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
@@ -1813,24 +1249,15 @@
         "ansi-regex": "^5.0.1"
       }
     },
-    "strip-ansi-cjs": {
-      "version": "npm:strip-ansi@6.0.1",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-      "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-      "requires": {
-        "ansi-regex": "^5.0.1"
-      }
-    },
     "tar": {
-      "version": "7.4.3",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz",
-      "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==",
+      "version": "7.5.11",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz",
+      "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==",
       "requires": {
         "@isaacs/fs-minipass": "^4.0.0",
         "chownr": "^3.0.0",
         "minipass": "^7.1.2",
-        "minizlib": "^3.0.1",
-        "mkdirp": "^3.0.1",
+        "minizlib": "^3.1.0",
         "yallist": "^5.0.0"
       },
       "dependencies": {
@@ -1842,9 +1269,9 @@
       }
     },
     "tar-fs": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz",
-      "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==",
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz",
+      "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==",
       "requires": {
         "chownr": "^1.1.1",
         "mkdirp-classic": "^0.5.2",
@@ -1889,14 +1316,6 @@
       "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
       "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ=="
     },
-    "which": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
-      "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
-      "requires": {
-        "isexe": "^2.0.0"
-      }
-    },
     "wrap-ansi": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
@@ -1907,16 +1326,6 @@
         "strip-ansi": "^6.0.0"
       }
     },
-    "wrap-ansi-cjs": {
-      "version": "npm:wrap-ansi@7.0.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
-      "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
-      "requires": {
-        "ansi-styles": "^4.0.0",
-        "string-width": "^4.1.0",
-        "strip-ansi": "^6.0.0"
-      }
-    },
     "wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json
index 09716c3094..1c95a4540c 100644
--- a/dev/deploy-to-container/package.json
+++ b/dev/deploy-to-container/package.json
@@ -2,12 +2,12 @@
   "name": "deploy-to-container",
   "type": "module",
   "dependencies": {
-    "dockerode": "^4.0.6",
-    "fs-extra": "^11.3.0",
-    "nanoid": "5.1.5",
+    "dockerode": "^4.0.9",
+    "fs-extra": "^11.3.4",
+    "nanoid": "5.1.7",
     "nanoid-dictionary": "5.0.0",
-    "slugify": "1.6.6",
-    "tar": "^7.4.3",
+    "slugify": "1.6.8",
+    "tar": "^7.5.11",
     "yargs": "^17.7.2"
   },
   "engines": {

From dcce2df0300879078690a2fbd3522602d467cf38 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Fri, 20 Mar 2026 12:03:18 +0900
Subject: [PATCH 589/656] feat: add attendance summary and pie chart to meeting
 attendees page (#10481)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* feat: add attendance summary and pie chart to meeting attendees page

For IETF meetings ≥ 118, the attendees proceedings page now shows an
Onsite / Remote / Total summary row matching the counts displayed on
registration.ietf.org, together with a "View chart" button that opens
a Bootstrap modal containing a Highcharts pie chart.

* Split out attendees-chart.js
---
 ietf/meeting/tests_views.py                   | 13 +++++
 ietf/meeting/views.py                         | 33 +++++++++--
 ietf/static/js/attendees-chart.js             | 58 +++++++++++++++++++
 .../meeting/proceedings_attendees.html        | 53 ++++++++++++++++-
 package.json                                  |  1 +
 5 files changed, 151 insertions(+), 7 deletions(-)
 create mode 100644 ietf/static/js/attendees-chart.js

diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 168999d0aa..258ffe554c 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -9007,6 +9007,8 @@ def test_proceedings_attendees(self):
            - assert onsite checkedin=True appears, not onsite checkedin=False
            - assert remote attended appears, not remote not attended
            - prefer onsite checkedin=True to remote attended when same person has both
+           - summary stats row shows correct counts
+           - chart data JSON is embedded with correct values
         """
 
         m = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number="118")
@@ -9028,6 +9030,17 @@ def test_proceedings_attendees(self):
         text = q('#id_attendees tbody tr').text().replace('\n', ' ')
         self.assertEqual(text, f"A Person {areg.affiliation} {areg.country_code} onsite C Person {creg.affiliation} {creg.country_code} remote")
 
+        # Summary stats row: Onsite / Remote / Total (matches registration.ietf.org)
+        self.assertContains(response, 'Onsite:')
+        self.assertContains(response, 'Remote:')
+        self.assertContains(response, 'Total:')
+        self.assertContains(response, '<strong>1</strong>')   # onsite and remote
+        self.assertContains(response, '<strong>2</strong>')   # total
+
+        # Chart data embedded in page
+        chart_json = json.loads(q('#attendees-chart-data').text())
+        self.assertEqual(chart_json['type'], [['Onsite', 1], ['Remote', 1]])
+
     def test_proceedings_overview(self):
         '''Test proceedings IETF Overview page.
         Note: old meetings aren't supported so need to add a new meeting then test.
diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py
index 731dfad88f..67a81305b4 100644
--- a/ietf/meeting/views.py
+++ b/ietf/meeting/views.py
@@ -109,7 +109,7 @@
 from ietf.meeting.utils import get_activity_stats, post_process, create_recording, delete_recording
 from ietf.meeting.utils import participants_for_meeting, generate_bluesheet, bluesheet_data, save_bluesheet
 from ietf.message.utils import infer_message
-from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName
+from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName, CountryName
 from ietf.utils import markdown
 from ietf.utils.decorators import require_api_key
 from ietf.utils.hedgedoc import Note, NoteError
@@ -4812,15 +4812,36 @@ def proceedings_attendees(request, num=None):
     template = None
     registrations = None
 
+    stats = None
+    chart_data = None
+
     if int(meeting.number) >= 118:
         checked_in, attended = participants_for_meeting(meeting)
         regs = list(Registration.objects.onsite().filter(meeting__number=num, checkedin=True))
-
-        for reg in Registration.objects.remote().filter(meeting__number=num).select_related('person'):
-            if reg.person.pk in attended and reg.person.pk not in checked_in:
-                regs.append(reg)
+        onsite_count = len(regs)
+        regs += [
+            reg
+            for reg in Registration.objects.remote().filter(meeting__number=num).select_related('person')
+            if reg.person.pk in attended and reg.person.pk not in checked_in
+        ]
+        remote_count = len(regs) - onsite_count
 
         registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name))
+
+        country_codes = [r.country_code for r in registrations if r.country_code]
+        stats = {
+            'total': onsite_count + remote_count,
+            'onsite': onsite_count,
+            'remote': remote_count,
+        }
+
+        code_to_name = dict(CountryName.objects.values_list('slug', 'name'))
+        country_counts = Counter(code_to_name.get(c, c) for c in country_codes).most_common()
+
+        chart_data = {
+            'type': [['Onsite', onsite_count], ['Remote', remote_count]],
+            'countries': country_counts,
+        }
     else:
         overview_template = "/meeting/proceedings/%s/attendees.html" % meeting.number
         try:
@@ -4832,6 +4853,8 @@ def proceedings_attendees(request, num=None):
         'meeting': meeting,
         'registrations': registrations,
         'template': template,
+        'stats': stats,
+        'chart_data': chart_data,
     })
 
 def proceedings_overview(request, num=None):
diff --git a/ietf/static/js/attendees-chart.js b/ietf/static/js/attendees-chart.js
new file mode 100644
index 0000000000..fed3b1289c
--- /dev/null
+++ b/ietf/static/js/attendees-chart.js
@@ -0,0 +1,58 @@
+(function () {
+    var raw = document.getElementById('attendees-chart-data');
+    if (!raw) return;
+    var chartData = JSON.parse(raw.textContent);
+    var chart = null;
+    var currentBreakdown = 'type';
+
+    // Override the global transparent background set by highcharts.js so the
+    // export menu and fullscreen view use the page background color.
+    var container = document.getElementById('attendees-pie-chart');
+    var bodyBg = getComputedStyle(document.body).backgroundColor;
+    container.style.setProperty('--highcharts-background-color', bodyBg);
+
+    function renderChart(breakdown) {
+        var seriesData = chartData[breakdown].map(function (item) {
+            return { name: item[0], y: item[1] };
+        });
+        if (chart) chart.destroy();
+        chart = Highcharts.chart(container, {
+            chart: { type: 'pie', height: 400 },
+            title: { text: null },
+            tooltip: { pointFormat: '{point.name}: <b>{point.y}</b> ({point.percentage:.1f}%)' },
+            plotOptions: {
+                pie: {
+                    dataLabels: {
+                        enabled: true,
+                        format: '<b>{point.name}</b><br>{point.y} ({point.percentage:.1f}%)',
+                    },
+                    showInLegend: false,
+                }
+            },
+            series: [{ name: 'Attendees', data: seriesData }],
+        });
+    }
+
+    var modal = document.getElementById('attendees-chart-modal');
+
+    // Render (or re-render) the chart each time the modal becomes fully visible,
+    // so Highcharts can measure the container dimensions correctly.
+    modal.addEventListener('shown.bs.modal', function () {
+        renderChart(currentBreakdown);
+    });
+
+    // Release the chart when the modal closes to avoid stale renders.
+    modal.addEventListener('hidden.bs.modal', function () {
+        if (chart) {
+            chart.destroy();
+            chart = null;
+        }
+    });
+
+    document.querySelectorAll('[name="attendees-breakdown"]').forEach(function (radio) {
+        radio.addEventListener('change', function () {
+            currentBreakdown = this.value;
+            renderChart(currentBreakdown);
+        });
+    });
+})();
diff --git a/ietf/templates/meeting/proceedings_attendees.html b/ietf/templates/meeting/proceedings_attendees.html
index 390ce00cad..0c59d4ab15 100644
--- a/ietf/templates/meeting/proceedings_attendees.html
+++ b/ietf/templates/meeting/proceedings_attendees.html
@@ -3,6 +3,7 @@
 {% load origin markup_tags static %}
 {% block pagehead %}
     <link rel="stylesheet" href="{% static "ietf/css/list.css" %}">
+    {% if chart_data %}<link rel="stylesheet" href="{% static "ietf/css/highcharts.css" %}">{% endif %}
 {% endblock %}
 {% block title %}IETF {{ meeting.number }} proceedings{% endblock %}
 {% block content %}
@@ -14,8 +15,52 @@ <h1>
         </a>
     </h1>
     <h2>Attendee list of IETF {{ meeting.number }} meeting</h2>
-    
+
+    {% if chart_data %}
+    <div class="d-flex align-items-center gap-4 mb-3 flex-wrap">
+        <div class="d-flex gap-4">
+            <div>Onsite: <strong>{{ stats.onsite }}</strong></div>
+            <div>Remote: <strong>{{ stats.remote }}</strong></div>
+            <div>Total: <strong>{{ stats.total }}</strong></div>
+        </div>
+        <button type="button" class="btn btn-primary btn-sm"
+                data-bs-toggle="modal" data-bs-target="#attendees-chart-modal">
+            <i class="bi bi-pie-chart-fill"></i> View chart
+        </button>
+    </div>
+
+    <div class="modal fade" id="attendees-chart-modal" tabindex="-1"
+         aria-labelledby="attendees-chart-modal-label" aria-hidden="true">
+        <div class="modal-dialog modal-lg">
+            <div class="modal-content">
+                <div class="modal-header">
+                    <h2 class="modal-title fs-5" id="attendees-chart-modal-label">
+                        IETF {{ meeting.number }} attendees
+                    </h2>
+                    <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+                </div>
+                <div class="modal-body">
+                    <div class="btn-group mb-3" role="group" aria-label="Chart breakdown">
+                        <input type="radio" class="btn-check" name="attendees-breakdown"
+                               id="breakdown-type" value="type" checked>
+                        <label class="btn btn-outline-secondary btn-sm" for="breakdown-type">Onsite/Remote</label>
+                        <input type="radio" class="btn-check" name="attendees-breakdown"
+                               id="breakdown-countries" value="countries">
+                        <label class="btn btn-outline-secondary btn-sm" for="breakdown-countries">Countries</label>
+                    </div>
+                    <div id="attendees-pie-chart" class="bg-body"></div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    {{ chart_data|json_script:"attendees-chart-data" }}
+    {% endif %}{# chart_data #}
+
     {% if template %}
+    <div class="alert alert-info" role="alert">
+        Attendee statistics are not available for this meeting.
+    </div>
       {{template|safe}}
     {% else %}
         <table id="id_attendees" class="table table-sm table-striped tablesorter">
@@ -44,4 +89,8 @@ <h2>Attendee list of IETF {{ meeting.number }} meeting</h2>
 {% endblock %}
 {% block js %}
     <script src="{% static "ietf/js/list.js" %}"></script>
-{% endblock %}
\ No newline at end of file
+    {% if chart_data %}
+    <script src="{% static "ietf/js/highcharts.js" %}"></script>
+    <script src="{% static "ietf/js/attendees-chart.js" %}"></script>
+    {% endif %}
+{% endblock %}
diff --git a/package.json b/package.json
index fec29275b4..bb71250c4b 100644
--- a/package.json
+++ b/package.json
@@ -112,6 +112,7 @@
         "ietf/static/images/irtf-logo-white.svg",
         "ietf/static/images/irtf-logo.svg",
         "ietf/static/js/add_session_recordings.js",
+        "ietf/static/js/attendees-chart.js",
         "ietf/static/js/agenda_filter.js",
         "ietf/static/js/agenda_materials.js",
         "ietf/static/js/announcement.js",

From 2c29cbaad91a7c076643167e1ffc056975b2c97e Mon Sep 17 00:00:00 2001
From: Tianyi Gao <tianyi.gao@ed.ac.uk>
Date: Fri, 20 Mar 2026 11:45:09 +0800
Subject: [PATCH 590/656] feat: add parent section in team about (#9148)
 (#10551)

fix: remove empty <tr> for area/parent on all groups
---
 ietf/group/tests_info.py              | 19 +++++++++++++++++++
 ietf/templates/group/group_about.html | 11 +++++++----
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py
index 34f8500854..3f24e2e3d6 100644
--- a/ietf/group/tests_info.py
+++ b/ietf/group/tests_info.py
@@ -543,6 +543,25 @@ def verify_can_edit_group(url, group, username):
                 for username in list(set(interesting_users)-set(can_edit[group.type_id])):
                     verify_cannot_edit_group(url, group, username)
 
+    def test_group_about_team_parent(self):
+        """Team about page should show parent when parent is not an area"""
+        GroupFactory(type_id='team', parent=GroupFactory(type_id='area', acronym='gen'))
+        GroupFactory(type_id='team', parent=GroupFactory(type_id='ietf', acronym='iab'))
+        GroupFactory(type_id='team', parent=None)
+
+        for team in Group.objects.filter(type='team').select_related('parent'):
+            url = urlreverse('ietf.group.views.group_about', kwargs=dict(acronym=team.acronym))
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            if team.parent and team.parent.type_id != 'area':
+                self.assertContains(r, 'Parent')
+                self.assertContains(r, team.parent.acronym)
+            elif team.parent and team.parent.type_id == 'area':
+                self.assertContains(r, team.parent.name)
+                self.assertNotContains(r, '>Parent<')
+            else:
+                self.assertNotContains(r, '>Parent<')
+
     def test_group_about_personnel(self):
         """Correct personnel should appear on the group About page"""
         group = GroupFactory()
diff --git a/ietf/templates/group/group_about.html b/ietf/templates/group/group_about.html
index cbc2e11536..6d1843383c 100644
--- a/ietf/templates/group/group_about.html
+++ b/ietf/templates/group/group_about.html
@@ -51,10 +51,13 @@
                         {{ group.parent.name }}
                         <a href="{% url "ietf.group.views.group_home" acronym=group.parent.acronym %}">({{ group.parent.acronym }})</a>
                     </td>
-                {% else %}
-                    <td></td>
+                {% elif group.parent and group.type_id == "team" %}
+                    <th scope="row">Parent</th>
                     <td class="edit"></td>
-                    <td></td>
+                    <td>
+                        {{ group.parent.name }}
+                        <a href="{% url "ietf.group.views.group_home" acronym=group.parent.acronym %}">({{ group.parent.acronym }})</a>
+                    </td>
                 {% endif %}
             </tr>
             <tr>
@@ -444,4 +447,4 @@ <h2 class="mt-4">
             group_stats("{% url 'ietf.group.views.group_stats_data' %}", ".chart");
         });
     </script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}

From abab6373f5f465bdcc052f45c5def0710f360dc7 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 19 Mar 2026 23:02:40 -0500
Subject: [PATCH 591/656] fix: reduce db churn and log noise for rebuilding
 references (#10563)

* fix: (wip) reduce db churn and log noise for rebuilding references

* fix: typo in log message

* fix: typo in log message

Co-authored-by: Jennifer Richards <jeni@borkbork.org>

---------

Co-authored-by: Jennifer Richards <jeni@borkbork.org>
---
 ietf/doc/utils.py    | 78 ++++++++++++++++++++++++++++++--------------
 ietf/submit/utils.py |  5 +--
 2 files changed, 54 insertions(+), 29 deletions(-)

diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index 396b3fcfa4..8cbe5e8f3e 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -4,6 +4,7 @@
 
 import datetime
 import io
+import json
 import math
 import os
 import re
@@ -954,58 +955,78 @@ def rebuild_reference_relations(doc, filenames):
     filenames should be a dict mapping file ext (i.e., type) to the full path of each file.
     """
     if doc.type.slug not in ["draft", "rfc"]:
+        log.log(f"rebuild_reference_relations called for non draft/rfc doc {doc.name}")
         return None
-    
-    log.log(f"Rebuilding reference relations for {doc.name}")
 
-    # try XML first
-    if "xml" in filenames:
-        refs = XMLDraft(filenames["xml"]).get_refs()
-    elif "txt" in filenames:
-        filename = filenames["txt"]
-        try:
-            refs = draft.PlaintextDraft.from_file(filename).get_refs()
-        except IOError as e:
-            return {"errors": [f"{e.strerror}: {filename}"]}
-    else:
+
+    if "xml" not in filenames and "txt" not in filenames:
+        log.log(f"rebuild_reference_relations error: no file available for {doc.name}")
         return {
             "errors": [
                 "No file available for rebuilding reference relations. Need XML or plaintext."
             ]
         }
-
-    doc.relateddocument_set.filter(
+    else:
+        try:
+            # try XML first
+            if "xml" in filenames:
+                refs = XMLDraft(filenames["xml"]).get_refs()
+            elif "txt" in filenames:
+                filename = filenames["txt"]
+                refs = draft.PlaintextDraft.from_file(filename).get_refs()
+        except (IOError, UnicodeDecodeError) as e:
+                log.log(f"rebuild_reference_relations error: On {doc.name}: {e}")
+                return {"errors": [f"{e}: {filename}"]}
+
+    before = set(doc.relateddocument_set.filter(
         relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"]
-    ).delete()
+    ).values_list("relationship__slug","target__name"))
 
     warnings = []
     errors = []
     unfound = set()
+    intended = set()
+    names = [ref for ref in refs]
+    names.extend([ref[:-3] for ref in refs if re.match(r"^draft-.*-\d{2}$", ref)])
+    queryset = Document.objects.filter(name__in=names)
     for ref, refType in refs.items():
-        refdoc = Document.objects.filter(name=ref)
-        if not refdoc and re.match(r"^draft-.*-\d{2}$", ref):
-            refdoc = Document.objects.filter(name=ref[:-3])
+        refdoc = queryset.filter(name=ref)
+        if not refdoc.exists() and re.match(r"^draft-.*-\d{2}$", ref):
+            refdoc = queryset.filter(name=ref[:-3])
         count = refdoc.count()
         if count == 0:
             unfound.add("%s" % ref)
             continue
         elif count > 1:
+            log.unreachable("2026-3-16") # This branch is holdover from DocAlias
             errors.append("Too many Document objects found for %s" % ref)
         else:
             # Don't add references to ourself
             if doc != refdoc[0]:
-                RelatedDocument.objects.get_or_create(
-                    source=doc,
-                    target=refdoc[0],
-                    relationship=DocRelationshipName.objects.get(
-                        slug="ref%s" % refType
-                    ),
-                )
+                intended.add((f"ref{refType}", refdoc[0].name))
+
     if unfound:
         warnings.append(
             "There were %d references with no matching Document" % len(unfound)
         )
 
+    if intended != before:
+        for slug, name in before-intended:
+            doc.relateddocument_set.filter(target__name=name, relationship_id=slug).delete()
+        for slug, name in intended-before:
+            doc.relateddocument_set.create(
+                target=queryset.get(name=name),
+                relationship_id=slug
+            )
+        after = set(doc.relateddocument_set.filter(
+            relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"]
+        ).values_list("relationship__slug","target__name"))
+        if after != intended:
+            errors.append("Attempted changed didn't achieve intended results")
+        changed_references = True
+    else:
+        changed_references = False
+
     ret = {}
     if errors:
         ret["errors"] = errors
@@ -1014,6 +1035,13 @@ def rebuild_reference_relations(doc, filenames):
     if unfound:
         ret["unfound"] = list(unfound)
 
+    logmsg = f"rebuild_reference_relations for {doc.name}: "
+    logmsg += "changed references" if changed_references else "references unchanged"
+    if ret:
+        logmsg += f" {json.dumps(ret)}"
+
+    log.log(logmsg)
+
     return ret
 
 def set_replaces_for_document(request, doc, new_replaces, by, email_subject, comment=""):
diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index 9a7c358a6d..7e3106f723 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -395,10 +395,7 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc):
 
     log.log(f"{submission.name}: updated state and info")
 
-    trouble = rebuild_reference_relations(draft, find_submission_filenames(draft))
-    if trouble:
-        log.log('Rebuild_reference_relations trouble: %s'%trouble)
-    log.log(f"{submission.name}: rebuilt reference relations")
+    rebuild_reference_relations(draft, find_submission_filenames(draft))
     
     if draft.stream_id == "ietf" and draft.group.type_id == "wg" and draft.rev == "00":
         # automatically set state "WG Document"

From b08945aaf4618613f668bb5c231533f709bea4d4 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 20 Mar 2026 05:17:10 -0300
Subject: [PATCH 592/656] fix: maintain column count in HTML template (#10593)

---
 ietf/templates/group/group_about.html | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ietf/templates/group/group_about.html b/ietf/templates/group/group_about.html
index 6d1843383c..0a8b9194f2 100644
--- a/ietf/templates/group/group_about.html
+++ b/ietf/templates/group/group_about.html
@@ -58,6 +58,10 @@
                         {{ group.parent.name }}
                         <a href="{% url "ietf.group.views.group_home" acronym=group.parent.acronym %}">({{ group.parent.acronym }})</a>
                     </td>
+                {% else %}
+                    <td></td>
+                    <td class="edit"></td>
+                    <td></td>
                 {% endif %}
             </tr>
             <tr>

From d39317b070a7af5db4f48edaf0e7f03fd0a29680 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 23 Mar 2026 12:33:23 -0300
Subject: [PATCH 593/656] feat: update typesense search index on rfc pub/update
 (#10575)

* chore: typesense API config for k8s

* feat: DocumentInfo.pub_datetime() helper

* chore(deps): install typesense library

* feat: searchindex (typesense) util module

* feat: sanitize abstract

* feat: add (sanitized) content

* style: ruff ruff on doc/tasks.py

* feat: search index update task

* chore: call the update task

* refactor: better settings management

* ci: update prod settings

* chore: typing

* test: searchindex tests

* test: searchindex task test

* style: ruff ruff

* chore: drop type hints to fix mypy errors

* test: fix tests

* test: improve coverage

* fix: handle missing content blob correctly
---
 ietf/api/serializers_rpc.py       |   5 +-
 ietf/api/tests_serializers_rpc.py |  18 +++-
 ietf/api/tests_views_rpc.py       |  44 ++++++---
 ietf/api/views_rpc.py             |   3 +-
 ietf/doc/models.py                |  17 ++--
 ietf/doc/tasks.py                 |  46 +++++++--
 ietf/doc/tests_tasks.py           |  64 ++++++++++--
 ietf/utils/searchindex.py         | 155 ++++++++++++++++++++++++++++++
 ietf/utils/tests_searchindex.py   | 128 ++++++++++++++++++++++++
 k8s/settings_local.py             |  20 ++--
 requirements.txt                  |   1 +
 11 files changed, 451 insertions(+), 50 deletions(-)
 create mode 100644 ietf/utils/searchindex.py
 create mode 100644 ietf/utils/tests_searchindex.py

diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py
index a18dc588c4..701f05eece 100644
--- a/ietf/api/serializers_rpc.py
+++ b/ietf/api/serializers_rpc.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2025, All Rights Reserved
+# Copyright The IETF Trust 2025-2026, All Rights Reserved
 import datetime
 from pathlib import Path
 from typing import Literal, Optional
@@ -20,6 +20,7 @@
     RfcAuthor,
 )
 from ietf.doc.serializers import RfcAuthorSerializer
+from ietf.doc.tasks import update_rfc_searchindex_task
 from ietf.doc.utils import (
     default_consensus,
     prettify_std_name,
@@ -682,6 +683,8 @@ def update(self, instance, validated_data):
                 stale_subseries_relations.delete()
             if len(rfc_events) > 0:
                 rfc.save_with_history(rfc_events)
+
+        update_rfc_searchindex_task.delay(rfc.rfc_number)
         return rfc
 
 
diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py
index 1babb4c30f..ed326be451 100644
--- a/ietf/api/tests_serializers_rpc.py
+++ b/ietf/api/tests_serializers_rpc.py
@@ -1,4 +1,6 @@
 # Copyright The IETF Trust 2026, All Rights Reserved
+from unittest import mock
+
 from django.utils import timezone
 
 from ietf.utils.test_utils import TestCase
@@ -32,7 +34,8 @@ def test_create(self):
         with self.assertRaises(RuntimeError, msg="serializer does not allow create()"):
             serializer.save()
 
-    def test_update(self):
+    @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task")
+    def test_update(self, mock_update_searchindex_task):
         rfc = WgRfcFactory(pages=10)
         serializer = EditableRfcSerializer(
             instance=rfc,
@@ -56,6 +59,11 @@ def test_update(self):
         )
         self.assertTrue(serializer.is_valid())
         result = serializer.save()
+        self.assertTrue(mock_update_searchindex_task.delay.called)
+        self.assertEqual(
+            mock_update_searchindex_task.delay.call_args,
+            mock.call(rfc.rfc_number),
+        )
         result.refresh_from_db()
         self.assertEqual(result.title, "Yadda yadda yadda")
         self.assertEqual(
@@ -84,7 +92,8 @@ def test_update(self):
             [Document.objects.get(name="fyi999")],
         )
 
-    def test_partial_update(self):
+    @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task")
+    def test_partial_update(self, mock_update_searchindex_task):
         # We could test other permutations of fields, but authors is a partial update
         # we know we are going to use, so verifying that one in particular.
         rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan")
@@ -104,6 +113,11 @@ def test_partial_update(self):
         )
         self.assertTrue(serializer.is_valid())
         result = serializer.save()
+        self.assertTrue(mock_update_searchindex_task.delay.called)
+        self.assertEqual(
+            mock_update_searchindex_task.delay.call_args,
+            mock.call(rfc.rfc_number),
+        )
         result.refresh_from_db()
         self.assertEqual(rfc.title, "padawan")
         self.assertEqual(
diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py
index 7ab8778d28..a679e74789 100644
--- a/ietf/api/tests_views_rpc.py
+++ b/ietf/api/tests_views_rpc.py
@@ -196,7 +196,8 @@ def test_notify_rfc_published(self, mock_task_delay):
         self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list)
 
     @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
-    def test_upload_rfc_files(self):
+    @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task")
+    def test_upload_rfc_files(self, mock_update_searchindex_task):
         def _valid_post_data():
             """Generate a valid post data dict
 
@@ -217,14 +218,7 @@ def _valid_post_data():
             }
 
         url = urlreverse("ietf.api.purple_api.upload_rfc_files")
-        unused_rfc_number = (
-            Document.objects.filter(rfc_number__isnull=False).aggregate(
-                unused_rfc_number=Max("rfc_number") + 1
-            )["unused_rfc_number"]
-            or 10000
-        )
-
-        rfc = WgRfcFactory(rfc_number=unused_rfc_number)
+        rfc = WgRfcFactory()
         assert isinstance(rfc, Document), "WgRfcFactory should generate a Document"
         with TemporaryDirectory() as rfc_dir:
             settings.RFC_PATH = rfc_dir  # affects overridden settings
@@ -236,15 +230,17 @@ def _valid_post_data():
             # no api key
             r = self.client.post(url, _valid_post_data(), format="multipart")
             self.assertEqual(r.status_code, 403)
+            self.assertFalse(mock_update_searchindex_task.delay.called)
 
             # invalid RFC
             r = self.client.post(
                 url,
-                _valid_post_data() | {"rfc": unused_rfc_number + 1},
+                _valid_post_data() | {"rfc": rfc.rfc_number + 10},
                 format="multipart",
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 400)
+            self.assertFalse(mock_update_searchindex_task.delay.called)
 
             # empty files
             r = self.client.post(
@@ -263,6 +259,7 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 400)
+            self.assertFalse(mock_update_searchindex_task.delay.called)
 
             # bad file type
             r = self.client.post(
@@ -276,9 +273,10 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 400)
+            self.assertFalse(mock_update_searchindex_task.delay.called)
 
             # Put a file in the way. Post should fail because replace = False
-            file_in_the_way = (rfc_path / f"rfc{unused_rfc_number}.txt")
+            file_in_the_way = (rfc_path / f"{rfc.name}.txt")
             file_in_the_way.touch()
             r = self.client.post(
                 url,
@@ -287,11 +285,12 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 409)  # conflict
+            self.assertFalse(mock_update_searchindex_task.delay.called)
             file_in_the_way.unlink()
             
             # Put a blob in the way. Post should fail because replace = False
             blob_in_the_way = Blob.objects.create(
-                bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b""
+                bucket="rfc", name=f"txt/{rfc.name}.txt", content=b""
             )
             r = self.client.post(
                 url,
@@ -300,6 +299,7 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 409)  # conflict
+            self.assertFalse(mock_update_searchindex_task.delay.called)
             blob_in_the_way.delete()
 
             # valid post
@@ -310,8 +310,13 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 200)
+            self.assertTrue(mock_update_searchindex_task.delay.called)
+            self.assertEqual(
+                mock_update_searchindex_task.delay.call_args,
+                mock.call(rfc.rfc_number),
+            )
             for extension in ["xml", "txt", "html", "pdf", "json"]:
-                filename = f"rfc{unused_rfc_number}.{extension}"
+                filename = f"{rfc.name}.{extension}"
                 self.assertEqual(
                     (rfc_path / filename)
                     .read_text(),
@@ -328,7 +333,7 @@ def _valid_post_data():
                     f"{extension} blob should contain the expected content",
                 )
             # special case for notprepped
-            notprepped_fn = f"rfc{unused_rfc_number}.notprepped.xml"
+            notprepped_fn = f"{rfc.name}.notprepped.xml"
             self.assertEqual(
                 (
                     rfc_path / "prerelease" / notprepped_fn
@@ -347,6 +352,7 @@ def _valid_post_data():
             )
 
             # re-post with replace = False should now fail
+            mock_update_searchindex_task.reset_mock()
             r = self.client.post(
                 url,
                 _valid_post_data(),
@@ -354,7 +360,8 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 409)  # conflict
-            
+            self.assertFalse(mock_update_searchindex_task.delay.called)
+
             # re-post with replace = True should succeed
             r = self.client.post(
                 url,
@@ -362,7 +369,12 @@ def _valid_post_data():
                 format="multipart",
                 headers={"X-Api-Key": "valid-token"},
             )
-            self.assertEqual(r.status_code, 200)  # conflict
+            self.assertEqual(r.status_code, 200)
+            self.assertTrue(mock_update_searchindex_task.delay.called)
+            self.assertEqual(
+                mock_update_searchindex_task.delay.call_args,
+                mock.call(rfc.rfc_number),
+            )
 
     @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
     @mock.patch("ietf.api.views_rpc.create_rfc_index_task")
diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index c7ae699005..cb6a59a167 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -38,7 +38,7 @@
 from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent
 from ietf.doc.serializers import RfcAuthorSerializer
 from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage
-from ietf.doc.tasks import signal_update_rfc_metadata_task
+from ietf.doc.tasks import signal_update_rfc_metadata_task, update_rfc_searchindex_task
 from ietf.person.models import Email, Person
 from ietf.sync.tasks import create_rfc_index_task
 
@@ -516,6 +516,7 @@ def post(self, request):
                     destination.parent.mkdir()
                 shutil.move(ftm, destination)
 
+        update_rfc_searchindex_task.delay(rfc.rfc_number)
         return Response(NotificationAckSerializer().data)
 
 
diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index 7b23a62c45..972f0a34e8 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -1285,11 +1285,8 @@ def submission(self):
         s = s.first()
         return s
 
-    def pub_date(self):
-        """Get the publication date for this document
-
-        This is the rfc publication date for RFCs, and the new-revision date for other documents.
-        """
+    def pub_datetime(self):
+        """Get the publication datetime of this document"""
         if self.type_id == "rfc":
             # As of Sept 2022, in ietf.sync.rfceditor.update_docs_from_rfc_index() `published_rfc` events are
             # created with a timestamp whose date *in the PST8PDT timezone* is the official publication date
@@ -1297,7 +1294,15 @@ def pub_date(self):
             event = self.latest_event(type='published_rfc')
         else:
             event = self.latest_event(type='new_revision')
-        return event.time.astimezone(RPC_TZINFO).date() if event else None
+        return event.time.astimezone(RPC_TZINFO) if event else None
+
+    def pub_date(self):
+        """Get the publication date for this document
+
+        This is the rfc publication date for RFCs, and the new-revision date for other documents.
+        """
+        pub_datetime = self.pub_datetime()
+        return None if pub_datetime is None else pub_datetime.date()
 
     def is_dochistory(self):
         return False
diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py
index 90f4c80af5..a38cd5eb5c 100644
--- a/ietf/doc/tasks.py
+++ b/ietf/doc/tasks.py
@@ -3,6 +3,7 @@
 # Celery task definitions
 #
 import datetime
+
 import debug  # pyflakes:ignore
 
 from celery import shared_task
@@ -11,7 +12,7 @@
 from django.conf import settings
 from django.utils import timezone
 
-from ietf.utils import log
+from ietf.utils import log, searchindex
 from ietf.utils.timezone import datetime_today
 
 from .expire import (
@@ -77,17 +78,19 @@ def expire_last_calls_task():
         try:
             expire_last_call(doc)
         except Exception:
-            log.log(f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})")
+            log.log(
+                f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})"
+            )
         else:
             log.log(f"Expired last call for {doc.file_tag()} (id={doc.pk})")
 
 
-@shared_task            
+@shared_task
 def generate_idnits2_rfc_status_task():
     outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status"
     blob = generate_idnits2_rfc_status()
     try:
-        outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE
+        outpath.write_text(blob, encoding="utf8")  # TODO-BLOBSTORE
     except Exception as e:
         log.log(f"failed to write idnits2-rfc-status: {e}")
 
@@ -97,7 +100,7 @@ def generate_idnits2_rfcs_obsoleted_task():
     outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted"
     blob = generate_idnits2_rfcs_obsoleted()
     try:
-        outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE
+        outpath.write_text(blob, encoding="utf8")  # TODO-BLOBSTORE
     except Exception as e:
         log.log(f"failed to write idnits2-rfcs-obsoleted: {e}")
 
@@ -105,7 +108,7 @@ def generate_idnits2_rfcs_obsoleted_task():
 @shared_task
 def generate_draft_bibxml_files_task(days=7, process_all=False):
     """Generate bibxml files for recently updated docs
-    
+
     If process_all is False (the default), processes only docs with new revisions
     in the last specified number of days.
     """
@@ -117,7 +120,9 @@ def generate_draft_bibxml_files_task(days=7, process_all=False):
         doc__type_id="draft",
     ).order_by("time")
     if not process_all:
-        doc_events = doc_events.filter(time__gte=timezone.now() - datetime.timedelta(days=days))
+        doc_events = doc_events.filter(
+            time__gte=timezone.now() - datetime.timedelta(days=days)
+        )
     for event in doc_events:
         try:
             update_or_create_draft_bibxml_file(event.doc, event.rev)
@@ -132,6 +137,7 @@ def investigate_fragment_task(name_fragment: str):
         "results": investigate_fragment(name_fragment),
     }
 
+
 @shared_task
 def rebuild_reference_relations_task(doc_names: list[str]):
     log.log(f"Task: Rebuilding reference relations for {doc_names}")
@@ -157,6 +163,32 @@ def rebuild_reference_relations_task(doc_names: list[str]):
 def fixup_bofreq_timestamps_task():  # pragma: nocover
     fixup_bofreq_timestamps()
 
+
 @shared_task
 def signal_update_rfc_metadata_task(rfc_number_list=()):
     signal_update_rfc_metadata(rfc_number_list)
+
+
+@shared_task(bind=True)
+def update_rfc_searchindex_task(self, rfc_number: int):
+    """Update the search index for one RFC"""
+    if not searchindex.enabled():
+        log.log("Search indexing is not enabled, skipping")
+        return
+
+    rfc = Document.objects.filter(type_id="rfc", rfc_number=rfc_number).first()
+    if rfc is None:
+        log.log(
+            f"ERROR: Document for rfc{rfc_number} not found, not updating search index"
+        )
+        return
+    try:
+        searchindex.update_or_create_rfc_entry(rfc)
+    except Exception as err:
+        log.log(f"Search index update for {rfc.name} failed ({err})")
+        if isinstance(err, searchindex.RETRYABLE_ERROR_CLASSES):
+            searchindex_settings = searchindex.get_settings()
+            self.retry(
+                countdown=searchindex_settings["TASK_RETRY_DELAY"],
+                max_retries=searchindex_settings["TASK_MAX_RETRIES"],
+            )
diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py
index 29689cd596..728d21f131 100644
--- a/ietf/doc/tests_tasks.py
+++ b/ietf/doc/tests_tasks.py
@@ -1,18 +1,20 @@
-# Copyright The IETF Trust 2024, All Rights Reserved
+# Copyright The IETF Trust 2024-2026, All Rights Reserved
 
-import debug    # pyflakes:ignore
 import datetime
 from unittest import mock
 
 from pathlib import Path
 
+from celery.exceptions import Retry
 from django.conf import settings
+from django.test.utils import override_settings
 from django.utils import timezone
+from typesense import exceptions as typesense_exceptions
 
 from ietf.utils.test_utils import TestCase
 from ietf.utils.timezone import datetime_today
 
-from .factories import DocumentFactory, NewRevisionDocEventFactory
+from .factories import DocumentFactory, NewRevisionDocEventFactory, WgRfcFactory
 from .models import Document, NewRevisionDocEvent
 from .tasks import (
     expire_ids_task,
@@ -22,8 +24,10 @@
     generate_idnits2_rfc_status_task,
     investigate_fragment_task,
     notify_expirations_task,
+    update_rfc_searchindex_task,
 )
 
+
 class TaskTests(TestCase):
     @mock.patch("ietf.doc.tasks.in_draft_expire_freeze")
     @mock.patch("ietf.doc.tasks.get_expired_drafts")
@@ -87,7 +91,7 @@ def test_expire_last_calls_task(self, mock_get_expired, mock_expire):
         self.assertEqual(mock_expire.call_args_list[0], mock.call(docs[0]))
         self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1]))
         self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2]))
-    
+
         # Check that it runs even if exceptions occur
         mock_get_expired.reset_mock()
         mock_expire.reset_mock()
@@ -111,9 +115,40 @@ def test_investigate_fragment_task(self):
             retval, {"name_fragment": "some fragment", "results": investigation_results}
         )
 
+    @mock.patch("ietf.doc.tasks.searchindex.update_or_create_rfc_entry")
+    @mock.patch("ietf.doc.tasks.searchindex.enabled")
+    def test_update_rfc_searchindex_task(
+        self, mock_searchindex_enabled, mock_create_entry
+    ):
+        mock_searchindex_enabled.return_value = False
+
+        self.assertFalse(Document.objects.filter(rfc_number=5073).exists())
+        rfc = WgRfcFactory()
+        update_rfc_searchindex_task(rfc_number=5073)
+        self.assertFalse(mock_create_entry.called)
+        update_rfc_searchindex_task(rfc_number=rfc.rfc_number)
+        self.assertFalse(mock_create_entry.called)
+
+        mock_searchindex_enabled.return_value = True
+        update_rfc_searchindex_task(rfc_number=5073)
+        self.assertFalse(mock_create_entry.called)
+        update_rfc_searchindex_task(rfc_number=rfc.rfc_number)
+        self.assertTrue(mock_create_entry.called)
+
+        with override_settings(SEARCHINDEX_CONFIG={"TASK_MAX_RETRIES": 0}):
+            # Try a non-retryable error (there are others)
+            mock_create_entry.side_effect = typesense_exceptions.RequestMalformed
+            update_rfc_searchindex_task(rfc_number=rfc.rfc_number)  # no retry
+            # Now what should be a retryable error
+            mock_create_entry.side_effect = typesense_exceptions.Timeout
+            with self.assertRaises(Retry):
+                update_rfc_searchindex_task(rfc_number=rfc.rfc_number)
+
 
 class Idnits2SupportTests(TestCase):
-    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR']
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [
+        "DERIVED_DIR"
+    ]
 
     @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted")
     def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate):
@@ -151,7 +186,9 @@ def setUp(self):
         )
         # a couple that should always be ignored
         NewRevisionDocEventFactory(
-            time=now - datetime.timedelta(days=6), rev="09", doc__type_id="rfc"  # not a draft
+            time=now - datetime.timedelta(days=6),
+            rev="09",
+            doc__type_id="rfc",  # not a draft
         )
         NewRevisionDocEventFactory(
             type="changed_document",  # not a "new_revision" type
@@ -164,7 +201,9 @@ def setUp(self):
 
     @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists")
     @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file")
-    def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensure_path):
+    def test_generate_bibxml_files_for_all_drafts_task(
+        self, mock_create, mock_ensure_path
+    ):
         generate_draft_bibxml_files_task(process_all=True)
         self.assertTrue(mock_ensure_path.called)
         self.assertCountEqual(
@@ -193,12 +232,15 @@ def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensur
 
     @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists")
     @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file")
-    def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_ensure_path):
+    def test_generate_bibxml_files_for_recent_drafts_task(
+        self, mock_create, mock_ensure_path
+    ):
         # default args - look back 7 days
         generate_draft_bibxml_files_task()
         self.assertTrue(mock_ensure_path.called)
         self.assertCountEqual(
-            mock_create.call_args_list, [mock.call(self.young_event.doc, self.young_event.rev)]
+            mock_create.call_args_list,
+            [mock.call(self.young_event.doc, self.young_event.rev)],
         )
         mock_create.reset_mock()
         mock_ensure_path.reset_mock()
@@ -223,7 +265,9 @@ def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_en
 
     @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists")
     @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file")
-    def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value(self, mock_create, mock_ensure_path):
+    def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value(
+        self, mock_create, mock_ensure_path
+    ):
         with self.assertRaises(ValueError):
             generate_draft_bibxml_files_task(days=0)
         self.assertFalse(mock_create.called)
diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py
new file mode 100644
index 0000000000..e4427b88b5
--- /dev/null
+++ b/ietf/utils/searchindex.py
@@ -0,0 +1,155 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+"""Search indexing utilities"""
+
+import re
+from math import floor
+
+import httpx  # just for exceptions
+import typesense
+import typesense.exceptions
+from django.conf import settings
+
+from ietf.doc.models import Document, StoredObject
+from ietf.doc.storage_utils import retrieve_str
+from ietf.utils.log import log
+
+# Error classes that might succeed just by retrying a failed attempt.
+# Must be a tuple for use with isinstance()
+RETRYABLE_ERROR_CLASSES = (
+    httpx.ConnectError,
+    httpx.ConnectTimeout,
+    typesense.exceptions.Timeout,
+    typesense.exceptions.ServerError,
+    typesense.exceptions.ServiceUnavailable,
+)
+
+
+DEFAULT_SETTINGS = {
+    "TYPESENSE_API_URL": "",
+    "TYPESENSE_API_KEY": "",
+    "TYPESENSE_COLLECTION_NAME": "docs",
+    "TASK_RETRY_DELAY": 10,
+    "TASK_MAX_RETRIES": 12,
+}
+
+
+def get_settings():
+    return DEFAULT_SETTINGS | getattr(settings, "SEARCHINDEX_CONFIG", {})
+
+
+def enabled():
+    _settings = get_settings()
+    return _settings["TYPESENSE_API_URL"] != ""
+
+
+def _sanitize_text(content):
+    """Sanitize content or abstract text for search"""
+    # REs (with approximate names)
+    RE_DOT_OR_BANG_SPACE = r"\. |! "  # -> " " (space)
+    RE_COMMENT_OR_TOC_CRUD = r"<--|-->|--+|\+|\.\.+"  # -> ""
+    RE_BRACKETED_REF = r"\[[a-zA-Z0-9 -]+\]"  # -> ""
+    RE_DOTTED_NUMBERS = r"[0-9]+\.[0-9]+(\.[0-9]+)?"  # -> ""
+    RE_MULTIPLE_WHITESPACE = r"\s+"  # -> " " (space)
+    # Replacement values (for clarity of intent)
+    SPACE = " "
+    EMPTY = ""
+    # Sanitizing begins here, order is significant!
+    content = re.sub(RE_DOT_OR_BANG_SPACE, SPACE, content.strip())
+    content = re.sub(RE_COMMENT_OR_TOC_CRUD, EMPTY, content)
+    content = re.sub(RE_BRACKETED_REF, EMPTY, content)
+    content = re.sub(RE_DOTTED_NUMBERS, EMPTY, content)
+    content = re.sub(RE_MULTIPLE_WHITESPACE, SPACE, content)
+    return content.strip()
+
+
+def update_or_create_rfc_entry(rfc: Document):
+    assert rfc.type_id == "rfc"
+    assert rfc.rfc_number is not None
+
+    keywords: list[str] = rfc.keywords  # help type checking
+
+    subseries = rfc.part_of()
+    if len(subseries) > 1:
+        log(
+            f"RFC {rfc.rfc_number} is in multiple subseries. "
+            f"Indexing as {subseries[0].name}"
+        )
+    subseries = subseries[0] if len(subseries) > 0 else None
+    obsoleted_by = rfc.relations_that("obs")
+    updated_by = rfc.relations_that("updates")
+
+    stored_txt = (
+        StoredObject.objects.exclude_deleted()
+        .filter(store="rfc", doc_name=rfc.name, name__startswith="txt/")
+        .first()
+    )
+    content = ""
+    if stored_txt is not None:
+        # Should be available in the blobdb, but be cautious...
+        try:
+            content = retrieve_str(kind=stored_txt.store, name=stored_txt.name)
+        except Exception as err:
+            log(f"Unable to retrieve {stored_txt} from storage: {err}")
+
+    ts_id = f"doc-{rfc.pk}"
+    ts_document = {
+        "rfcNumber": rfc.rfc_number,
+        "rfc": str(rfc.rfc_number),
+        "filename": rfc.name,
+        "title": rfc.title,
+        "abstract": _sanitize_text(rfc.abstract),
+        "keywords": keywords,
+        "type": "rfc",
+        "state": [state.name for state in rfc.states.all()],
+        "status": {"slug": rfc.std_level.slug, "name": rfc.std_level.name},
+        "date": floor(rfc.time.timestamp()),
+        "publicationDate": floor(rfc.pub_datetime().timestamp()),
+        "stream": {"slug": rfc.stream.slug, "name": rfc.stream.name},
+        "authors": [
+            {"name": rfc_author.titlepage_name, "affiliation": rfc_author.affiliation}
+            for rfc_author in rfc.rfcauthor_set.all()
+        ],
+        "flags": {
+            "hiddenDefault": False,
+            "obsoleted": len(obsoleted_by) > 0,
+            "updated": len(updated_by) > 0,
+        },
+        "obsoletedBy": [str(doc.rfc_number) for doc in obsoleted_by],
+        "updatedBy": [str(doc.rfc_number) for doc in updated_by],
+        "ranking": rfc.rfc_number,
+    }
+    if subseries is not None:
+        ts_document["subseries"] = {
+            "acronym": subseries.type.slug,
+            "number": int(subseries.name[len(subseries.type.slug) :]),
+            "total": len(subseries.contains()),
+        }
+    if rfc.group is not None:
+        ts_document["group"] = {
+            "acronym": rfc.group.acronym,
+            "name": rfc.group.name,
+            "full": f"{rfc.group.acronym} - {rfc.group.name}",
+        }
+    if (
+        rfc.group.parent is not None
+        and rfc.stream_id not in ["ise", "irtf", "iab"]  # exclude editorial?
+    ):
+        ts_document["area"] = {
+            "acronym": rfc.group.parent.acronym,
+            "name": rfc.group.parent.name,
+            "full": f"{rfc.group.parent.acronym} - {rfc.group.parent.name}",
+        }
+    if rfc.ad is not None:
+        ts_document["adName"] = rfc.ad.name
+    if content != "":
+        ts_document["content"] = _sanitize_text(content)
+    _settings = get_settings()
+    client = typesense.Client(
+        {
+            "api_key": _settings["TYPESENSE_API_KEY"],
+            "nodes": [_settings["TYPESENSE_API_URL"]],
+        }
+    )
+    client.collections[_settings["TYPESENSE_COLLECTION_NAME"]].documents.upsert(
+        {"id": ts_id} | ts_document
+    )
diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py
new file mode 100644
index 0000000000..8740716c85
--- /dev/null
+++ b/ietf/utils/tests_searchindex.py
@@ -0,0 +1,128 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+from unittest import mock
+
+from django.conf import settings
+from django.test.utils import override_settings
+
+from . import searchindex
+from .test_utils import TestCase
+from ..blobdb.models import Blob
+from ..doc.factories import (
+    WgDraftFactory,
+    WgRfcFactory,
+    PublishedRfcDocEventFactory,
+    BcpFactory,
+    StdFactory,
+)
+from ..doc.models import Document
+from ..doc.storage_utils import store_str
+from ..person.factories import PersonFactory
+
+
+class SearchindexTests(TestCase):
+    def test_enabled(self):
+        with override_settings():
+            try:
+                del settings.SEARCHINDEX_CONFIG
+            except AttributeError:
+                pass
+            self.assertFalse(searchindex.enabled())
+        with override_settings(
+            SEARCHINDEX_CONFIG={"TYPESENSE_API_KEY": "this-is-not-a-key"}
+        ):
+            self.assertFalse(searchindex.enabled())
+        with override_settings(
+            SEARCHINDEX_CONFIG={"TYPESENSE_API_URL": "http://example.com"}
+        ):
+            self.assertTrue(searchindex.enabled())
+
+    def test_sanitize_text(self):
+        dirty_text = """
+        
+        This is text.  It + is <---- full    of \tprobl.....ems! Fix it. 
+        """
+        sanitized = "This is text It is full of problems Fix it."
+        self.assertEqual(searchindex._sanitize_text(dirty_text), sanitized)
+
+    @override_settings(
+        SEARCHINDEX_CONFIG={
+            "TYPESENSE_API_URL": "http://ts.example.com",
+            "TYPESENSE_API_KEY": "test-api-key",
+            "TYPESENSE_COLLECTION_NAME": "frogs",
+        }
+    )
+    @mock.patch("ietf.utils.searchindex.typesense.Client")
+    def test_update_or_create_rfc_entry(self, mock_ts_client_constructor):
+        not_rfc = WgDraftFactory()
+        assert isinstance(not_rfc, Document)
+        with self.assertRaises(AssertionError):
+            searchindex.update_or_create_rfc_entry(not_rfc)
+        self.assertFalse(mock_ts_client_constructor.called)
+
+        invalid_rfc = WgRfcFactory(name="rfc1000000", rfc_number=None)
+        assert isinstance(invalid_rfc, Document)
+        with self.assertRaises(AssertionError):
+            searchindex.update_or_create_rfc_entry(invalid_rfc)
+        self.assertFalse(mock_ts_client_constructor.called)
+
+        rfc = PublishedRfcDocEventFactory().doc
+        assert isinstance(rfc, Document)
+        searchindex.update_or_create_rfc_entry(rfc)
+        self.assertTrue(mock_ts_client_constructor.called)
+        # walk the tree down to the method we expected to be called...
+        mock_upsert = mock_ts_client_constructor.return_value.collections[
+            "frogs"
+        ].documents.upsert  # matches value in override_settings above
+        self.assertTrue(mock_upsert.called)
+        upserted_dict = mock_upsert.call_args[0][0]
+        # Check a few values, not exhaustive
+        self.assertEqual(upserted_dict["id"], f"doc-{rfc.pk}")
+        self.assertEqual(upserted_dict["rfcNumber"], rfc.rfc_number)
+        self.assertEqual(
+            upserted_dict["abstract"], searchindex._sanitize_text(rfc.abstract)
+        )
+        self.assertNotIn("adName", upserted_dict)
+        self.assertNotIn("content", upserted_dict)  # no blob
+        self.assertNotIn("subseries", upserted_dict)
+
+        # repeat, this time with contents, an AD, and subseries docs
+        mock_upsert.reset_mock()
+        store_str(
+            kind="rfc",
+            name=f"txt/{rfc.name}.txt",
+            content="The contents of this RFC",
+            doc_name=rfc.name,
+            doc_rev=rfc.rev,  # expected to be None
+        )
+        rfc.ad = PersonFactory(name="Alfred D. Rector")
+        # Put it in two Subseries docs to be sure this does not break things
+        # (the typesense schema does not support this for real at the moment)
+        BcpFactory(contains=[rfc], name="bcp1234")
+        StdFactory(contains=[rfc], name="std1234")
+        searchindex.update_or_create_rfc_entry(rfc)
+        self.assertTrue(mock_upsert.called)
+        upserted_dict = mock_upsert.call_args[0][0]
+        # Check a few values, not exhaustive
+        self.assertEqual(
+            upserted_dict["content"],
+            searchindex._sanitize_text("The contents of this RFC"),
+        )
+        self.assertEqual(upserted_dict["adName"], "Alfred D. Rector")
+        self.assertIn("subseries", upserted_dict)
+        ss_dict = upserted_dict["subseries"]
+        # We should get one of the two subseries docs, but neither is more correct
+        # than the other...
+        self.assertTrue(
+            any(
+                ss_dict == {"acronym": ss_type, "number": 1234, "total": 1}
+                for ss_type in ["bcp", "std"]
+            )
+        )
+
+        # Finally, delete the contents blob and make sure things don't blow up 
+        mock_upsert.reset_mock()
+        Blob.objects.get(bucket="rfc", name=f"txt/{rfc.name}.txt").delete()
+        searchindex.update_or_create_rfc_entry(rfc)
+        self.assertTrue(mock_upsert.called)
+        upserted_dict = mock_upsert.call_args[0][0]
+        self.assertNotIn("content", upserted_dict)
diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 56e395c5ac..8c0c66cdf2 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2007-2024, All Rights Reserved
+# Copyright The IETF Trust 2007-2026, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 from base64 import b64decode
@@ -443,12 +443,8 @@ def _multiline_to_list(s):
     ),
 }
 RFCINDEX_DELETE_THEN_WRITE = False  # S3Storage allows file_overwrite by default
-RFCINDEX_OUTPUT_PATH = os.environ.get(
-    "DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/"
-)
-RFCINDEX_INPUT_PATH = os.environ.get(
-    "DATATRACKER_RFCINDEX_INPUT_PATH", ""
-)
+RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/")
+RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH", "")
 
 # Configure the blobdb app for artifact storage
 _blobdb_replication_enabled = (
@@ -471,3 +467,13 @@ def _multiline_to_list(s):
 PASSWORD_POLICY_ENFORCE_AT_LOGIN = (
     os.environ.get("DATATRACKER_ENFORCE_PW_POLICY", "true").lower() != "false"
 )
+
+# Typesense search indexing
+SEARCHINDEX_CONFIG = {
+    "TYPESENSE_API_URL": os.environ.get("DATATRACKER_TYPESENSE_API_URL", ""),
+    "TYPESENSE_API_KEY": os.environ.get("DATATRACKER_TYPESENSE_API_KEY", ""),
+    "TASK_RETRY_DELAY": os.environ.get("DATATRACKER_SEARCHINDEX_TASK_RETRY_DELAY", 10),
+    "TASK_MAX_RETRIES": os.environ.get(
+        "DATATRACKER_SEARCHINDEX_TASK_MAX_RETRIES", "12"
+    ),
+}
diff --git a/requirements.txt b/requirements.txt
index 3d54b104ee..2b8185dab9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -75,6 +75,7 @@ pymemcache>=4.0.0  # for django.core.cache.backends.memcached.PyMemcacheCache
 python-mimeparse>=2.0.0  # from TastyPie
 pytz==2025.2  # Pinned as changes need to be vetted for their effect on Meeting fields
 types-pytz==2025.2.0.20251108  # match pytz version
+typesense>=2.0.0
 requests>=2.32.4
 types-requests>=2.32.4
 requests-mock>=1.12.1

From e6a3b3ebc03ef539454cfa154ad0242b32c6a335 Mon Sep 17 00:00:00 2001
From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com>
Date: Mon, 23 Mar 2026 15:46:07 +0000
Subject: [PATCH 594/656] ci: update base image target version to 20260323T1533

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index ce1828052e..af43e990e0 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20260304T1633
+FROM ghcr.io/ietf-tools/datatracker-app-base:20260323T1533
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 6be54fb6b0..09f74cce28 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20260304T1633
+20260323T1533

From 33f0dbf9e969a233f46251909515b249330fbb79 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 23 Mar 2026 12:39:15 -0500
Subject: [PATCH 595/656] feat: trigger red recomputation on RFC publication or
 metadata update (#10567)

* feat: trigger red recomputation on RFC publication or metadata update

* fix: move red precomputer call out of transaction

* chore: remove old comment, simplify request call

* fix: isolate delayed task in test

* test: give settings_test an InMemoryStorage for r2-rfc

* fix: follow obs/updates both ways when notifying red of changes

* fix: improve red utils, test red and r2 utils

* chore: ruff

* chore: remove unused import

* test: fix patch paths

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 ietf/api/serializers_rpc.py       |  15 +++-
 ietf/api/tests_serializers_rpc.py |  88 ++++++++++++++++---
 ietf/api/tests_views_rpc.py       |  24 ++++-
 ietf/api/views_rpc.py             |  12 ++-
 ietf/doc/tasks.py                 |  17 ++++
 ietf/doc/tests_utils.py           | 140 +++++++++++++++++++++++++++++-
 ietf/doc/utils_r2.py              |  17 ++++
 ietf/doc/utils_red.py             |  31 +++++++
 ietf/settings_test.py             |   6 +-
 k8s/settings_local.py             |  10 +++
 10 files changed, 341 insertions(+), 19 deletions(-)
 create mode 100644 ietf/doc/utils_r2.py
 create mode 100644 ietf/doc/utils_red.py

diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py
index 701f05eece..397ca05d9b 100644
--- a/ietf/api/serializers_rpc.py
+++ b/ietf/api/serializers_rpc.py
@@ -20,7 +20,7 @@
     RfcAuthor,
 )
 from ietf.doc.serializers import RfcAuthorSerializer
-from ietf.doc.tasks import update_rfc_searchindex_task
+from ietf.doc.tasks import trigger_red_precomputer_task, update_rfc_searchindex_task
 from ietf.doc.utils import (
     default_consensus,
     prettify_std_name,
@@ -683,7 +683,18 @@ def update(self, instance, validated_data):
                 stale_subseries_relations.delete()
             if len(rfc_events) > 0:
                 rfc.save_with_history(rfc_events)
-
+        # Gather obs and updates in both directions as a title/author change to
+        # this doc affects the info rendering of all of the other RFCs
+        needs_updating = sorted(
+            [
+                d.rfc_number
+                for d in [rfc]
+                + rfc.related_that_doc(("obs", "updates"))
+                + rfc.related_that(("obs", "updates"))
+            ]
+        )
+        trigger_red_precomputer_task.delay(rfc_number_list=needs_updating)
+        # Update the search index also
         update_rfc_searchindex_task.delay(rfc.rfc_number)
         return rfc
 
diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py
index ed326be451..167ffcd3ee 100644
--- a/ietf/api/tests_serializers_rpc.py
+++ b/ietf/api/tests_serializers_rpc.py
@@ -1,4 +1,5 @@
 # Copyright The IETF Trust 2026, All Rights Reserved
+
 from unittest import mock
 
 from django.utils import timezone
@@ -35,8 +36,21 @@ def test_create(self):
             serializer.save()
 
     @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task")
-    def test_update(self, mock_update_searchindex_task):
+    @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task")
+    def test_update(self, mock_trigger_red_task, mock_update_searchindex_task):
+        updates = WgRfcFactory.create_batch(2)
+        obsoletes = WgRfcFactory.create_batch(2)
         rfc = WgRfcFactory(pages=10)
+        updated_by = WgRfcFactory.create_batch(2)
+        obsoleted_by = WgRfcFactory.create_batch(2)
+        for d in updates:
+            rfc.relateddocument_set.create(relationship_id="updates",target=d)
+        for d in obsoletes:
+            rfc.relateddocument_set.create(relationship_id="updates",target=d)
+        for d in updated_by:
+            d.relateddocument_set.create(relationship_id="updates",target=rfc)
+        for d in obsoleted_by:
+            d.relateddocument_set.create(relationship_id="updates",target=rfc)        
         serializer = EditableRfcSerializer(
             instance=rfc,
             data={
@@ -59,11 +73,6 @@ def test_update(self, mock_update_searchindex_task):
         )
         self.assertTrue(serializer.is_valid())
         result = serializer.save()
-        self.assertTrue(mock_update_searchindex_task.delay.called)
-        self.assertEqual(
-            mock_update_searchindex_task.delay.call_args,
-            mock.call(rfc.rfc_number),
-        )
         result.refresh_from_db()
         self.assertEqual(result.title, "Yadda yadda yadda")
         self.assertEqual(
@@ -91,12 +100,42 @@ def test_update(self, mock_update_searchindex_task):
             result.part_of(),
             [Document.objects.get(name="fyi999")],
         )
+        # Confirm that red precomputer was triggered correctly
+        self.assertTrue(mock_trigger_red_task.delay.called)
+        _, mock_kwargs = mock_trigger_red_task.delay.call_args
+        self.assertIn("rfc_number_list", mock_kwargs)
+        expected_numbers = sorted(
+            [
+                d.rfc_number
+                for d in [rfc] + updates + obsoletes + updated_by + obsoleted_by
+            ]
+        )
+        self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers)
+        # Confirm that the search index update task was triggered correctly
+        self.assertTrue(mock_update_searchindex_task.delay.called)
+        self.assertEqual(
+            mock_update_searchindex_task.delay.call_args,
+            mock.call(rfc.rfc_number),
+        )
 
     @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task")
-    def test_partial_update(self, mock_update_searchindex_task):
+    @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task")
+    def test_partial_update(self, mock_trigger_red_task, mock_update_searchindex_task):
         # We could test other permutations of fields, but authors is a partial update
         # we know we are going to use, so verifying that one in particular.
+        updates = WgRfcFactory.create_batch(2)
+        obsoletes = WgRfcFactory.create_batch(2)
         rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan")
+        updated_by = WgRfcFactory.create_batch(2)
+        obsoleted_by = WgRfcFactory.create_batch(2)
+        for d in updates:
+            rfc.relateddocument_set.create(relationship_id="updates",target=d)
+        for d in obsoletes:
+            rfc.relateddocument_set.create(relationship_id="updates",target=d)
+        for d in updated_by:
+            d.relateddocument_set.create(relationship_id="updates",target=rfc)
+        for d in obsoleted_by:
+            d.relateddocument_set.create(relationship_id="updates",target=rfc) 
         serializer = EditableRfcSerializer(
             partial=True,
             instance=rfc,
@@ -113,11 +152,6 @@ def test_partial_update(self, mock_update_searchindex_task):
         )
         self.assertTrue(serializer.is_valid())
         result = serializer.save()
-        self.assertTrue(mock_update_searchindex_task.delay.called)
-        self.assertEqual(
-            mock_update_searchindex_task.delay.call_args,
-            mock.call(rfc.rfc_number),
-        )
         result.refresh_from_db()
         self.assertEqual(rfc.title, "padawan")
         self.assertEqual(
@@ -140,8 +174,27 @@ def test_partial_update(self, mock_update_searchindex_task):
         self.assertEqual(result.pages, 10)
         self.assertEqual(result.std_level_id, "ps")
         self.assertEqual(result.part_of(), [])
+        # Confirm that the red precomputer was triggered correctly
+        self.assertTrue(mock_trigger_red_task.delay.called)
+        _, mock_kwargs = mock_trigger_red_task.delay.call_args
+        self.assertIn("rfc_number_list", mock_kwargs)
+        expected_numbers = sorted(
+            [
+                d.rfc_number
+                for d in [rfc] + updates + obsoletes + updated_by + obsoleted_by
+            ]
+        )
+        self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers)
+        # Confirm that the search index update task was called correctly
+        self.assertTrue(mock_update_searchindex_task.delay.called)
+        self.assertEqual(
+            mock_update_searchindex_task.delay.call_args,
+            mock.call(rfc.rfc_number),
+        )
 
         # Test only a field on the Document itself to be sure that it works
+        mock_trigger_red_task.delay.reset_mock()
+        mock_update_searchindex_task.delay.reset_mock()
         serializer = EditableRfcSerializer(
             partial=True,
             instance=rfc,
@@ -151,3 +204,14 @@ def test_partial_update(self, mock_update_searchindex_task):
         result = serializer.save()
         result.refresh_from_db()
         self.assertEqual(rfc.title, "jedi master")
+        # Confirm that the red precomputer was triggered correctly
+        self.assertTrue(mock_trigger_red_task.delay.called)
+        _, mock_kwargs = mock_trigger_red_task.delay.call_args
+        self.assertIn("rfc_number_list", mock_kwargs)
+        self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers)
+        # Confirm that the search index update task was called correctly
+        self.assertTrue(mock_update_searchindex_task.delay.called)
+        self.assertEqual(
+            mock_update_searchindex_task.delay.call_args,
+            mock.call(rfc.rfc_number),
+        )
diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py
index a679e74789..6d10bee8e8 100644
--- a/ietf/api/tests_views_rpc.py
+++ b/ietf/api/tests_views_rpc.py
@@ -197,7 +197,8 @@ def test_notify_rfc_published(self, mock_task_delay):
 
     @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
     @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task")
-    def test_upload_rfc_files(self, mock_update_searchindex_task):
+    @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task")
+    def test_upload_rfc_files(self, mock_trigger_red_task, mock_update_searchindex_task):
         def _valid_post_data():
             """Generate a valid post data dict
 
@@ -218,7 +219,14 @@ def _valid_post_data():
             }
 
         url = urlreverse("ietf.api.purple_api.upload_rfc_files")
+        updates = RfcFactory.create_batch(2)
+        obsoletes = RfcFactory.create_batch(2)
+
         rfc = WgRfcFactory()
+        for r in obsoletes:
+            rfc.relateddocument_set.create(relationship_id="obs", target=r)
+        for r in updates:
+            rfc.relateddocument_set.create(relationship_id="updates", target=r)
         assert isinstance(rfc, Document), "WgRfcFactory should generate a Document"
         with TemporaryDirectory() as rfc_dir:
             settings.RFC_PATH = rfc_dir  # affects overridden settings
@@ -303,6 +311,7 @@ def _valid_post_data():
             blob_in_the_way.delete()
 
             # valid post
+            mock_trigger_red_task.delay.reset_mock()
             r = self.client.post(
                 url,
                 _valid_post_data(),
@@ -310,7 +319,6 @@ def _valid_post_data():
                 headers={"X-Api-Key": "valid-token"},
             )
             self.assertEqual(r.status_code, 200)
-            self.assertTrue(mock_update_searchindex_task.delay.called)
             self.assertEqual(
                 mock_update_searchindex_task.delay.call_args,
                 mock.call(rfc.rfc_number),
@@ -350,6 +358,18 @@ def _valid_post_data():
                 b"This is .notprepped.xml",
                 ".notprepped.xml blob should contain the expected content",
             )
+            # Confirm that the red precomputer was triggered correctly
+            self.assertTrue(mock_trigger_red_task.delay.called)
+            _, mock_kwargs = mock_trigger_red_task.delay.call_args
+            self.assertIn("rfc_number_list", mock_kwargs)
+            expected_rfc_number_list = [rfc.rfc_number]
+            expected_rfc_number_list.extend(
+                [d.rfc_number for d in updates + obsoletes]
+            )
+            expected_rfc_number_list = sorted(set(expected_rfc_number_list))
+            self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list)
+            # Confirm that the search index update task was called correctly
+            self.assertTrue(mock_update_searchindex_task.delay.called)
 
             # re-post with replace = False should now fail
             mock_update_searchindex_task.reset_mock()
diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index cb6a59a167..59eed1e10e 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -38,7 +38,11 @@
 from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent
 from ietf.doc.serializers import RfcAuthorSerializer
 from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage
-from ietf.doc.tasks import signal_update_rfc_metadata_task, update_rfc_searchindex_task
+from ietf.doc.tasks import (
+    signal_update_rfc_metadata_task,
+    trigger_red_precomputer_task,
+    update_rfc_searchindex_task,
+)
 from ietf.person.models import Email, Person
 from ietf.sync.tasks import create_rfc_index_task
 
@@ -516,6 +520,12 @@ def post(self, request):
                     destination.parent.mkdir()
                 shutil.move(ftm, destination)
 
+        # Trigger red precomputer
+        needs_updating = [rfc.rfc_number]
+        for rel in rfc.relateddocument_set.filter(relationship_id__in=["obs","updates"]):
+            needs_updating.append(rel.target.rfc_number)
+        trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating))
+        # Trigger search index update
         update_rfc_searchindex_task.delay(rfc.rfc_number)
         return Response(NotificationAckSerializer().data)
 
diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py
index a38cd5eb5c..19edb39014 100644
--- a/ietf/doc/tasks.py
+++ b/ietf/doc/tasks.py
@@ -7,11 +7,14 @@
 import debug  # pyflakes:ignore
 
 from celery import shared_task
+from celery.exceptions import MaxRetriesExceededError
 from pathlib import Path
 
 from django.conf import settings
 from django.utils import timezone
 
+from ietf.doc.utils_r2 import rfcs_are_in_r2
+from ietf.doc.utils_red import trigger_red_precomputer
 from ietf.utils import log, searchindex
 from ietf.utils.timezone import datetime_today
 
@@ -169,6 +172,20 @@ def signal_update_rfc_metadata_task(rfc_number_list=()):
     signal_update_rfc_metadata(rfc_number_list)
 
 
+@shared_task(bind=True)
+def trigger_red_precomputer_task(self, rfc_number_list=()):
+    if not rfcs_are_in_r2(rfc_number_list):
+        log.log(f"Objects are not yet in R2 for RFCs {rfc_number_list}")
+        try:
+            countdown = getattr(settings, "RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", 10)
+            max_retries = getattr(settings, "RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", 12)
+            self.retry(countdown=countdown, max_retries=max_retries)
+        except MaxRetriesExceededError:
+            log.log(f"Gave up waiting for objects in R2 for RFCs {rfc_number_list}")
+    else:
+        trigger_red_precomputer(rfc_number_list)
+
+
 @shared_task(bind=True)
 def update_rfc_searchindex_task(self, rfc_number: int):
     """Update the search index for one RFC"""
diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py
index a2784bc85e..ba672cd847 100644
--- a/ietf/doc/tests_utils.py
+++ b/ietf/doc/tests_utils.py
@@ -1,15 +1,23 @@
 # Copyright The IETF Trust 2020, All Rights Reserved
 import datetime
+from io import BytesIO
+
+import mock
 import debug  # pyflakes:ignore
+import requests
 
 from pathlib import Path
 from unittest.mock import call, patch
 
 from django.conf import settings
+from django.core.files.storage import storages
 from django.db import IntegrityError
 from django.test.utils import override_settings
 from django.utils import timezone
 
+
+from ietf.doc.utils_r2 import rfcs_are_in_r2
+from ietf.doc.utils_red import trigger_red_precomputer
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.name.models import DocTagName
 from ietf.person.factories import PersonFactory
@@ -17,11 +25,12 @@
 from ietf.utils.test_utils import TestCase, name_of_file_containing, reload_db_objects
 from ietf.person.models import Person
 from ietf.doc.factories import DocumentFactory, WgRfcFactory, WgDraftFactory
-from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor
+from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor, StoredObject
 from ietf.doc.utils import (update_action_holders, add_state_change_event, update_documentauthors,
                             fuzzy_find_documents, rebuild_reference_relations, build_file_urls,
                             ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file,
                             last_ballot_doc_revision)
+from ietf.doc.storage_utils import store_str
 from ietf.utils.draft import Draft, PlaintextDraft
 from ietf.utils.xmldraft import XMLDraft
 
@@ -559,3 +568,132 @@ def test_last_ballot_doc_revision(self):
         nobody = PersonFactory()
         self.assertIsNone(last_ballot_doc_revision(doc, nobody))
         self.assertEqual(rev, last_ballot_doc_revision(doc, ad))
+
+
+class UtilsRedTests(TestCase):
+    @mock.patch("ietf.doc.utils_red.log")
+    @mock.patch("ietf.doc.utils_red.requests.post")
+    def test_trigger_red_precomputer_not_configured(self, mock_post, mock_log):
+        with override_settings():
+            try:
+                del settings.CUSTOM_SETTING_NAME
+            except AttributeError:
+                pass
+            trigger_red_precomputer(rfc_number_list=[1, 2, 3])
+            self.assertEqual(mock_log.call_count, 1)
+            mock_args, _ = mock_log.call_args
+            self.assertEqual(
+                mock_args,
+                ("No URL configured for triggering red precompute multiple, skipping",),
+            )
+
+        mock_log.reset_mock()
+        with override_settings(TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL=None):
+            trigger_red_precomputer(rfc_number_list=[1, 2, 3])
+            self.assertFalse(mock_post.called)
+            self.assertEqual(mock_log.call_count, 1)
+            mock_args, _ = mock_log.call_args
+            self.assertEqual(
+                mock_args,
+                ("No URL configured for triggering red precompute multiple, skipping",),
+            )
+
+    @override_settings(
+        TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits",
+    )
+    @mock.patch("ietf.doc.utils_red.log")
+    @mock.patch("ietf.doc.utils_red.requests.post", side_effect=requests.Timeout())
+    def test_trigger_red_precomputer_swallows_timeout_exception(
+        self, mock_post, mock_log
+    ):
+        exception_raised = False
+        try:
+            trigger_red_precomputer(rfc_number_list=[1, 2, 3])
+        except Exception:
+            exception_raised = True
+        self.assertFalse(exception_raised)
+        self.assertEqual(mock_log.call_count, 2)
+        # only checking the last log call
+        mock_args, _ = mock_log.call_args
+        self.assertEqual(len(mock_args), 1)
+        self.assertIn("POST request timed out", mock_args[0])
+
+    @override_settings(
+        TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits",
+    )
+    @mock.patch("ietf.doc.utils_red.requests.post", side_effect=Exception())
+    def test_trigger_red_precomputer_does_not_swallow_too_much(self, mock_post):
+        exception_raised = False
+        try:
+            trigger_red_precomputer(rfc_number_list=[1, 2, 3])
+        except Exception:
+            exception_raised = True
+        self.assertTrue(exception_raised)
+
+    @override_settings(
+        TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits",
+        DEFAULT_REQUESTS_TIMEOUT=314159265,
+    )
+    @mock.patch("ietf.doc.utils_red.log")
+    @mock.patch("ietf.doc.utils_red.requests.post")
+    def test_trigger_red_precomputer(self, mock_post, mock_log):
+        mock_post.return_value = mock.Mock(status_code=200)
+        trigger_red_precomputer(rfc_number_list=[1, 2, 3])
+        self.assertTrue(mock_post.called)
+        _, mock_kwargs = mock_post.call_args
+        self.assertIn("url", mock_kwargs)
+        self.assertEqual(mock_kwargs["url"], "urlbits")
+        self.assertIn("json", mock_kwargs)
+        self.assertEqual(mock_kwargs["json"], {"rfcs": "1,2,3"})
+        self.assertIn("timeout", mock_kwargs)
+        self.assertEqual(mock_kwargs["timeout"], 314159265)
+        self.assertEqual(mock_log.call_count, 1)  # Not testing the first info log value
+        mock_log.reset_mock()
+        mock_post.reset_mock()
+        mock_post.return_value = mock.Mock(
+            status_code=500,
+        )
+        trigger_red_precomputer(rfc_number_list=[1, 2, 3])
+        self.assertEqual(mock_log.call_count, 2)
+        mock_args, _ = mock_log.call_args
+        self.assertEqual(len(mock_args), 1)
+        expected = f"POST request failed for {settings.TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL} : status_code=500"
+        self.assertEqual(mock_args[0], expected)
+
+
+class UtilsR2TestCase(TestCase):
+    def test_rfcs_are_in_r2(self):
+        rfcs = WgRfcFactory.create_batch(2)
+        rfc_name_list = [rfc.name for rfc in rfcs]
+        rfc_number_list = [rfc.rfc_number for rfc in rfcs]
+        r2_rfc_bucket = storages["r2-rfc"]
+        # Right now the various doc Factories do not populate any content
+        self.assertEqual(
+            StoredObject.objects.filter(
+                store="rfc", doc_name__in=rfc_name_list
+            ).count(),
+            0,
+        )
+        self.assertTrue(rfcs_are_in_r2(rfc_number_list=rfc_number_list))
+        for rfc in rfcs:
+            store_str(
+                kind="rfc",
+                name=f"testartifact/{rfc.name}.testartifact",
+                content="",
+                doc_name=rfc.name,
+                doc_rev=None,
+            )
+        self.assertEqual(
+            StoredObject.objects.filter(
+                store="rfc", doc_name__in=rfc_name_list
+            ).count(),
+            2,
+        )
+        self.assertFalse(rfcs_are_in_r2(rfc_number_list=rfc_number_list))
+        r2_rfc_bucket.save(f"testartifact/{rfcs[0].name}.testartifact", BytesIO(b""))
+        self.assertFalse(rfcs_are_in_r2(rfc_number_list=rfc_number_list))
+        r2_rfc_bucket.save(f"testartifact/{rfcs[1].name}.testartifact", BytesIO(b""))
+        self.assertTrue(rfcs_are_in_r2(rfc_number_list=rfc_number_list))
+
+
+
diff --git a/ietf/doc/utils_r2.py b/ietf/doc/utils_r2.py
new file mode 100644
index 0000000000..53fb978303
--- /dev/null
+++ b/ietf/doc/utils_r2.py
@@ -0,0 +1,17 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+from django.core.files.storage import storages
+
+from ietf.doc.models import StoredObject
+
+
+def rfcs_are_in_r2(rfc_number_list=()):
+    r2_rfc_bucket = storages["r2-rfc"]
+    for rfc_number in rfc_number_list:
+        stored_objects = StoredObject.objects.filter(
+            store="rfc", doc_name=f"rfc{rfc_number}"
+        )
+        for stored_object in stored_objects:
+            if not r2_rfc_bucket.exists(stored_object.name):
+                return False
+    return True
diff --git a/ietf/doc/utils_red.py b/ietf/doc/utils_red.py
new file mode 100644
index 0000000000..bcda893dca
--- /dev/null
+++ b/ietf/doc/utils_red.py
@@ -0,0 +1,31 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+import requests
+
+from django.conf import settings
+
+from ietf.utils.log import log
+
+
+def trigger_red_precomputer(rfc_number_list=()):
+    url = getattr(settings, "TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None)
+    if url is not None:
+        payload = {
+            "rfcs": ",".join([str(n) for n in rfc_number_list]),
+        }
+        try:
+            log(f"Triggering red precompute multiple for RFCs {rfc_number_list}")
+            response = requests.post(
+                url=url,
+                json=payload,
+                timeout=settings.DEFAULT_REQUESTS_TIMEOUT,
+            )
+        except requests.Timeout as e:
+            log(f"POST request timed out for {url} : {e}")
+            return
+        if response.status_code != 200:
+            log(
+                f"POST request failed for {url} : status_code={response.status_code}"
+            )
+    else:
+        log("No URL configured for triggering red precompute multiple, skipping")
diff --git a/ietf/settings_test.py b/ietf/settings_test.py
index 1f5a7e8ddc..e7ebc13eb2 100755
--- a/ietf/settings_test.py
+++ b/ietf/settings_test.py
@@ -115,8 +115,12 @@ def tempdir_with_cleanup(**kwargs):
 except NameError:
     pass
 
-# Use InMemoryStorage for red bucket storage
+# Use InMemoryStorage for red bucket and r2-rfc storages
 STORAGES["red_bucket"] = {
     "BACKEND": "django.core.files.storage.InMemoryStorage",
     "OPTIONS": {"location": "red_bucket"},
 }
+STORAGES["r2-rfc"] = {
+    "BACKEND": "django.core.files.storage.InMemoryStorage",
+    "OPTIONS": {"location": "r2-rfc"},
+}
diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 8c0c66cdf2..323b7fd45a 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -80,6 +80,16 @@ def _multiline_to_list(s):
 else:
     raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set")
 
+_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None)
+if _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY is not None:
+    RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY 
+_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", None)
+if _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES is not None:
+    RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES
+_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = os.environ.get("DATATRACKER_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None)
+if _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL is not None:
+    TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL
+
 # Set DEBUG if DATATRACKER_DEBUG env var is the word "true"
 DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true"
 

From e5b037ba83c2275efcd5a034c4bd1af67932d23f Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 23 Mar 2026 13:19:54 -0500
Subject: [PATCH 596/656] fix: rebuild reference relations once we have rfc
 contents (#10578)

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
---
 ietf/api/tests_views_rpc.py | 13 ++++++++++++-
 ietf/api/views_rpc.py       |  4 ++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py
index 6d10bee8e8..0db67e126f 100644
--- a/ietf/api/tests_views_rpc.py
+++ b/ietf/api/tests_views_rpc.py
@@ -196,9 +196,15 @@ def test_notify_rfc_published(self, mock_task_delay):
         self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list)
 
     @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
+    @mock.patch("ietf.api.views_rpc.rebuild_reference_relations_task")
     @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task")
     @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task")
-    def test_upload_rfc_files(self, mock_trigger_red_task, mock_update_searchindex_task):
+    def test_upload_rfc_files(
+        self,
+        mock_trigger_red_task,
+        mock_update_searchindex_task,
+        mock_rebuild_relations,
+    ):
         def _valid_post_data():
             """Generate a valid post data dict
 
@@ -370,6 +376,11 @@ def _valid_post_data():
             self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list)
             # Confirm that the search index update task was called correctly
             self.assertTrue(mock_update_searchindex_task.delay.called)
+            # Confirm reference relations rebuild task was called correctly
+            self.assertTrue(mock_rebuild_relations.delay.called)
+            _, mock_kwargs = mock_rebuild_relations.delay.call_args
+            self.assertIn("doc_names", mock_kwargs)
+            self.assertEqual(mock_kwargs["doc_names"], [rfc.name])
 
             # re-post with replace = False should now fail
             mock_update_searchindex_task.reset_mock()
diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index 59eed1e10e..6c7464e252 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -40,6 +40,7 @@
 from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage
 from ietf.doc.tasks import (
     signal_update_rfc_metadata_task,
+    rebuild_reference_relations_task,
     trigger_red_precomputer_task,
     update_rfc_searchindex_task,
 )
@@ -527,6 +528,9 @@ def post(self, request):
         trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating))
         # Trigger search index update
         update_rfc_searchindex_task.delay(rfc.rfc_number)
+        # Trigger reference relation srebuild
+        rebuild_reference_relations_task.delay(doc_names=[rfc.name])
+
         return Response(NotificationAckSerializer().data)
 
 

From 10ebdf9a6433b34d32352b4bb1b4e9b285773de8 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 23 Mar 2026 15:44:22 -0300
Subject: [PATCH 597/656] chore: deduplicate logging and clean up config
 (#10592)

* fix: remove redundant loggers + cleanup

* style: ruff ruff (logging config)

* chore: alphabetize loggers

* refactor: modern suppression of DisallowedHost log

* style: minor cleanup / comments

* fix: roll back accidental commit

* fix: django.request at ERROR level

* fix: squelch other SuspiciousOperation mail
---
 ietf/settings.py | 188 +++++++++++++++++++----------------------------
 1 file changed, 74 insertions(+), 114 deletions(-)

diff --git a/ietf/settings.py b/ietf/settings.py
index e0b4f20118..40a4cb5c56 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2007-2025, All Rights Reserved
+# Copyright The IETF Trust 2007-2026, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -13,6 +13,7 @@
 import warnings
 from hashlib import sha384
 from typing import Any, Dict, List, Tuple # pyflakes:ignore
+from django.http import UnreadablePostError
 
 # DeprecationWarnings are suppressed by default, enable them
 warnings.simplefilter("always", DeprecationWarning)
@@ -236,153 +237,112 @@
 
 FILE_UPLOAD_PERMISSIONS = 0o644          
 
-# ------------------------------------------------------------------------
-# Django/Python Logging Framework Modifications
 
-# Filter out "Invalid HTTP_HOST" emails
-# Based on http://www.tiwoc.de/blog/2013/03/django-prevent-email-notification-on-suspiciousoperation/
-from django.core.exceptions import SuspiciousOperation
-def skip_suspicious_operations(record):
-    if record.exc_info:
-        exc_value = record.exc_info[1]
-        if isinstance(exc_value, SuspiciousOperation):
-            return False
-    return True
+#
+# Logging config
+#
 
-# Filter out UreadablePostError:
-from django.http import UnreadablePostError
+# Callback to filter out UnreadablePostError:
 def skip_unreadable_post(record):
     if record.exc_info:
-        exc_type, exc_value = record.exc_info[:2] # pylint: disable=unused-variable
+        exc_type, exc_value = record.exc_info[:2]  # pylint: disable=unused-variable
         if isinstance(exc_value, UnreadablePostError):
             return False
     return True
 
-# Copied from DEFAULT_LOGGING as of Django 1.10.5 on 22 Feb 2017, and modified
-# to incorporate html logging, invalid http_host filtering, and more.
-# Changes from the default has comments.
-
-# The Python logging flow is as follows:
-# (see https://docs.python.org/2.7/howto/logging.html#logging-flow)
-#
-#   Init: get a Logger: logger = logging.getLogger(name)
-#
-#   Logging call, e.g. logger.error(level, msg, *args, exc_info=(...), extra={...})
-#   --> Logger (discard if level too low for this logger)
-#       (create log record from level, msg, args, exc_info, extra)
-#       --> Filters (discard if any filter attach to logger rejects record)
-#           --> Handlers (discard if level too low for handler)
-#               --> Filters (discard if any filter attached to handler rejects record)
-#                   --> Formatter (format log record and emit)
-#
-
 LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    #
-    'loggers': {
-        'django': {
-            'handlers': ['console', 'mail_admins'],
-            'level': 'INFO',
-        },
-        'django.request': {
-            'handlers': ['console'],
-            'level': 'ERROR',
+    "version": 1,
+    "disable_existing_loggers": False,
+    "loggers": {
+        "celery": {
+            "handlers": ["console"],
+            "level": "INFO",
         },
-        'django.server': {
-            'handlers': ['django.server'],
-            'level': 'INFO',
+        "datatracker": {
+            "handlers": ["console"],
+            "level": "INFO",
         },
-        'django.security': {
-            'handlers': ['console', ],
-            'level': 'INFO',
+        "django": {
+            "handlers": ["console", "mail_admins"],
+            "level": "INFO",
         },
-        'oidc_provider': {
-            'handlers': ['console', ],
-            'level': 'DEBUG',
+        "django.request": {"level": "ERROR"},  # only log 5xx, ignore 4xx
+        "django.security": {
+            # SuspiciousOperation errors - log to console only
+            "handlers": ["console"],
+            "propagate": False,  # no further handling please
         },
-        'datatracker': {
-            'handlers': ['console'],
-            'level': 'INFO',
+        "django.server": {
+            # Only used by Django's runserver development server
+            "handlers": ["django.server"],
+            "level": "INFO",
         },
-        'celery': {
-            'handlers': ['console'],
-            'level': 'INFO',
+        "oidc_provider": {
+            "handlers": ["console"],
+            "level": "DEBUG",
         },
     },
-    #
-    # No logger filters
-    #
-    'handlers': {
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-            'formatter': 'plain',
+    "handlers": {
+        "console": {
+            "level": "DEBUG",
+            "class": "logging.StreamHandler",
+            "formatter": "plain",
         },
-        'debug_console': {
-            # Active only when DEBUG=True
-            'level': 'DEBUG',
-            'filters': ['require_debug_true'],
-            'class': 'logging.StreamHandler',
-            'formatter': 'plain',
+        "debug_console": {
+            "level": "DEBUG",
+            "filters": ["require_debug_true"],
+            "class": "logging.StreamHandler",
+            "formatter": "plain",
         },
-        'django.server': {
-            'level': 'INFO',
-            'class': 'logging.StreamHandler',
-            'formatter': 'django.server',
+        "django.server": {
+            "level": "INFO",
+            "class": "logging.StreamHandler",
+            "formatter": "django.server",
         },
-        'mail_admins': {
-            'level': 'ERROR',
-            'filters': [
-                'require_debug_false',
-                'skip_suspicious_operations', # custom
-                'skip_unreadable_posts', # custom
+        "mail_admins": {
+            "level": "ERROR",
+            "filters": [
+                "require_debug_false",
+                "skip_unreadable_posts",
             ],
-            'class': 'django.utils.log.AdminEmailHandler',
-            'include_html': True,       # non-default
-        }
+            "class": "django.utils.log.AdminEmailHandler",
+            "include_html": True,
+        },
     },
-    #
     # All these are used by handlers
-    'filters': {
-        'require_debug_false': {
-            '()': 'django.utils.log.RequireDebugFalse',
-        },
-        'require_debug_true': {
-            '()': 'django.utils.log.RequireDebugTrue',
+    "filters": {
+        "require_debug_false": {
+            "()": "django.utils.log.RequireDebugFalse",
         },
-        # custom filter, function defined above:
-        'skip_suspicious_operations': {
-            '()': 'django.utils.log.CallbackFilter',
-            'callback': skip_suspicious_operations,
+        "require_debug_true": {
+            "()": "django.utils.log.RequireDebugTrue",
         },
         # custom filter, function defined above:
-        'skip_unreadable_posts': {
-            '()': 'django.utils.log.CallbackFilter',
-            'callback': skip_unreadable_post,
+        "skip_unreadable_posts": {
+            "()": "django.utils.log.CallbackFilter",
+            "callback": skip_unreadable_post,
         },
     },
-    # And finally the formatters
-    'formatters': {
-        'django.server': {
-            '()': 'django.utils.log.ServerFormatter',
-            'format': '[%(server_time)s] %(message)s',
+    "formatters": {
+        "django.server": {
+            "()": "django.utils.log.ServerFormatter",
+            "format": "[%(server_time)s] %(message)s",
         },
-        'plain': {
-            'style': '{',
-            'format': '{levelname}: {name}:{lineno}: {message}',
+        "plain": {
+            "style": "{",
+            "format": "{levelname}: {name}:{lineno}: {message}",
         },
-        'json' : {
+        "json": {
             "class": "ietf.utils.jsonlogger.DatatrackerJsonFormatter",
             "style": "{",
-            "format": "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}{process}",
-        }
+            "format": (
+                "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}"
+                "{process}{status_code}"
+            ),
+        },
     },
 }
 
-# End logging
-# ------------------------------------------------------------------------
-
 
 X_FRAME_OPTIONS = 'SAMEORIGIN'
 CSRF_TRUSTED_ORIGINS = [

From 14dd4cfdacb49552a2fcb9d9525e713f8ebd3c26 Mon Sep 17 00:00:00 2001
From: Tianyi Gao <tianyi.gao@ed.ac.uk>
Date: Tue, 24 Mar 2026 02:45:24 +0800
Subject: [PATCH 598/656] feat: show parents on list of teams with grouping
 (#8635) (#10552)

* feat: show parents on list of teams with grouping (#8635)

* fix: sort teams by parent type then parent name in active teams list
---
 ietf/group/views.py                    | 13 +++++++--
 ietf/templates/group/active_teams.html | 38 ++++++++++++++++----------
 2 files changed, 34 insertions(+), 17 deletions(-)

diff --git a/ietf/group/views.py b/ietf/group/views.py
index efe3eca15d..8561a5059f 100644
--- a/ietf/group/views.py
+++ b/ietf/group/views.py
@@ -245,10 +245,19 @@ def active_review_dirs(request):
     return render(request, 'group/active_review_dirs.html', {'dirs' : dirs })
 
 def active_teams(request):
-    teams = Group.objects.filter(type="team", state="active").order_by("name")
+    parent_type_order = {"area": 1, "adm": 3, None: 4}
+
+    def team_sort_key(group):
+        type_id = group.parent.type_id if group.parent else None
+        return (parent_type_order.get(type_id, 2), group.parent.name if group.parent else "", group.name)
+
+    teams = sorted(
+        Group.objects.filter(type="team", state="active").select_related("parent"),
+        key=team_sort_key,
+    )
     for group in teams:
         group.chairs = sorted(roles(group, "chair"), key=extract_last_name)
-    return render(request, 'group/active_teams.html', {'teams' : teams })
+    return render(request, 'group/active_teams.html', {'teams': teams})
 
 def active_iab(request):
     iabgroups = Group.objects.filter(type__in=("program","iabasg","iabworkshop"), state="active").order_by("-type_id","name")
diff --git a/ietf/templates/group/active_teams.html b/ietf/templates/group/active_teams.html
index 502d971a20..771dfda290 100644
--- a/ietf/templates/group/active_teams.html
+++ b/ietf/templates/group/active_teams.html
@@ -16,21 +16,29 @@ <h1>Active teams</h1>
                 <th scope="col" data-sort="chairs">Chairs</th>
             </tr>
         </thead>
-        <tbody>
-            {% for group in teams %}
-                <tr>
-                    <td>
-                        <a href="{% url "ietf.group.views.group_home" acronym=group.acronym %}">{{ group.acronym }}</a>
-                    </td>
-                    <td>{{ group.name }}</td>
-                    <td>
-                        {% for chair in group.chairs %}
-                            {% person_link chair.person %}{% if not forloop.last %},{% endif %}
-                        {% endfor %}
-                    </td>
-                </tr>
-            {% endfor %}
-        </tbody>
+        {% regroup teams by parent as grouped_teams %}
+        {% for group_entry in grouped_teams %}
+            <tbody>
+                <tr class="table-info"><th scope="col" colspan="3">
+                    {% if group_entry.grouper %}{{ group_entry.grouper.name }}{% else %}Other{% endif %}
+                </th></tr>
+            </tbody>
+            <tbody>
+                {% for group in group_entry.list %}
+                    <tr>
+                        <td>
+                            <a href="{% url "ietf.group.views.group_home" acronym=group.acronym %}">{{ group.acronym }}</a>
+                        </td>
+                        <td>{{ group.name }}</td>
+                        <td>
+                            {% for chair in group.chairs %}
+                                {% person_link chair.person %}{% if not forloop.last %},{% endif %}
+                            {% endfor %}
+                        </td>
+                    </tr>
+                {% endfor %}
+            </tbody>
+        {% endfor %}
     </table>
 {% endblock %}
 {% block js %}

From 057d52b76666ab6fcfd366700862be10db4844bb Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 23 Mar 2026 15:56:39 -0300
Subject: [PATCH 599/656] ci: update actions to avoid deprecations (#10604)

* ci: upload-artifact -> v7

* ci: checkout -> v6
---
 .github/workflows/build-base-app.yml          |  2 +-
 .github/workflows/build-devblobstore.yml      |  2 +-
 .github/workflows/build-mq-broker.yml         |  2 +-
 .github/workflows/build.yml                   |  8 ++++----
 .github/workflows/ci-run-tests.yml            |  2 +-
 .github/workflows/codeql-analysis.yml         |  2 +-
 .github/workflows/dependency-review.yml       |  2 +-
 .github/workflows/dev-assets-sync-nightly.yml |  2 +-
 .github/workflows/tests.yml                   | 14 +++++++-------
 9 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index 4a4394fca0..2b937cbfef 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -18,7 +18,7 @@ jobs:
       packages: write
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         token: ${{ secrets.GH_COMMON_TOKEN }}
 
diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml
index f49a11af19..41b2e0d47a 100644
--- a/.github/workflows/build-devblobstore.yml
+++ b/.github/workflows/build-devblobstore.yml
@@ -20,7 +20,7 @@ jobs:
       packages: write
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml
index 4de861dbcd..76c9b93168 100644
--- a/.github/workflows/build-mq-broker.yml
+++ b/.github/workflows/build-mq-broker.yml
@@ -24,7 +24,7 @@ jobs:
       packages: write
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d97889fbb8..8872c7f7d3 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -66,7 +66,7 @@ jobs:
       base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }}
 
     steps:   
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 1
         fetch-tags: false
@@ -164,7 +164,7 @@ jobs:
       TARGET_BASE: ${{needs.prepare.outputs.base_image_version}}
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 1
         fetch-tags: false
@@ -341,7 +341,7 @@ jobs:
         token: ${{ secrets.GITHUB_TOKEN }}
         
     - name: Upload Build Artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: release-${{ env.PKG_VERSION }}
         path: /home/runner/work/release/release.tar.gz
@@ -403,7 +403,7 @@ jobs:
       PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
       
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           ref: main
           
diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml
index 278bd8af2f..5349f1ac7a 100644
--- a/.github/workflows/ci-run-tests.yml
+++ b/.github/workflows/ci-run-tests.yml
@@ -23,7 +23,7 @@ jobs:
       base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }}
 
     steps:   
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 1
         fetch-tags: false
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 3444c03b5e..4ab32d27a6 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -26,7 +26,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v3
diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
index 6d0683c471..e255b270ff 100644
--- a/.github/workflows/dependency-review.yml
+++ b/.github/workflows/dependency-review.yml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: 'Dependency Review'
         uses: actions/dependency-review-action@v4
         with:
diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml
index 4cfbf6365b..926d816b38 100644
--- a/.github/workflows/dev-assets-sync-nightly.yml
+++ b/.github/workflows/dev-assets-sync-nightly.yml
@@ -29,7 +29,7 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 836314bac0..be7b834b7a 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -32,7 +32,7 @@ jobs:
         image: ghcr.io/ietf-tools/datatracker-devblobstore:latest
     
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     
     - name: Prepare for tests
       run: |
@@ -68,7 +68,7 @@ jobs:
         coverage xml
 
     - name: Upload geckodriver.log
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       if: ${{ failure() }}
       with:
         name: geckodriverlog
@@ -87,7 +87,7 @@ jobs:
         mv latest-coverage.json coverage.json
         
     - name: Upload Coverage Results as Build Artifact
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       if: ${{ always() }}
       with:
         name: coverage
@@ -102,7 +102,7 @@ jobs:
         project: [chromium, firefox]
     
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - uses: actions/setup-node@v6
       with:
@@ -121,7 +121,7 @@ jobs:
         npx playwright test --project=${{ matrix.project }}
         
     - name: Upload Report
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       if: ${{ always() }}
       continue-on-error: true
       with:
@@ -143,7 +143,7 @@ jobs:
         image: ghcr.io/ietf-tools/datatracker-db:latest
     
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     
     - name: Prepare for tests
       run: |
@@ -180,7 +180,7 @@ jobs:
         npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js
         
     - name: Upload Report
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       if: ${{ always() }}
       continue-on-error: true
       with:

From 02070ee2f4dc6ee599e08a87e92b345198ae40fa Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Mar 2026 16:04:08 -0300
Subject: [PATCH 600/656] chore(deps): bump actions/setup-python from 5 to 6
 (#9480)

Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6.
- [Release notes](https://github.com/actions/setup-python/releases)
- [Commits](https://github.com/actions/setup-python/compare/v5...v6)

---
updated-dependencies:
- dependency-name: actions/setup-python
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8872c7f7d3..07a304cac2 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -175,7 +175,7 @@ jobs:
         node-version: 18.x
 
     - name: Setup Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: "3.x"
 

From 4a6627826993863bcb9cd5ede6b6e6f5b19eb0e9 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Mar 2026 17:07:28 -0300
Subject: [PATCH 601/656] chore(deps): bump ncipollo/release-action from 1.18.0
 to 1.20.0 (#9478)

Bumps [ncipollo/release-action](https://github.com/ncipollo/release-action) from 1.18.0 to 1.20.0.
- [Release notes](https://github.com/ncipollo/release-action/releases)
- [Commits](https://github.com/ncipollo/release-action/compare/v1.18.0...v1.20.0)

---
updated-dependencies:
- dependency-name: ncipollo/release-action
  dependency-version: 1.20.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 07a304cac2..ed425f9ae5 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -98,7 +98,7 @@ jobs:
         echo "IS_RELEASE=true" >> $GITHUB_ENV
         
     - name: Create Draft Release
-      uses: ncipollo/release-action@v1.18.0
+      uses: ncipollo/release-action@v1.21.0
       if: ${{ github.ref_name == 'release' }}
       with:
         prerelease: true
@@ -315,7 +315,7 @@ jobs:
         histCoveragePath: historical-coverage.json
         
     - name: Create Release
-      uses: ncipollo/release-action@v1.18.0
+      uses: ncipollo/release-action@v1.21.0
       if: ${{ env.SHOULD_DEPLOY == 'true' }}
       with:
         allowUpdates: true
@@ -328,7 +328,7 @@ jobs:
         token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Update Baseline Coverage
-      uses: ncipollo/release-action@v1.18.0
+      uses: ncipollo/release-action@v1.21.0
       if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }}
       with:
         allowUpdates: true

From 753bd507c5d9cfdad4793d0e3feed68726fecf1e Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 23 Mar 2026 15:38:15 -0500
Subject: [PATCH 602/656] fix: include editorial docs in sent-to-rpc (#10605)

---
 ietf/api/views_rpc.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index 6c7464e252..1e96118e58 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -210,19 +210,19 @@ def submitted_to_rpc(self, request):
         Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue.
         """
         ietf_docs = Q(states__type_id="draft-iesg", states__slug__in=["ann"])
-        irtf_iab_ise_docs = Q(
+        irtf_iab_ise_editorial_docs = Q(
             states__type_id__in=[
                 "draft-stream-iab",
                 "draft-stream-irtf",
                 "draft-stream-ise",
+                "draft-stream-editorial",
             ],
             states__slug__in=["rfc-edit"],
         )
-        # TODO: Need a way to talk about editorial stream docs
         docs = (
             self.get_queryset()
             .filter(type_id="draft")
-            .filter(ietf_docs | irtf_iab_ise_docs)
+            .filter(ietf_docs | irtf_iab_ise_editorial_docs)
         )
         serializer = self.get_serializer(docs, many=True)
         return Response(serializer.data)

From 4308162174bb565b988c4fca9289c424c736ecba Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 23 Mar 2026 17:38:34 -0300
Subject: [PATCH 603/656] ci: handle rabbitmq version for push trigger (#10606)

---
 .github/workflows/build-mq-broker.yml | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml
index 76c9b93168..50472122c4 100644
--- a/.github/workflows/build-mq-broker.yml
+++ b/.github/workflows/build-mq-broker.yml
@@ -39,6 +39,15 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
+    - name: Set rabbitmq version
+      id: rabbitmq-version
+      run: |
+        if [[ "${{ inputs.rabbitmq_version }}" == "" ]]; then
+          echo "RABBITMQ_VERSION=3.13-alpine" >> $GITHUB_OUTPUT
+        else
+          echo "RABBITMQ_VERSION=${{ inputs.rabbitmq_version }}" >> $GITHUB_OUTPUT
+        fi
+
     - name: Docker Build & Push
       uses: docker/build-push-action@v6
       env:
@@ -48,7 +57,7 @@ jobs:
         file: dev/mq/Dockerfile
         platforms: linux/amd64,linux/arm64
         push: true
-        build-args: RABBITMQ_VERSION=${{ inputs.rabbitmq_version }}
+        build-args: RABBITMQ_VERSION=${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }}
         tags: |
-          ghcr.io/ietf-tools/datatracker-mq:${{ inputs.rabbitmq_version }}
+          ghcr.io/ietf-tools/datatracker-mq:${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }}
           ghcr.io/ietf-tools/datatracker-mq:latest

From eb041f7d81c7469f0b4c765150ccfadba5604177 Mon Sep 17 00:00:00 2001
From: Martin Thomson <mt@lowentropy.net>
Date: Tue, 24 Mar 2026 05:39:01 +0900
Subject: [PATCH 604/656] fix: Rewrite CSS style attributes in SVG (#10584)

This makes the dark mode work properly for drafts like
https://datatracker.ietf.org/doc/html/draft-hajdusek-qirg-timing-physics-01
which have diagrams that use a mix of ordinary attributes and the style
attribute.  Using the style attribute makes the rules there invisible to
the method we use for the remapping of black and white for dark mode.
---
 ietf/static/js/document_html.js | 79 +++++++++++++++++++++++++++++++++
 1 file changed, 79 insertions(+)

diff --git a/ietf/static/js/document_html.js b/ietf/static/js/document_html.js
index 6e8861739a..3e609f3965 100644
--- a/ietf/static/js/document_html.js
+++ b/ietf/static/js/document_html.js
@@ -117,4 +117,83 @@ document.addEventListener("DOMContentLoaded", function (event) {
                 }
             });
     }
+
+    // Rewrite these CSS properties so that the values are available for restyling.
+    document.querySelectorAll("svg [style]").forEach(el => {
+        // Push these CSS properties into their own attributes
+        const SVG_PRESENTATION_ATTRS = new Set([
+            'alignment-baseline', 'baseline-shift', 'clip', 'clip-path', 'clip-rule',
+            'color', 'color-interpolation', 'color-interpolation-filters',
+            'color-rendering', 'cursor', 'direction', 'display', 'dominant-baseline',
+            'fill', 'fill-opacity', 'fill-rule', 'filter', 'flood-color',
+            'flood-opacity', 'font-family', 'font-size', 'font-size-adjust',
+            'font-stretch', 'font-style', 'font-variant', 'font-weight',
+            'image-rendering', 'letter-spacing', 'lighting-color', 'marker-end',
+            'marker-mid', 'marker-start', 'mask', 'opacity', 'overflow', 'paint-order',
+            'pointer-events', 'shape-rendering', 'stop-color', 'stop-opacity',
+            'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap',
+            'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', 'stroke-width',
+            'text-anchor', 'text-decoration', 'text-rendering', 'unicode-bidi',
+            'vector-effect', 'visibility', 'word-spacing', 'writing-mode',
+        ]);
+
+        // Simple CSS splitter: respects quoted strings and parens so semicolons
+        // inside url(...) or "..." don't get treated as declaration boundaries.
+        function parseDeclarations(styleText) {
+            const decls = [];
+            let buf = '';
+            let inStr = false;
+            let strChar = '';
+            let escaped = false;
+            let depth = 0;
+
+            for (const ch of styleText) {
+                if (inStr) {
+                    if (escaped) {
+                        escaped = false;
+                    } else if (ch === '\\') {
+                        escaped = true;
+                    } else if (ch === strChar) {
+                        inStr = false;
+                    }
+                } else if (ch === '"' || ch === "'") {
+                    inStr = true;
+                    strChar = ch;
+                } else if (ch === '(') {
+                    depth++;
+                } else if (ch === ')') {
+                    depth--;
+                } else if (ch === ';' && depth === 0) {
+                    const trimmed = buf.trim();
+                    if (trimmed) {
+                        decls.push(trimmed);
+                    }
+                    buf = '';
+                    continue;
+                }
+                buf += ch;
+            }
+            const trimmed = buf.trim();
+            if (trimmed) {
+                decls.push(trimmed);
+            }
+            return decls;
+        }
+
+        const remainder = [];
+        for (const decl of parseDeclarations(el.getAttribute('style'))) {
+            const [prop, val] = decl.split(":", 2).map(v => v.trim());
+            if (val && !/!important$/.test(val) && SVG_PRESENTATION_ATTRS.has(prop)) {
+                el.setAttribute(prop, val);
+            } else {
+                remainder.push(decl);
+            }
+        }
+
+        if (remainder.length > 0) {
+            el.setAttribute('style', remainder.join('; '));
+        } else {
+            el.removeAttribute('style');
+        }
+    });
 });

From 93e9bd3aad53808e791302c8fd99d74eb0873385 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Mar 2026 17:44:26 -0300
Subject: [PATCH 605/656] chore(deps): bump github/codeql-action from 3 to 4
 (#9956)

Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3 to 4.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/v3...v4)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-version: '4'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/codeql-analysis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 4ab32d27a6..bc20779ae6 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -29,9 +29,9 @@ jobs:
       uses: actions/checkout@v6
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v4

From f9aebd5aa881557d7493db415d04a3d89494d637 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Mar 2026 17:48:56 -0300
Subject: [PATCH 606/656] chore(deps): bump actions/download-artifact from
 4.3.0 to 6.0.0 (#9805)

Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 4.3.0 to 6.0.0.
- [Release notes](https://github.com/actions/download-artifact/releases)
- [Commits](https://github.com/actions/download-artifact/compare/v4.3.0...v6.0.0)

---
updated-dependencies:
- dependency-name: actions/download-artifact
  dependency-version: 6.0.0
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ed425f9ae5..74791747b6 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -186,7 +186,7 @@ jobs:
 
     - name: Download a Coverage Results
       if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
-      uses: actions/download-artifact@v4.3.0
+      uses: actions/download-artifact@v6.0.0
       with:
         name: coverage
 
@@ -291,7 +291,7 @@ jobs:
 
     - name: Download Coverage Results
       if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
-      uses: actions/download-artifact@v4.3.0
+      uses: actions/download-artifact@v6.0.0
       with:
         name: coverage
 

From 7d84aacad621b83753d6701afbcc864592d4822f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Mar 2026 17:50:26 -0300
Subject: [PATCH 607/656] chore(deps): bump the npm group (#10602)

Bumps the npm group in /dev/deploy-to-container with 2 updates: [dockerode](https://github.com/apocas/dockerode) and [tar](https://github.com/isaacs/node-tar).


Updates `dockerode` from 4.0.9 to 4.0.10
- [Release notes](https://github.com/apocas/dockerode/releases)
- [Commits](https://github.com/apocas/dockerode/compare/v4.0.9...v4.0.10)

Updates `tar` from 7.5.11 to 7.5.12
- [Release notes](https://github.com/isaacs/node-tar/releases)
- [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md)
- [Commits](https://github.com/isaacs/node-tar/compare/v7.5.11...v7.5.12)

---
updated-dependencies:
- dependency-name: dockerode
  dependency-version: 4.0.10
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
- dependency-name: tar
  dependency-version: 7.5.12
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/deploy-to-container/package-lock.json | 113 ++++++++++------------
 dev/deploy-to-container/package.json      |   4 +-
 2 files changed, 54 insertions(+), 63 deletions(-)

diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json
index b62109f0e2..a68f170c4b 100644
--- a/dev/deploy-to-container/package-lock.json
+++ b/dev/deploy-to-container/package-lock.json
@@ -6,12 +6,12 @@
     "": {
       "name": "deploy-to-container",
       "dependencies": {
-        "dockerode": "^4.0.9",
+        "dockerode": "^4.0.10",
         "fs-extra": "^11.3.4",
         "nanoid": "5.1.7",
         "nanoid-dictionary": "5.0.0",
         "slugify": "1.6.8",
-        "tar": "^7.5.11",
+        "tar": "^7.5.12",
         "yargs": "^17.7.2"
       },
       "engines": {
@@ -160,7 +160,6 @@
       "version": "0.2.6",
       "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz",
       "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==",
-      "license": "MIT",
       "dependencies": {
         "safer-buffer": "~2.1.0"
       }
@@ -188,7 +187,6 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
       "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==",
-      "license": "BSD-3-Clause",
       "dependencies": {
         "tweetnacl": "^0.14.3"
       }
@@ -227,9 +225,9 @@
       }
     },
     "node_modules/buildcheck": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz",
-      "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==",
+      "version": "0.0.7",
+      "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz",
+      "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==",
       "optional": true,
       "engines": {
         "node": ">=10.0.0"
@@ -284,10 +282,9 @@
       }
     },
     "node_modules/debug": {
-      "version": "4.4.0",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
-      "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
-      "license": "MIT",
+      "version": "4.4.3",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
+      "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
       "dependencies": {
         "ms": "^2.1.3"
       },
@@ -301,10 +298,9 @@
       }
     },
     "node_modules/docker-modem": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz",
-      "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==",
-      "license": "Apache-2.0",
+      "version": "5.0.7",
+      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz",
+      "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==",
       "dependencies": {
         "debug": "^4.1.1",
         "readable-stream": "^3.5.0",
@@ -316,14 +312,14 @@
       }
     },
     "node_modules/dockerode": {
-      "version": "4.0.9",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz",
-      "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==",
+      "version": "4.0.10",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.10.tgz",
+      "integrity": "sha512-8L/P9JynLBiG7/coiA4FlQXegHltRqS0a+KqI44P1zgQh8QLHTg7FKOwhkBgSJwZTeHsq30WRoVFLuwkfK0YFg==",
       "dependencies": {
         "@balena/dockerignore": "^1.0.2",
         "@grpc/grpc-js": "^1.11.1",
         "@grpc/proto-loader": "^0.7.13",
-        "docker-modem": "^5.0.6",
+        "docker-modem": "^5.0.7",
         "protobufjs": "^7.3.2",
         "tar-fs": "^2.1.4",
         "uuid": "^10.0.0"
@@ -464,14 +460,12 @@
     "node_modules/ms": {
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
-      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
-      "license": "MIT"
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
     "node_modules/nan": {
-      "version": "2.22.0",
-      "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz",
-      "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==",
-      "license": "MIT",
+      "version": "2.26.2",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz",
+      "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==",
       "optional": true
     },
     "node_modules/nanoid": {
@@ -580,8 +574,7 @@
     "node_modules/safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
-      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
-      "license": "MIT"
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
     "node_modules/slugify": {
       "version": "1.6.8",
@@ -594,13 +587,12 @@
     "node_modules/split-ca": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz",
-      "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==",
-      "license": "ISC"
+      "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ=="
     },
     "node_modules/ssh2": {
-      "version": "1.16.0",
-      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz",
-      "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==",
+      "version": "1.17.0",
+      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz",
+      "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==",
       "hasInstallScript": true,
       "dependencies": {
         "asn1": "^0.2.6",
@@ -611,7 +603,7 @@
       },
       "optionalDependencies": {
         "cpu-features": "~0.0.10",
-        "nan": "^2.20.0"
+        "nan": "^2.23.0"
       }
     },
     "node_modules/string_decoder": {
@@ -647,9 +639,9 @@
       }
     },
     "node_modules/tar": {
-      "version": "7.5.11",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz",
-      "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==",
+      "version": "7.5.12",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.12.tgz",
+      "integrity": "sha512-9TsuLcdhOn4XztcQqhNyq1KOwOOED/3k58JAvtULiYqbO8B/0IBAAIE1hj0Svmm58k27TmcigyDI0deMlgG3uw==",
       "dependencies": {
         "@isaacs/fs-minipass": "^4.0.0",
         "chownr": "^3.0.0",
@@ -698,8 +690,7 @@
     "node_modules/tweetnacl": {
       "version": "0.14.5",
       "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
-      "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==",
-      "license": "Unlicense"
+      "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA=="
     },
     "node_modules/undici-types": {
       "version": "6.20.0",
@@ -949,9 +940,9 @@
       }
     },
     "buildcheck": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz",
-      "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==",
+      "version": "0.0.7",
+      "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz",
+      "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==",
       "optional": true
     },
     "chownr": {
@@ -993,17 +984,17 @@
       }
     },
     "debug": {
-      "version": "4.4.0",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
-      "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
+      "version": "4.4.3",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
+      "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
       "requires": {
         "ms": "^2.1.3"
       }
     },
     "docker-modem": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz",
-      "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==",
+      "version": "5.0.7",
+      "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz",
+      "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==",
       "requires": {
         "debug": "^4.1.1",
         "readable-stream": "^3.5.0",
@@ -1012,14 +1003,14 @@
       }
     },
     "dockerode": {
-      "version": "4.0.9",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.9.tgz",
-      "integrity": "sha512-iND4mcOWhPaCNh54WmK/KoSb35AFqPAUWFMffTQcp52uQt36b5uNwEJTSXntJZBbeGad72Crbi/hvDIv6us/6Q==",
+      "version": "4.0.10",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.10.tgz",
+      "integrity": "sha512-8L/P9JynLBiG7/coiA4FlQXegHltRqS0a+KqI44P1zgQh8QLHTg7FKOwhkBgSJwZTeHsq30WRoVFLuwkfK0YFg==",
       "requires": {
         "@balena/dockerignore": "^1.0.2",
         "@grpc/grpc-js": "^1.11.1",
         "@grpc/proto-loader": "^0.7.13",
-        "docker-modem": "^5.0.6",
+        "docker-modem": "^5.0.7",
         "protobufjs": "^7.3.2",
         "tar-fs": "^2.1.4",
         "uuid": "^10.0.0"
@@ -1126,9 +1117,9 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
     "nan": {
-      "version": "2.22.0",
-      "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz",
-      "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==",
+      "version": "2.26.2",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz",
+      "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==",
       "optional": true
     },
     "nanoid": {
@@ -1213,14 +1204,14 @@
       "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ=="
     },
     "ssh2": {
-      "version": "1.16.0",
-      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz",
-      "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==",
+      "version": "1.17.0",
+      "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz",
+      "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==",
       "requires": {
         "asn1": "^0.2.6",
         "bcrypt-pbkdf": "^1.0.2",
         "cpu-features": "~0.0.10",
-        "nan": "^2.20.0"
+        "nan": "^2.23.0"
       }
     },
     "string_decoder": {
@@ -1250,9 +1241,9 @@
       }
     },
     "tar": {
-      "version": "7.5.11",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.11.tgz",
-      "integrity": "sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==",
+      "version": "7.5.12",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.12.tgz",
+      "integrity": "sha512-9TsuLcdhOn4XztcQqhNyq1KOwOOED/3k58JAvtULiYqbO8B/0IBAAIE1hj0Svmm58k27TmcigyDI0deMlgG3uw==",
       "requires": {
         "@isaacs/fs-minipass": "^4.0.0",
         "chownr": "^3.0.0",
diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json
index 1c95a4540c..aa9e82dbdf 100644
--- a/dev/deploy-to-container/package.json
+++ b/dev/deploy-to-container/package.json
@@ -2,12 +2,12 @@
   "name": "deploy-to-container",
   "type": "module",
   "dependencies": {
-    "dockerode": "^4.0.9",
+    "dockerode": "^4.0.10",
     "fs-extra": "^11.3.4",
     "nanoid": "5.1.7",
     "nanoid-dictionary": "5.0.0",
     "slugify": "1.6.8",
-    "tar": "^7.5.11",
+    "tar": "^7.5.12",
     "yargs": "^17.7.2"
   },
   "engines": {

From 3d00e594e6a667fe8084bc8548d6993c29d515e6 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 23 Mar 2026 19:50:19 -0300
Subject: [PATCH 608/656] chore(deps): bump more action versions (#10608)

---
 .github/workflows/build-base-app.yml          |  6 +++---
 .github/workflows/build-devblobstore.yml      |  6 +++---
 .github/workflows/build-mq-broker.yml         |  6 +++---
 .github/workflows/build.yml                   | 10 +++++-----
 .github/workflows/dev-assets-sync-nightly.yml |  4 ++--
 5 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index 2b937cbfef..1b0855cc47 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -31,17 +31,17 @@ jobs:
       uses: docker/setup-qemu-action@v3
     
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@v4
 
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v3
+      uses: docker/login-action@v4
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Docker Build & Push
-      uses: docker/build-push-action@v6
+      uses: docker/build-push-action@v7
       env:
         DOCKER_BUILD_SUMMARY: false
       with:
diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml
index 41b2e0d47a..14c4b1a135 100644
--- a/.github/workflows/build-devblobstore.yml
+++ b/.github/workflows/build-devblobstore.yml
@@ -23,17 +23,17 @@ jobs:
     - uses: actions/checkout@v6
     
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@v4
 
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v3
+      uses: docker/login-action@v4
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Docker Build & Push
-      uses: docker/build-push-action@v6
+      uses: docker/build-push-action@v7
       env:
         DOCKER_BUILD_SUMMARY: false
       with:
diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml
index 50472122c4..ef7ed2f65c 100644
--- a/.github/workflows/build-mq-broker.yml
+++ b/.github/workflows/build-mq-broker.yml
@@ -30,10 +30,10 @@ jobs:
       uses: docker/setup-qemu-action@v3
     
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@v4
 
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v3
+      uses: docker/login-action@v4
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
@@ -49,7 +49,7 @@ jobs:
         fi
 
     - name: Docker Build & Push
-      uses: docker/build-push-action@v6
+      uses: docker/build-push-action@v7
       env:
         DOCKER_BUILD_SUMMARY: false
       with:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 74791747b6..8ec806b229 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -253,10 +253,10 @@ jobs:
         EOL
 
     - name: Setup Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      uses: docker/setup-buildx-action@v4
 
     - name: Login to GitHub Container Registry
-      uses: docker/login-action@v3
+      uses: docker/login-action@v4
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
@@ -267,7 +267,7 @@ jobs:
       run: echo "FEATURE_LATEST_TAG=$(echo $GITHUB_REF_NAME | tr / -)" >> $GITHUB_ENV
 
     - name: Build Images
-      uses: docker/build-push-action@v6
+      uses: docker/build-push-action@v7
       env:
         DOCKER_BUILD_SUMMARY: false
       with:
@@ -360,7 +360,7 @@ jobs:
     steps:      
     - name: Notify on Slack (Success)
       if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
-      uses: slackapi/slack-github-action@v2
+      uses: slackapi/slack-github-action@v3
       with:
         token: ${{ secrets.SLACK_GH_BOT }}
         method: chat.postMessage
@@ -375,7 +375,7 @@ jobs:
                   value: "Completed"
     - name: Notify on Slack (Failure)
       if: ${{ contains(join(needs.*.result, ','), 'failure') }}
-      uses: slackapi/slack-github-action@v2
+      uses: slackapi/slack-github-action@v3
       with:
         token: ${{ secrets.SLACK_GH_BOT }}
         method: chat.postMessage
diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml
index 926d816b38..cd986f06f3 100644
--- a/.github/workflows/dev-assets-sync-nightly.yml
+++ b/.github/workflows/dev-assets-sync-nightly.yml
@@ -32,14 +32,14 @@ jobs:
       - uses: actions/checkout@v6
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Docker Build & Push
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         env:
           DOCKER_BUILD_SUMMARY: false
         with:

From e51469a5d437491071610156d56dcb73191ad61c Mon Sep 17 00:00:00 2001
From: Rudi Matz <rudi@staff.ietf.org>
Date: Fri, 27 Mar 2026 13:44:35 -0400
Subject: [PATCH 609/656] feat: add email/name for ADs and WG Chairs

---
 ietf/api/serializers_rpc.py | 28 +++++++++++++++++++++++++++-
 ietf/group/serializers.py   |  6 ++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py
index 397ca05d9b..d888de4586 100644
--- a/ietf/api/serializers_rpc.py
+++ b/ietf/api/serializers_rpc.py
@@ -27,7 +27,7 @@
     update_action_holders,
     update_rfcauthors,
 )
-from ietf.group.models import Group
+from ietf.group.models import Group, Role
 from ietf.group.serializers import AreaSerializer
 from ietf.name.models import StreamName, StdLevelName
 from ietf.person.models import Person
@@ -97,6 +97,21 @@ class Meta:
         fields = ["draft_name", "authors"]
 
 
+class WgChairSerializer(serializers.Serializer):
+    """Serialize a WG chair's name and email from a Role"""
+
+    name = serializers.SerializerMethodField()
+    email = serializers.SerializerMethodField()
+
+    @extend_schema_field(serializers.CharField)
+    def get_name(self, role: Role) -> str:
+        return role.person.plain_name()
+
+    @extend_schema_field(serializers.EmailField)
+    def get_email(self, role: Role) -> str:
+        return role.email.email_address()
+
+
 class DocumentAuthorSerializer(serializers.ModelSerializer):
     """Serializer for a Person in a response"""
 
@@ -126,6 +141,7 @@ class FullDraftSerializer(serializers.ModelSerializer):
         source="shepherd.person", read_only=True
     )
     consensus = serializers.SerializerMethodField()
+    wg_chairs = serializers.SerializerMethodField()
 
     class Meta:
         model = Document
@@ -145,11 +161,21 @@ class Meta:
             "consensus",
             "shepherd",
             "ad",
+            "wg_chairs",
         ]
 
     def get_consensus(self, doc: Document) -> Optional[bool]:
         return default_consensus(doc)
 
+    @extend_schema_field(WgChairSerializer(many=True))
+    def get_wg_chairs(self, doc: Document):
+        if doc.group is None:
+            return []
+        chairs = doc.group.role_set.filter(name_id="chair").select_related(
+            "person", "email"
+        )
+        return WgChairSerializer(chairs, many=True).data
+
     def get_source_format(
         self, doc: Document
     ) -> Literal["unknown", "xml-v2", "xml-v3", "txt"]:
diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py
index db3b37af48..e789ba46bf 100644
--- a/ietf/group/serializers.py
+++ b/ietf/group/serializers.py
@@ -20,8 +20,14 @@ class AreaDirectorSerializer(serializers.Serializer):
     Works with Email or Role
     """
 
+    name = serializers.SerializerMethodField()
     email = serializers.SerializerMethodField()
 
+    @extend_schema_field(serializers.CharField)
+    def get_name(self, instance: Email | Role):
+        person = getattr(instance, 'person', None)
+        return person.plain_name() if person else None
+
     @extend_schema_field(serializers.EmailField)
     def get_email(self, instance: Email | Role):
         if isinstance(instance, Role):

From b1cc7edc7ff5e80f7eb0072657e88450a4b2c06b Mon Sep 17 00:00:00 2001
From: Rudi Matz <rudi@staff.ietf.org>
Date: Fri, 27 Mar 2026 14:32:07 -0400
Subject: [PATCH 610/656] adapt test

---
 ietf/group/tests_serializers.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/ietf/group/tests_serializers.py b/ietf/group/tests_serializers.py
index bf29e6c8fd..b584a17ae2 100644
--- a/ietf/group/tests_serializers.py
+++ b/ietf/group/tests_serializers.py
@@ -31,7 +31,7 @@ def test_serializes_role(self):
         serialized = AreaDirectorSerializer(role).data
         self.assertEqual(
             serialized,
-            {"email": role.email.email_address()},
+            {"email": role.email.email_address(), "name": role.person.plain_name()},
         )
 
     def test_serializes_email(self):
@@ -40,7 +40,10 @@ def test_serializes_email(self):
         serialized = AreaDirectorSerializer(email).data
         self.assertEqual(
             serialized,
-            {"email": email.email_address()},
+            {
+                "email": email.email_address(),
+                "name": email.person.plain_name() if email.person else None,
+            },
         )
 
 
@@ -63,7 +66,10 @@ def test_serializes_active_area(self):
         self.assertEqual(serialized["name"], area.name)
         self.assertCountEqual(
             serialized["ads"],
-            [{"email": ad.email.email_address()} for ad in ad_roles],
+            [
+                {"email": ad.email.email_address(), "name": ad.person.plain_name()}
+                for ad in ad_roles
+            ],
         )
 
     def test_serializes_inactive_area(self):

From 5775077317640de8981cf27b0b8c54e42d8ae9a2 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 1 Apr 2026 18:53:13 -0300
Subject: [PATCH 611/656] fix: limit access to manual post cancellation
 (#10638)

* fix: drop access_token from URL

* test: update test case

* test: remove unneeded test

There is no longer a dedicated manual post cancel action

* chore: update copyrights
---
 ietf/submit/tests.py                   | 30 +++++++++++++++-----------
 ietf/templates/submit/manual_post.html | 16 ++++----------
 2 files changed, 21 insertions(+), 25 deletions(-)

diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index 400d0d8c7d..ad361d31b2 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2011-2023, All Rights Reserved
+# Copyright The IETF Trust 2011-2026, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
@@ -207,20 +207,24 @@ def test_manualpost_view(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertIn(
-            urlreverse(
-                "ietf.submit.views.submission_status", 
-                kwargs=dict(submission_id=submission.pk)
-            ),
-            q("#manual.submissions td a").attr("href")
-        )
-        self.assertIn(
-            submission.name,
-            q("#manual.submissions td a").text()
+        # Validate that the basic submission status URL is on the manual post page
+        # _without_ an access token, even if logged in as various users.
+        expected_url = urlreverse(
+            "ietf.submit.views.submission_status", 
+            kwargs=dict(submission_id=submission.pk)
         )
+        selected_elts = q("#manual.submissions td a")
+        self.assertEqual(expected_url, selected_elts.attr("href"))
+        self.assertIn(submission.name, selected_elts.text())
+        for username in ["plain", "secretary"]:
+            self.client.login(username=username, password=username + "+password")
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            selected_elts = q("#manual.submissions td a")
+            self.assertEqual(expected_url, selected_elts.attr("href"))
+            self.assertIn(submission.name, selected_elts.text())
 
-    def test_manualpost_cancel(self):
-        pass
 
 class SubmitTests(BaseSubmitTestCase):
     def setUp(self):
diff --git a/ietf/templates/submit/manual_post.html b/ietf/templates/submit/manual_post.html
index 6e4a2ba42a..0da83e750f 100644
--- a/ietf/templates/submit/manual_post.html
+++ b/ietf/templates/submit/manual_post.html
@@ -1,5 +1,5 @@
 {% extends "submit/submit_base.html" %}
-{# Copyright The IETF Trust 2015, All Rights Reserved #}
+{# Copyright The IETF Trust 2015-2026, All Rights Reserved #}
 {% load origin static %}
 {% block pagehead %}
     <link rel="stylesheet" href="{% static "ietf/css/list.css" %}">
@@ -27,17 +27,9 @@ <h2 class="mt-3" id="man_post">Submissions needing manual posting</h2>
             <tbody>
                 {% for s in manual %}
                     <tr>
-                        {% if user.is_authenticated %}
-                            <td>
-                                <a href="{% url "ietf.submit.views.submission_status" submission_id=s.pk access_token=s.access_token %}">
-                                    {{ s.name }}-{{ s.rev }}
-                                </a>
-                            </td>
-                        {% else %}
-                            <td>
-                                <a href="{% url "ietf.submit.views.submission_status" submission_id=s.pk %}">{{ s.name }}-{{ s.rev }}</a>
-                            </td>
-                        {% endif %}
+                        <td>
+                            <a href="{% url "ietf.submit.views.submission_status" submission_id=s.pk %}">{{ s.name }}-{{ s.rev }}</a>
+                        </td>
                         <td>{{ s.submission_date }}</td>
                         <td>
                             {% if s.passes_checks %}

From 6058769a64778679d4b3b5ca5e6937ed5f2ec6c8 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 2 Apr 2026 15:57:49 -0300
Subject: [PATCH 612/656] ci: optional bucket suffix for storage cfg (#10637)

* ci: optional bucket suffix for storage cfg

* style: ruff ruff

* fix: roll back bizarre editor glitch
---
 docker/scripts/app-configure-blobstore.py | 10 +++++++---
 k8s/settings_local.py                     | 22 ++++++++++++++++------
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py
index 3140e39306..9ae64e0041 100755
--- a/docker/scripts/app-configure-blobstore.py
+++ b/docker/scripts/app-configure-blobstore.py
@@ -24,10 +24,13 @@ def init_blobstore():
         ),
     )
     for bucketname in ARTIFACT_STORAGE_NAMES:
+        adjusted_bucket_name = (
+            os.environ.get("BLOB_STORE_BUCKET_PREFIX", "")
+            + bucketname
+            + os.environ.get("BLOB_STORE_BUCKET_SUFFIX", "")
+        ).strip()
         try:
-            blobstore.create_bucket(
-                Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip()
-            )
+            blobstore.create_bucket(Bucket=adjusted_bucket_name)
         except botocore.exceptions.ClientError as err:
             if err.response["Error"]["Code"] == "BucketAlreadyExists":
                 print(f"Bucket {bucketname} already exists")
@@ -36,5 +39,6 @@ def init_blobstore():
         else:
             print(f"Bucket {bucketname} created")
 
+
 if __name__ == "__main__":
     sys.exit(init_blobstore())
diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 323b7fd45a..b45cbbe260 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -18,7 +18,7 @@
 
 
 def _multiline_to_list(s):
-    """Helper to split at newlines and conver to list"""
+    """Helper to split at newlines and convert to list"""
     return [item.strip() for item in s.split("\n")]
 
 
@@ -80,13 +80,19 @@ def _multiline_to_list(s):
 else:
     raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set")
 
-_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None)
+_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get(
+    "DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None
+)
 if _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY is not None:
-    RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY 
-_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = os.environ.get("DATATRACKER_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", None)
+    RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY
+_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = os.environ.get(
+    "DATATRACKER_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", None
+)
 if _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES is not None:
     RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES
-_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = os.environ.get("DATATRACKER_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None)
+_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = os.environ.get(
+    "DATATRACKER_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None
+)
 if _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL is not None:
     TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL
 
@@ -387,6 +393,7 @@ def _multiline_to_list(s):
         "and DATATRACKER_BLOB_STORE_SECRET_KEY must be set"
     )
 _blob_store_bucket_prefix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "")
+_blob_store_bucket_suffix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_SUFFIX", "")
 _blob_store_enable_profiling = (
     os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true"
 )
@@ -406,6 +413,9 @@ def _multiline_to_list(s):
     if storagename in ["staging"]:
         continue
     replica_storagename = f"r2-{storagename}"
+    adjusted_bucket_name = (
+        _blob_store_bucket_prefix + storagename + _blob_store_bucket_suffix
+    ).strip()
     STORAGES[replica_storagename] = {
         "BACKEND": "ietf.doc.storage.MetadataS3Storage",
         "OPTIONS": dict(
@@ -422,7 +432,7 @@ def _multiline_to_list(s):
                 retries={"total_max_attempts": _blob_store_max_attempts},
             ),
             verify=False,
-            bucket_name=f"{_blob_store_bucket_prefix}{storagename}".strip(),
+            bucket_name=adjusted_bucket_name,
             ietf_log_blob_timing=_blob_store_enable_profiling,
         ),
     }

From a46a2efc05b2e7f5d1b50c76d543e1ca16ae8918 Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Tue, 7 Apr 2026 09:25:24 +1200
Subject: [PATCH 613/656] feat: Generate bcp-index.txt (#10631)

* feat: Generate bcp-index.txt

* fix: Fix issue with author names

* feat: Update bcp-index.txt header

* refactor: Generalize some functions

* fix: Sort RFCs

* test: Add tests for bcp-index.txt

* fix: Fix range bug

* test: Add test for BCP entry

* test: Fix test_create_bcp_txt_index
---
 ietf/sync/rfcindex.py             | 98 +++++++++++++++++++++++++++++++
 ietf/sync/tests_rfcindex.py       | 69 ++++++++++++++++++++--
 ietf/templates/sync/bcp-index.txt | 52 ++++++++++++++++
 3 files changed, 215 insertions(+), 4 deletions(-)
 create mode 100644 ietf/templates/sync/bcp-index.txt

diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py
index 63c2044931..357cc4069a 100644
--- a/ietf/sync/rfcindex.py
+++ b/ietf/sync/rfcindex.py
@@ -24,6 +24,8 @@
 from ietf.utils.log import log
 
 FORMATS_FOR_INDEX = ["txt", "html", "pdf", "xml", "ps"]
+SS_TXT_MARGIN = 3
+SS_TXT_CUE_COL_WIDTH = 14
 
 
 def format_rfc_number(n):
@@ -267,6 +269,87 @@ def get_rfc_text_index_entries():
     return entries
 
 
+def subseries_text_line(line, first=False):
+    """Return subseries text entry line"""
+    indent = " " * SS_TXT_CUE_COL_WIDTH
+    if first:
+        initial_indent = " " * SS_TXT_MARGIN
+    else:
+        initial_indent = indent
+    return fill(
+        line,
+        initial_indent=initial_indent,
+        subsequent_indent=indent,
+        width=80,
+        break_on_hyphens=False,
+    )
+
+
+def get_bcp_text_index_entries():
+    """Returns BCP entries for bcp-index.txt"""
+    entries = []
+
+    highest_bcp_number = (
+        Document.objects.filter(type_id="bcp")
+        .annotate(
+            number=Cast(
+                Substr("name", 4, None),
+                output_field=models.IntegerField(),
+            )
+        )
+        .order_by("-number")
+        .first()
+        .number
+    )
+
+    for bcp_number in range(1, highest_bcp_number + 1):
+        bcp_name = f"BCP{bcp_number}"
+        bcp = Document.objects.filter(type_id="bcp", name=f"{bcp_name.lower()}").first()
+
+        if bcp:
+            entry = subseries_text_line(
+                (
+                    f"[{bcp_name}]"
+                    f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(bcp_name) - 2 - SS_TXT_MARGIN)}"
+                    f"Best Current Practice {bcp_number},"
+                ),
+                first=True,
+            )
+            entry += "\n"
+            entry += subseries_text_line(
+                f"<{settings.RFC_EDITOR_INFO_BASE_URL}{bcp_name.lower()}>."
+            )
+            entry += "\n"
+            entry += subseries_text_line(
+                "At the time of writing, this BCP comprises the following:"
+            )
+            entry += "\n\n"
+            rfcs = sorted(bcp.contains(), key=lambda x: x.rfc_number)
+            for rfc in rfcs:
+                authors = ", ".join(
+                    author.format_for_titlepage() for author in rfc.rfcauthor_set.all()
+                )
+                entry += subseries_text_line(
+                    (
+                        f'{authors}, "{rfc.title}", BCP¶{bcp_number}, RFC¶{rfc.rfc_number}, '
+                        f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, "
+                        f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>."
+                    )
+                ).replace("¶", " ")
+                entry += "\n\n"
+        else:
+            entry = subseries_text_line(
+                (
+                    f"[{bcp_name}]"
+                    f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(bcp_name) - 2 - SS_TXT_MARGIN)}"
+                    f"Best Current Practice {bcp_number} currently contains no RFCs"
+                ),
+                first=True,
+            )
+        entries.append(entry)
+    return entries
+
+
 def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False):
     """Add subseries entries for rfc-index.xml"""
     # subseries docs annotated with numeric number
@@ -481,3 +564,18 @@ def create_rfc_xml_index():
         pretty_print=4,
     )
     save_to_red_bucket("rfc-index.xml", pretty_index)
+
+
+def create_bcp_txt_index():
+    """Create text index of BCPs"""
+    DATE_FMT = "%m/%d/%Y"
+    created_on = timezone.now().strftime(DATE_FMT)
+    log("Creating bcp-index.txt")
+    index = render_to_string(
+        "sync/bcp-index.txt",
+        {
+            "created_on": created_on,
+            "bcps": get_bcp_text_index_entries(),
+        },
+    )
+    save_to_red_bucket("bcp-index.txt", index)
diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py
index e682c016f5..cad5b577d4 100644
--- a/ietf/sync/tests_rfcindex.py
+++ b/ietf/sync/tests_rfcindex.py
@@ -7,16 +7,22 @@
 from django.test.utils import override_settings
 from lxml import etree
 
-from ietf.doc.factories import PublishedRfcDocEventFactory, IndividualRfcFactory
+from ietf.doc.factories import (
+    BcpFactory,
+    IndividualRfcFactory,
+    PublishedRfcDocEventFactory,
+)
 from ietf.name.models import DocTagName
 from ietf.sync.rfcindex import (
+    create_bcp_txt_index,
     create_rfc_txt_index,
     create_rfc_xml_index,
     format_rfc_number,
-    save_to_red_bucket,
-    get_unusable_rfc_numbers,
     get_april1_rfc_numbers,
     get_publication_std_levels,
+    get_unusable_rfc_numbers,
+    save_to_red_bucket,
+    subseries_text_line,
 )
 from ietf.utils.test_utils import TestCase
 
@@ -69,6 +75,9 @@ def setUp(self):
         ).doc
         self.rfc.tags.add(DocTagName.objects.get(slug="errata"))
 
+        # Create a BCP with non-April Fools RFC
+        self.bcp = BcpFactory(contains=[self.rfc], name="bcp11")
+
         # Set up a publication-std-levels.json file to indicate the publication
         # standard of self.rfc as different from its current value
         red_bucket.save(
@@ -137,7 +146,7 @@ def test_create_rfc_xml_index(self, mock_save):
 
         children = list(index)  # elements as list
         # Should be one rfc-not-issued-entry
-        self.assertEqual(len(children), 3)
+        self.assertEqual(len(children), 14)
         self.assertEqual(
             [
                 c.find(f"{ns}doc-id").text
@@ -184,6 +193,53 @@ def test_create_rfc_xml_index(self, mock_save):
             [(f"{ns}month", "April"), (f"{ns}year", "2021")],
         )
 
+    @override_settings(RFCINDEX_INPUT_PATH="input/")
+    @mock.patch("ietf.sync.rfcindex.save_to_red_bucket")
+    def test_create_bcp_txt_index(self, mock_save):
+        create_bcp_txt_index()
+        self.assertEqual(mock_save.call_count, 1)
+        self.assertEqual(mock_save.call_args[0][0], "bcp-index.txt")
+        contents = mock_save.call_args[0][1]
+        self.assertTrue(isinstance(contents, str))
+        # starts from 1
+        self.assertIn(
+            "[BCP1]",
+            contents,
+        )
+        # fill up to 11
+        self.assertIn(
+            "[BCP10]",
+            contents,
+        )
+        # but not to 12
+        self.assertNotIn(
+            "[BCP12]",
+            contents,
+        )
+        # Test empty BCPs
+        self.assertIn(
+            "Best Current Practice 9 currently contains no RFCs",
+            contents,
+        )
+        # No zero prefix!
+        self.assertNotIn(
+            "[BCP0001]",
+            contents,
+        )
+        # Has BCP11 with a RFC
+        self.assertIn(
+            "Best Current Practice 11,",
+            contents,
+        )
+        self.assertIn(
+            f'"{self.rfc.title}"',
+            contents,
+        )
+        self.assertIn(
+            f'BCP 11, RFC {self.rfc.rfc_number},',
+            contents,
+        )
+
 
 class HelperTests(TestCase):
     def test_format_rfc_number(self):
@@ -234,3 +290,8 @@ def test_get_publication_std_levels_raises(self):
         with self.assertRaises(json.JSONDecodeError):
             get_publication_std_levels()
         red_bucket.delete("publication-std-levels.json")
+
+    def test_subseries_text_line(self):
+        text = "foobar"
+        self.assertEqual(subseries_text_line(line=text, first=True), f"   {text}")
+        self.assertEqual(subseries_text_line(line=text), f"              {text}")
diff --git a/ietf/templates/sync/bcp-index.txt b/ietf/templates/sync/bcp-index.txt
new file mode 100644
index 0000000000..dd19920eba
--- /dev/null
+++ b/ietf/templates/sync/bcp-index.txt
@@ -0,0 +1,52 @@
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+                             BCP INDEX
+                           -------------
+
+(CREATED ON: {{created_on}}.)
+
+This file contains citations for all BCPs in numeric order.  The BCPs
+form a sub-series of the RFC document series, specifically those RFCs
+with the status BEST CURRENT PRACTICE.
+
+BCP citations appear in this format:
+
+   [BCP#]     Best Current Practice #,
+              <BCP URL>.
+              At the time of writing, this BCP comprises the following:
+
+              Author 1, Author 2, "Title of the RFC", BCP #, RFC №,
+              DOI DOI string, Issue date,
+              <RFC URL>.
+
+For example:
+
+   [BCP3]     Best Current Practice 3,
+              <https://www.rfc-editor.org/info/bcp3>.
+              At the time of writing, this BCP comprises the following:
+
+              F. Kastenholz, "Variance for The PPP Compression Control Protocol
+              and The PPP Encryption Control Protocol", BCP 3, RFC 1915,
+              DOI 10.17487/RFC1915, February 1996,
+              <https://www.rfc-editor.org/info/rfc1915>.
+
+Key to fields:
+
+# is the BCP number.
+
+№ is the RFC number.
+
+BCPs and other RFCs may be obtained from https://www.rfc-editor.org.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+                                BCP INDEX
+                                ---------
+   
+
+
+{% for bcp in bcps %}{{bcp|safe}}
+
+{% endfor %}

From 7c7219f0dcf326f369c7b4bd04337f95f0a7a9f4 Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Wed, 8 Apr 2026 11:16:57 +1200
Subject: [PATCH 614/656] feat: Generate std-index.txt (#10665)

* feat: Generate std-index.txt

* style: Ruff ruff

Good boy!

* test: Fix flaky test

* test: Add tests for std-index.txt
---
 ietf/sync/rfcindex.py             | 80 +++++++++++++++++++++++++++++++
 ietf/sync/tests_rfcindex.py       | 64 ++++++++++++++++++++++++-
 ietf/templates/sync/std-index.txt | 51 ++++++++++++++++++++
 3 files changed, 193 insertions(+), 2 deletions(-)
 create mode 100644 ietf/templates/sync/std-index.txt

diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py
index 357cc4069a..6a6a4bfa9f 100644
--- a/ietf/sync/rfcindex.py
+++ b/ietf/sync/rfcindex.py
@@ -350,6 +350,71 @@ def get_bcp_text_index_entries():
     return entries
 
 
+def get_std_text_index_entries():
+    """Returns STD entries for std-index.txt"""
+    entries = []
+
+    highest_std_number = (
+        Document.objects.filter(type_id="std")
+        .annotate(
+            number=Cast(
+                Substr("name", 4, None),
+                output_field=models.IntegerField(),
+            )
+        )
+        .order_by("-number")
+        .first()
+        .number
+    )
+
+    for std_number in range(1, highest_std_number + 1):
+        std_name = f"STD{std_number}"
+        std = Document.objects.filter(type_id="std", name=f"{std_name.lower()}").first()
+
+        if std and std.contains():
+            entry = subseries_text_line(
+                (
+                    f"[{std_name}]"
+                    f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(std_name) - 2 - SS_TXT_MARGIN)}"
+                    f"Internet Standard {std_number},"
+                ),
+                first=True,
+            )
+            entry += "\n"
+            entry += subseries_text_line(
+                f"<{settings.RFC_EDITOR_INFO_BASE_URL}{std_name.lower()}>."
+            )
+            entry += "\n"
+            entry += subseries_text_line(
+                "At the time of writing, this STD comprises the following:"
+            )
+            entry += "\n\n"
+            rfcs = sorted(std.contains(), key=lambda x: x.rfc_number)
+            for rfc in rfcs:
+                authors = ", ".join(
+                    author.format_for_titlepage() for author in rfc.rfcauthor_set.all()
+                )
+                entry += subseries_text_line(
+                    (
+                        f'{authors}, "{rfc.title}", STD¶{std_number}, RFC¶{rfc.rfc_number}, '
+                        f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, "
+                        f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>."
+                    )
+                ).replace("¶", " ")
+                entry += "\n\n"
+        else:
+            entry = subseries_text_line(
+                (
+                    f"[{std_name}]"
+                    f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(std_name) - 2 - SS_TXT_MARGIN)}"
+                    f"Internet Standard {std_number} currently contains no RFCs"
+                ),
+                first=True,
+            )
+        entries.append(entry)
+    return entries
+
+
 def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False):
     """Add subseries entries for rfc-index.xml"""
     # subseries docs annotated with numeric number
@@ -579,3 +644,18 @@ def create_bcp_txt_index():
         },
     )
     save_to_red_bucket("bcp-index.txt", index)
+
+
+def create_std_txt_index():
+    """Create text index of STDs"""
+    DATE_FMT = "%m/%d/%Y"
+    created_on = timezone.now().strftime(DATE_FMT)
+    log("Creating std-index.txt")
+    index = render_to_string(
+        "sync/std-index.txt",
+        {
+            "created_on": created_on,
+            "stds": get_std_text_index_entries(),
+        },
+    )
+    save_to_red_bucket("std-index.txt", index)
diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py
index cad5b577d4..70bc41b992 100644
--- a/ietf/sync/tests_rfcindex.py
+++ b/ietf/sync/tests_rfcindex.py
@@ -9,6 +9,7 @@
 
 from ietf.doc.factories import (
     BcpFactory,
+    StdFactory,
     IndividualRfcFactory,
     PublishedRfcDocEventFactory,
 )
@@ -17,6 +18,7 @@
     create_bcp_txt_index,
     create_rfc_txt_index,
     create_rfc_xml_index,
+    create_std_txt_index,
     format_rfc_number,
     get_april1_rfc_numbers,
     get_publication_std_levels,
@@ -78,6 +80,9 @@ def setUp(self):
         # Create a BCP with non-April Fools RFC
         self.bcp = BcpFactory(contains=[self.rfc], name="bcp11")
 
+        # Create a STD with non-April Fools RFC
+        self.std = StdFactory(contains=[self.rfc], name="std11")
+
         # Set up a publication-std-levels.json file to indicate the publication
         # standard of self.rfc as different from its current value
         red_bucket.save(
@@ -146,7 +151,7 @@ def test_create_rfc_xml_index(self, mock_save):
 
         children = list(index)  # elements as list
         # Should be one rfc-not-issued-entry
-        self.assertEqual(len(children), 14)
+        self.assertEqual(len(children), 15)
         self.assertEqual(
             [
                 c.find(f"{ns}doc-id").text
@@ -236,7 +241,62 @@ def test_create_bcp_txt_index(self, mock_save):
             contents,
         )
         self.assertIn(
-            f'BCP 11, RFC {self.rfc.rfc_number},',
+            "BCP 11,",
+            contents,
+        )
+        self.assertIn(
+            f"RFC {self.rfc.rfc_number},",
+            contents,
+        )
+
+    @override_settings(RFCINDEX_INPUT_PATH="input/")
+    @mock.patch("ietf.sync.rfcindex.save_to_red_bucket")
+    def test_create_std_txt_index(self, mock_save):
+        create_std_txt_index()
+        self.assertEqual(mock_save.call_count, 1)
+        self.assertEqual(mock_save.call_args[0][0], "std-index.txt")
+        contents = mock_save.call_args[0][1]
+        self.assertTrue(isinstance(contents, str))
+        # starts from 1
+        self.assertIn(
+            "[STD1]",
+            contents,
+        )
+        # fill up to 11
+        self.assertIn(
+            "[STD10]",
+            contents,
+        )
+        # but not to 12
+        self.assertNotIn(
+            "[STD12]",
+            contents,
+        )
+        # Test empty STDs
+        self.assertIn(
+            "Internet Standard 9 currently contains no RFCs",
+            contents,
+        )
+        # No zero prefix!
+        self.assertNotIn(
+            "[STD0001]",
+            contents,
+        )
+        # Has STD11 with a RFC
+        self.assertIn(
+            "Internet Standard 11,",
+            contents,
+        )
+        self.assertIn(
+            f'"{self.rfc.title}"',
+            contents,
+        )
+        self.assertIn(
+            "STD 11,",
+            contents,
+        )
+        self.assertIn(
+            f"RFC {self.rfc.rfc_number},",
             contents,
         )
 
diff --git a/ietf/templates/sync/std-index.txt b/ietf/templates/sync/std-index.txt
new file mode 100644
index 0000000000..c075d1d43e
--- /dev/null
+++ b/ietf/templates/sync/std-index.txt
@@ -0,0 +1,51 @@
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+                             STD INDEX
+                           -------------
+
+(CREATED ON: {{created_on}}.)
+
+This file contains citations for all STDs in numeric order.  Each
+STD represents a single Internet Standard technical specification,
+composed of one or more RFCs with Internet Standard status.
+
+STD citations appear in this format:
+
+   [STD#]     Best Current Practice #,
+              <STD URL>.
+              At the time of writing, this STD comprises the following:
+
+              Author 1, Author 2, "Title of the RFC", STD #, RFC №,
+              DOI DOI string, Issue date,
+              <RFC URL>.
+
+For example:
+
+   [STD6]     Internet Standard 6,
+              <https://www.rfc-editor.org/info/std6>.
+              At the time of writing, this STD comprises the following:
+
+              J. Postel, "User Datagram Protocol", STD 6, RFC 768,
+              DOI 10.17487/RFC0768, August 1980,
+              <https://www.rfc-editor.org/info/rfc768>.
+
+Key to fields:
+
+# is the STD number.
+
+№ is the RFC number.
+
+STDs and other RFCs may be obtained from https://www.rfc-editor.org.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+                                STD INDEX
+                                ---------
+   
+
+
+{% for std in stds %}{{std|safe}}
+
+{% endfor %}

From e72ead86dee707b5cbd9aeea96437dbaee78c88d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 8 Apr 2026 12:35:06 -0300
Subject: [PATCH 615/656] chore(deps): bump appleboy/ssh-action from 1.2.2 to
 1.2.5 (#10623)

Bumps [appleboy/ssh-action](https://github.com/appleboy/ssh-action) from 1.2.2 to 1.2.5.
- [Release notes](https://github.com/appleboy/ssh-action/releases)
- [Commits](https://github.com/appleboy/ssh-action/compare/2ead5e36573f08b82fbfce1504f1a4b05a647c6f...0ff4204d59e8e51228ff73bce53f80d53301dee2)

---
updated-dependencies:
- dependency-name: appleboy/ssh-action
  dependency-version: 1.2.5
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/tests-az.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml
index 8553563a19..833ca89bef 100644
--- a/.github/workflows/tests-az.yml
+++ b/.github/workflows/tests-az.yml
@@ -38,7 +38,7 @@ jobs:
         ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts
 
     - name: Remote SSH into VM
-      uses: appleboy/ssh-action@2ead5e36573f08b82fbfce1504f1a4b05a647c6f
+      uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:

From c7657c3f22f5f7a906fd2cf01aaed7b54feca9e3 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 8 Apr 2026 12:37:35 -0300
Subject: [PATCH 616/656] chore(deps): bump
 stefanzweifel/git-auto-commit-action from 6 to 7 (#10624)

Bumps [stefanzweifel/git-auto-commit-action](https://github.com/stefanzweifel/git-auto-commit-action) from 6 to 7.
- [Release notes](https://github.com/stefanzweifel/git-auto-commit-action/releases)
- [Changelog](https://github.com/stefanzweifel/git-auto-commit-action/blob/master/CHANGELOG.md)
- [Commits](https://github.com/stefanzweifel/git-auto-commit-action/compare/v6...v7)

---
updated-dependencies:
- dependency-name: stefanzweifel/git-auto-commit-action
  dependency-version: '7'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build-base-app.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index 1b0855cc47..5e274838a1 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -60,7 +60,7 @@ jobs:
         echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE
 
     - name: Commit CHANGELOG.md
-      uses: stefanzweifel/git-auto-commit-action@v6
+      uses: stefanzweifel/git-auto-commit-action@v7
       with:
         branch: ${{ github.ref_name }}
         commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}'

From f39e916a73eaab6c0172a09e98c28ba628b7bcc4 Mon Sep 17 00:00:00 2001
From: Eric Rescorla <ekr@rtfm.com>
Date: Wed, 8 Apr 2026 08:50:19 -0700
Subject: [PATCH 617/656] fix: Rewrite upper right document search box (#10538)

* Rewrite upper right document search box. Fixes #10358

This is a fix to the problem where the first item in the dropdown
is auto-selected and then when you hit return you go to that
rather than searching for what's in the text field.

It appears to be challenging to get this behavior with select2,
so this is actually a rewrite of the box with explicit behavior.
As a side effect, the draft names actually render a bit better.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* Respond to review comments

---------

Co-authored-by: EKR aibot <ekr.aibot@gmail.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
---
 ietf/static/css/ietf.scss           |  17 +++++
 ietf/static/js/navbar-doc-search.js | 113 ++++++++++++++++++++++++++++
 ietf/templates/base.html            |  24 +++---
 package.json                        |   1 +
 4 files changed, 143 insertions(+), 12 deletions(-)
 create mode 100644 ietf/static/js/navbar-doc-search.js

diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss
index df973863d5..6695c57b13 100644
--- a/ietf/static/css/ietf.scss
+++ b/ietf/static/css/ietf.scss
@@ -1216,3 +1216,20 @@ iframe.status {
 .overflow-shadows--bottom-only {
     box-shadow: inset 0px -21px 18px -20px var(--bs-body-color);
 }
+
+#navbar-doc-search-wrapper {
+    position: relative;
+}
+
+#navbar-doc-search-results {
+    max-height: 400px;
+    overflow-y: auto;
+    min-width: auto;
+    left: 0;
+    right: 0;
+
+    .dropdown-item {
+        white-space: normal;
+        overflow-wrap: break-word;
+    }
+}
diff --git a/ietf/static/js/navbar-doc-search.js b/ietf/static/js/navbar-doc-search.js
new file mode 100644
index 0000000000..c36c032310
--- /dev/null
+++ b/ietf/static/js/navbar-doc-search.js
@@ -0,0 +1,113 @@
+$(function () {
+    var $input = $('#navbar-doc-search');
+    var $results = $('#navbar-doc-search-results');
+    var ajaxUrl = $input.data('ajax-url');
+    var debounceTimer = null;
+    var highlightedIndex = -1;
+    var keyboardHighlight = false;
+    var currentItems = [];
+
+    function showDropdown() {
+        $results.addClass('show');
+    }
+
+    function hideDropdown() {
+        $results.removeClass('show');
+        highlightedIndex = -1;
+        keyboardHighlight = false;
+        updateHighlight();
+    }
+
+    function updateHighlight() {
+        $results.find('.dropdown-item').removeClass('active');
+        if (highlightedIndex >= 0 && highlightedIndex < currentItems.length) {
+            $results.find('.dropdown-item').eq(highlightedIndex).addClass('active');
+        }
+    }
+
+    function doSearch(query) {
+        if (query.length < 2) {
+            hideDropdown();
+            return;
+        }
+        $.ajax({
+            url: ajaxUrl,
+            dataType: 'json',
+            data: { q: query },
+            success: function (data) {
+                currentItems = data;
+                highlightedIndex = -1;
+                $results.empty();
+                if (data.length === 0) {
+                    $results.append('<li><span class="dropdown-item text-muted">No results found</span></li>');
+                } else {
+                    data.forEach(function (item) {
+                        var $li = $('<li>');
+                        var $a = $('<a class="dropdown-item" href="' + item.url + '">' + item.text + '</a>');
+                        $li.append($a);
+                        $results.append($li);
+                    });
+                }
+                showDropdown();
+            }
+        });
+    }
+
+    $input.on('input', function () {
+        clearTimeout(debounceTimer);
+        var query = $(this).val().trim();
+        debounceTimer = setTimeout(function () {
+            doSearch(query);
+        }, 250);
+    });
+
+    $input.on('keydown', function (e) {
+        if (e.key === 'ArrowDown') {
+            e.preventDefault();
+            if (highlightedIndex < currentItems.length - 1) {
+                highlightedIndex++;
+                keyboardHighlight = true;
+                updateHighlight();
+            }
+        } else if (e.key === 'ArrowUp') {
+            e.preventDefault();
+            if (highlightedIndex > 0) {
+                highlightedIndex--;
+                keyboardHighlight = true;
+                updateHighlight();
+            }
+        } else if (e.key === 'Enter') {
+            e.preventDefault();
+            if (keyboardHighlight && highlightedIndex >= 0 && highlightedIndex < currentItems.length) {
+                window.location.href = currentItems[highlightedIndex].url;
+            } else {
+                var query = $(this).val().trim();
+                if (query) {
+                    window.location.href = '/doc/search/?name=' + encodeURIComponent(query) + '&rfcs=on&activedrafts=on&olddrafts=on';
+                }
+            }
+        } else if (e.key === 'Escape') {
+            hideDropdown();
+            $input.blur();
+        }
+    });
+
+    // Hover highlights (visual only — Enter still submits the text)
+    $results.on('mouseenter', '.dropdown-item', function () {
+        highlightedIndex = $results.find('.dropdown-item').index(this);
+        keyboardHighlight = false;
+        updateHighlight();
+    });
+
+    $results.on('mouseleave', '.dropdown-item', function () {
+        highlightedIndex = -1;
+        updateHighlight();
+    });
+
+    // Click outside closes dropdown
+    $(document).on('click', function (e) {
+        if (!$(e.target).closest('#navbar-doc-search-wrapper').length) {
+            hideDropdown();
+        }
+    });
+});
diff --git a/ietf/templates/base.html b/ietf/templates/base.html
index 25ce50c467..b0df04f30a 100644
--- a/ietf/templates/base.html
+++ b/ietf/templates/base.html
@@ -67,13 +67,17 @@
                         </a>
                     {% endif %}
 
-                    <label class="d-none d-md-block" aria-label="Document search">
-                    <input class="form-control select2-field search-select"
-                           id="navbar-doc-search"
-                           data-select2-ajax-url="{% url 'ietf.doc.views_search.ajax_select2_search_docs' model_name='document' doc_type='all' %}"
-                           type="text"
-                           data-placeholder="Document search">
-                   </label>
+                    <div class="d-none d-md-block dropdown" id="navbar-doc-search-wrapper">
+                        <input class="form-control"
+                               id="navbar-doc-search"
+                               type="text"
+                               placeholder="Document search"
+                               autocomplete="off"
+                               data-ajax-url="{% url 'ietf.doc.views_search.ajax_select2_search_docs' model_name='document' doc_type='all' %}"
+                               aria-label="Document search">
+                        <ul class="dropdown-menu" id="navbar-doc-search-results">
+                        </ul>
+                    </div>
                 </div>
                 <button class="navbar-toggler"
                         type="button"
@@ -164,11 +168,7 @@
         {% block js %}
         {% endblock %}
         <script src="{% static 'ietf/js/select2.js' %}"></script>
-        <script>
-            $('#navbar-doc-search').on('select2:select', function (e) {
-                window.location.href = e.params.data.url;
-            });
-        </script>
+        <script src="{% static 'ietf/js/navbar-doc-search.js' %}"></script>
       {% analytical_body_bottom %}
     </body>
 </html>
diff --git a/package.json b/package.json
index bb71250c4b..57642d7860 100644
--- a/package.json
+++ b/package.json
@@ -147,6 +147,7 @@
         "ietf/static/js/manage-review-requests.js",
         "ietf/static/js/meeting-interim-request.js",
         "ietf/static/js/moment.js",
+        "ietf/static/js/navbar-doc-search.js",
         "ietf/static/js/password_strength.js",
         "ietf/static/js/select2.js",
         "ietf/static/js/session_details.js",

From a85432fecfa4c7f2ee0ad2dfcc0073936cb0304b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 8 Apr 2026 13:27:18 -0300
Subject: [PATCH 618/656] chore(deps): bump actions/download-artifact from
 6.0.0 to 8.0.1 (#10625)

Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 6.0.0 to 8.0.1.
- [Release notes](https://github.com/actions/download-artifact/releases)
- [Commits](https://github.com/actions/download-artifact/compare/v6.0.0...v8.0.1)

---
updated-dependencies:
- dependency-name: actions/download-artifact
  dependency-version: 8.0.1
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8ec806b229..49a0e5b53b 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -186,7 +186,7 @@ jobs:
 
     - name: Download a Coverage Results
       if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
-      uses: actions/download-artifact@v6.0.0
+      uses: actions/download-artifact@v8.0.1
       with:
         name: coverage
 
@@ -291,7 +291,7 @@ jobs:
 
     - name: Download Coverage Results
       if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
-      uses: actions/download-artifact@v6.0.0
+      uses: actions/download-artifact@v8.0.1
       with:
         name: coverage
 

From 38158873cab134fedbda452351bb750259937bac Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 8 Apr 2026 13:30:57 -0300
Subject: [PATCH 619/656] chore(deps): bump docker/setup-qemu-action from 3 to
 4 (#10626)

Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 3 to 4.
- [Release notes](https://github.com/docker/setup-qemu-action/releases)
- [Commits](https://github.com/docker/setup-qemu-action/compare/v3...v4)

---
updated-dependencies:
- dependency-name: docker/setup-qemu-action
  dependency-version: '4'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/build-base-app.yml  | 2 +-
 .github/workflows/build-mq-broker.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml
index 5e274838a1..35172aa299 100644
--- a/.github/workflows/build-base-app.yml
+++ b/.github/workflows/build-base-app.yml
@@ -28,7 +28,7 @@ jobs:
         echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
+      uses: docker/setup-qemu-action@v4
     
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v4
diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml
index ef7ed2f65c..b297e34b47 100644
--- a/.github/workflows/build-mq-broker.yml
+++ b/.github/workflows/build-mq-broker.yml
@@ -27,7 +27,7 @@ jobs:
     - uses: actions/checkout@v6
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
+      uses: docker/setup-qemu-action@v4
     
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v4

From e774233b8dfbbe5717156f55cc1f4fcdbb5f5686 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 8 Apr 2026 13:32:25 -0300
Subject: [PATCH 620/656] chore(deps): bump codecov/codecov-action from 5 to 6
 (#10628)

Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5 to 6.
- [Release notes](https://github.com/codecov/codecov-action/releases)
- [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/codecov/codecov-action/compare/v5...v6)

---
updated-dependencies:
- dependency-name: codecov/codecov-action
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index be7b834b7a..30b2771897 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -75,7 +75,7 @@ jobs:
         path: geckodriver.log
 
     - name: Upload Coverage Results to Codecov
-      uses: codecov/codecov-action@v5
+      uses: codecov/codecov-action@v6
       with:
         disable_search: true
         files: coverage.xml

From a52f18d4a8c8b7aee0bb0556e77c256e2845e6c1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 8 Apr 2026 13:47:13 -0300
Subject: [PATCH 621/656] chore(deps): bump the npm group across 1 directory
 with 2 updates (#10658)

Bumps the npm group with 2 updates in the /dev/deploy-to-container directory: [slugify](https://github.com/simov/slugify) and [tar](https://github.com/isaacs/node-tar).


Updates `slugify` from 1.6.8 to 1.6.9
- [Changelog](https://github.com/simov/slugify/blob/master/CHANGELOG.md)
- [Commits](https://github.com/simov/slugify/commits)

Updates `tar` from 7.5.12 to 7.5.13
- [Release notes](https://github.com/isaacs/node-tar/releases)
- [Changelog](https://github.com/isaacs/node-tar/blob/main/CHANGELOG.md)
- [Commits](https://github.com/isaacs/node-tar/compare/v7.5.12...v7.5.13)

---
updated-dependencies:
- dependency-name: slugify
  dependency-version: 1.6.9
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
- dependency-name: tar
  dependency-version: 7.5.13
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: npm
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 dev/deploy-to-container/package-lock.json | 28 +++++++++++------------
 dev/deploy-to-container/package.json      |  4 ++--
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json
index a68f170c4b..5d5bef5604 100644
--- a/dev/deploy-to-container/package-lock.json
+++ b/dev/deploy-to-container/package-lock.json
@@ -10,8 +10,8 @@
         "fs-extra": "^11.3.4",
         "nanoid": "5.1.7",
         "nanoid-dictionary": "5.0.0",
-        "slugify": "1.6.8",
-        "tar": "^7.5.12",
+        "slugify": "1.6.9",
+        "tar": "^7.5.13",
         "yargs": "^17.7.2"
       },
       "engines": {
@@ -577,9 +577,9 @@
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
     "node_modules/slugify": {
-      "version": "1.6.8",
-      "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.8.tgz",
-      "integrity": "sha512-HVk9X1E0gz3mSpoi60h/saazLKXKaZThMLU3u/aNwoYn8/xQyX2MGxL0ui2eaokkD7tF+Zo+cKTHUbe1mmmGzA==",
+      "version": "1.6.9",
+      "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.9.tgz",
+      "integrity": "sha512-vZ7rfeehZui7wQs438JXBckYLkIIdfHOXsaVEUMyS5fHo1483l1bMdo0EDSWYclY0yZKFOipDy4KHuKs6ssvdg==",
       "engines": {
         "node": ">=8.0.0"
       }
@@ -639,9 +639,9 @@
       }
     },
     "node_modules/tar": {
-      "version": "7.5.12",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.12.tgz",
-      "integrity": "sha512-9TsuLcdhOn4XztcQqhNyq1KOwOOED/3k58JAvtULiYqbO8B/0IBAAIE1hj0Svmm58k27TmcigyDI0deMlgG3uw==",
+      "version": "7.5.13",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.13.tgz",
+      "integrity": "sha512-tOG/7GyXpFevhXVh8jOPJrmtRpOTsYqUIkVdVooZYJS/z8WhfQUX8RJILmeuJNinGAMSu1veBr4asSHFt5/hng==",
       "dependencies": {
         "@isaacs/fs-minipass": "^4.0.0",
         "chownr": "^3.0.0",
@@ -1194,9 +1194,9 @@
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
     "slugify": {
-      "version": "1.6.8",
-      "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.8.tgz",
-      "integrity": "sha512-HVk9X1E0gz3mSpoi60h/saazLKXKaZThMLU3u/aNwoYn8/xQyX2MGxL0ui2eaokkD7tF+Zo+cKTHUbe1mmmGzA=="
+      "version": "1.6.9",
+      "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.9.tgz",
+      "integrity": "sha512-vZ7rfeehZui7wQs438JXBckYLkIIdfHOXsaVEUMyS5fHo1483l1bMdo0EDSWYclY0yZKFOipDy4KHuKs6ssvdg=="
     },
     "split-ca": {
       "version": "1.0.1",
@@ -1241,9 +1241,9 @@
       }
     },
     "tar": {
-      "version": "7.5.12",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.12.tgz",
-      "integrity": "sha512-9TsuLcdhOn4XztcQqhNyq1KOwOOED/3k58JAvtULiYqbO8B/0IBAAIE1hj0Svmm58k27TmcigyDI0deMlgG3uw==",
+      "version": "7.5.13",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.13.tgz",
+      "integrity": "sha512-tOG/7GyXpFevhXVh8jOPJrmtRpOTsYqUIkVdVooZYJS/z8WhfQUX8RJILmeuJNinGAMSu1veBr4asSHFt5/hng==",
       "requires": {
         "@isaacs/fs-minipass": "^4.0.0",
         "chownr": "^3.0.0",
diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json
index aa9e82dbdf..ccc78fc63b 100644
--- a/dev/deploy-to-container/package.json
+++ b/dev/deploy-to-container/package.json
@@ -6,8 +6,8 @@
     "fs-extra": "^11.3.4",
     "nanoid": "5.1.7",
     "nanoid-dictionary": "5.0.0",
-    "slugify": "1.6.8",
-    "tar": "^7.5.12",
+    "slugify": "1.6.9",
+    "tar": "^7.5.13",
     "yargs": "^17.7.2"
   },
   "engines": {

From 83a08c4f073653bc0634d6d9ed1c853686b1d930 Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Thu, 9 Apr 2026 06:56:35 +1200
Subject: [PATCH 622/656] fix: Fix typo in std-index template (#10671)

---
 ietf/templates/sync/std-index.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ietf/templates/sync/std-index.txt b/ietf/templates/sync/std-index.txt
index c075d1d43e..a4a5fba946 100644
--- a/ietf/templates/sync/std-index.txt
+++ b/ietf/templates/sync/std-index.txt
@@ -13,7 +13,7 @@ composed of one or more RFCs with Internet Standard status.
 
 STD citations appear in this format:
 
-   [STD#]     Best Current Practice #,
+   [STD#]     Internet Standard #,
               <STD URL>.
               At the time of writing, this STD comprises the following:
 

From 03080a7f5b6034e2cd4b0c9b1d9b7b4bbf74ae61 Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Thu, 9 Apr 2026 06:57:07 +1200
Subject: [PATCH 623/656] feat: Generate fyi-index.txt (#10670)

* feat: Generate fyi-index.txt

* test: Add tests for fyi-index.txt
---
 ietf/sync/rfcindex.py             | 80 +++++++++++++++++++++++++++++++
 ietf/sync/tests_rfcindex.py       | 58 +++++++++++++++++++++-
 ietf/templates/sync/fyi-index.txt | 52 ++++++++++++++++++++
 3 files changed, 189 insertions(+), 1 deletion(-)
 create mode 100644 ietf/templates/sync/fyi-index.txt

diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py
index 6a6a4bfa9f..0ea6fb939f 100644
--- a/ietf/sync/rfcindex.py
+++ b/ietf/sync/rfcindex.py
@@ -415,6 +415,71 @@ def get_std_text_index_entries():
     return entries
 
 
+def get_fyi_text_index_entries():
+    """Returns FYI entries for fyi-index.txt"""
+    entries = []
+
+    highest_fyi_number = (
+        Document.objects.filter(type_id="fyi")
+        .annotate(
+            number=Cast(
+                Substr("name", 4, None),
+                output_field=models.IntegerField(),
+            )
+        )
+        .order_by("-number")
+        .first()
+        .number
+    )
+
+    for fyi_number in range(1, highest_fyi_number + 1):
+        fyi_name = f"FYI{fyi_number}"
+        fyi = Document.objects.filter(type_id="fyi", name=f"{fyi_name.lower()}").first()
+
+        if fyi and fyi.contains():
+            entry = subseries_text_line(
+                (
+                    f"[{fyi_name}]"
+                    f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(fyi_name) - 2 - SS_TXT_MARGIN)}"
+                    f"For Your Information {fyi_number},"
+                ),
+                first=True,
+            )
+            entry += "\n"
+            entry += subseries_text_line(
+                f"<{settings.RFC_EDITOR_INFO_BASE_URL}{fyi_name.lower()}>."
+            )
+            entry += "\n"
+            entry += subseries_text_line(
+                "At the time of writing, this FYI comprises the following:"
+            )
+            entry += "\n\n"
+            rfcs = sorted(fyi.contains(), key=lambda x: x.rfc_number)
+            for rfc in rfcs:
+                authors = ", ".join(
+                    author.format_for_titlepage() for author in rfc.rfcauthor_set.all()
+                )
+                entry += subseries_text_line(
+                    (
+                        f'{authors}, "{rfc.title}", FYI¶{fyi_number}, RFC¶{rfc.rfc_number}, '
+                        f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, "
+                        f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>."
+                    )
+                ).replace("¶", " ")
+                entry += "\n\n"
+        else:
+            entry = subseries_text_line(
+                (
+                    f"[{fyi_name}]"
+                    f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(fyi_name) - 2 - SS_TXT_MARGIN)}"
+                    f"For Your Information {fyi_number} currently contains no RFCs"
+                ),
+                first=True,
+            )
+        entries.append(entry)
+    return entries
+
+
 def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False):
     """Add subseries entries for rfc-index.xml"""
     # subseries docs annotated with numeric number
@@ -659,3 +724,18 @@ def create_std_txt_index():
         },
     )
     save_to_red_bucket("std-index.txt", index)
+
+
+def create_fyi_txt_index():
+    """Create text index of FYIs"""
+    DATE_FMT = "%m/%d/%Y"
+    created_on = timezone.now().strftime(DATE_FMT)
+    log("Creating fyi-index.txt")
+    index = render_to_string(
+        "sync/fyi-index.txt",
+        {
+            "created_on": created_on,
+            "fyis": get_fyi_text_index_entries(),
+        },
+    )
+    save_to_red_bucket("fyi-index.txt", index)
diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py
index 70bc41b992..541ffbb228 100644
--- a/ietf/sync/tests_rfcindex.py
+++ b/ietf/sync/tests_rfcindex.py
@@ -9,6 +9,7 @@
 
 from ietf.doc.factories import (
     BcpFactory,
+    FyiFactory,
     StdFactory,
     IndividualRfcFactory,
     PublishedRfcDocEventFactory,
@@ -16,6 +17,7 @@
 from ietf.name.models import DocTagName
 from ietf.sync.rfcindex import (
     create_bcp_txt_index,
+    create_fyi_txt_index,
     create_rfc_txt_index,
     create_rfc_xml_index,
     create_std_txt_index,
@@ -83,6 +85,9 @@ def setUp(self):
         # Create a STD with non-April Fools RFC
         self.std = StdFactory(contains=[self.rfc], name="std11")
 
+        # Create a FYI with non-April Fools RFC
+        self.fyi = FyiFactory(contains=[self.rfc], name="fyi11")
+
         # Set up a publication-std-levels.json file to indicate the publication
         # standard of self.rfc as different from its current value
         red_bucket.save(
@@ -151,7 +156,7 @@ def test_create_rfc_xml_index(self, mock_save):
 
         children = list(index)  # elements as list
         # Should be one rfc-not-issued-entry
-        self.assertEqual(len(children), 15)
+        self.assertEqual(len(children), 16)
         self.assertEqual(
             [
                 c.find(f"{ns}doc-id").text
@@ -300,6 +305,57 @@ def test_create_std_txt_index(self, mock_save):
             contents,
         )
 
+    @override_settings(RFCINDEX_INPUT_PATH="input/")
+    @mock.patch("ietf.sync.rfcindex.save_to_red_bucket")
+    def test_create_fyi_txt_index(self, mock_save):
+        create_fyi_txt_index()
+        self.assertEqual(mock_save.call_count, 1)
+        self.assertEqual(mock_save.call_args[0][0], "fyi-index.txt")
+        contents = mock_save.call_args[0][1]
+        self.assertTrue(isinstance(contents, str))
+        # starts from 1
+        self.assertIn(
+            "[FYI1]",
+            contents,
+        )
+        # fill up to 11
+        self.assertIn(
+            "[FYI10]",
+            contents,
+        )
+        # but not to 12
+        self.assertNotIn(
+            "[FYI12]",
+            contents,
+        )
+        # Test empty FYIs
+        self.assertIn(
+            "For Your Information 9 currently contains no RFCs",
+            contents,
+        )
+        # No zero prefix!
+        self.assertNotIn(
+            "[FYI0001]",
+            contents,
+        )
+        # Has FYI11 with a RFC
+        self.assertIn(
+            "For Your Information 11,",
+            contents,
+        )
+        self.assertIn(
+            f'"{self.rfc.title}"',
+            contents,
+        )
+        self.assertIn(
+            "FYI 11,",
+            contents,
+        )
+        self.assertIn(
+            f"RFC {self.rfc.rfc_number},",
+            contents,
+        )
+
 
 class HelperTests(TestCase):
     def test_format_rfc_number(self):
diff --git a/ietf/templates/sync/fyi-index.txt b/ietf/templates/sync/fyi-index.txt
new file mode 100644
index 0000000000..cf9d57d570
--- /dev/null
+++ b/ietf/templates/sync/fyi-index.txt
@@ -0,0 +1,52 @@
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+                             FYI INDEX
+                           -------------
+
+(CREATED ON: {{created_on}}.)
+
+This file contains citations for all FYIs in numeric order.  The FYIs
+(For Your Information) documents form a sub-series of the RFC series,
+specifically those documents that may be of particular interest
+to Internet users.  The corresponding RFCs have status INFORMATIONAL.
+
+FYI citations appear in this format:
+
+   [FYI#]     For Your Information #,
+              <FYI URL>.
+              At the time of writing, this FYI comprises the following:
+
+              Author 1, Author 2, "Title of the RFC", FYI #, RFC №,
+              DOI DOI string, Issue date,
+              <RFC URL>.
+
+For example:
+
+   [FYI8]     For Your Information 8,
+              <https://www.rfc-editor.org/info/fyi8>.
+              At the time of writing, this FYI comprises the following:
+
+              B. Fraser, "Site Security Handbook", FYI 8, RFC 2196,
+              DOI 10.17487/RFC2196, September 1997,
+              <https://www.rfc-editor.org/info/rfc2196>.
+
+Key to fields:
+
+# is the FYI number.
+
+№ is the RFC number.
+
+FYIs and other RFCs may be obtained from https://www.rfc-editor.org.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+                                FYI INDEX
+                                ---------
+   
+
+
+{% for fyi in fyis %}{{fyi|safe}}
+
+{% endfor %}

From 6c6919f851ac6de2ca63c7bd903b1ddeb198fafc Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 8 Apr 2026 16:07:32 -0300
Subject: [PATCH 624/656] chore(deps): require httpx since we import it
 (#10678)

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 2b8185dab9..4fe55d3600 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -41,6 +41,7 @@ gunicorn>=23.0.0
 hashids>=1.3.1
 html2text>=2025.4.15  # Used only to clean comment field of secr/sreq
 html5lib>=1.1    # Only used in tests
+httpx>=0.28.1  # Indirect req of typesense, but we import and refer to exceptions
 icalendar>=5.0.0 
 inflect>= 7.5.0
 jsonfield>=3.2.0  # deprecated - need to replace with Django's JSONField

From e1ae8d3dfef3254713a9c9abea0512baa95c4ec3 Mon Sep 17 00:00:00 2001
From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com>
Date: Wed, 8 Apr 2026 19:21:37 +0000
Subject: [PATCH 625/656] ci: update base image target version to 20260408T1908

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index af43e990e0..72511b6652 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20260323T1533
+FROM ghcr.io/ietf-tools/datatracker-app-base:20260408T1908
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index 09f74cce28..c8ef587199 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20260323T1533
+20260408T1908

From 68104b53108e28f368cb95b10b667c4818f98498 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 9 Apr 2026 13:09:54 -0300
Subject: [PATCH 626/656] ci: disable playwright legacy tests

These are always failing. Seems to be an issue with the test runner
rather than with the code.
---
 .github/workflows/tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 30b2771897..ad2e35408d 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -130,6 +130,7 @@ jobs:
         if-no-files-found: ignore
 
   tests-playwright-legacy:
+    if: ${{ false }}  # disable until we sort out suspected test runner issue
     name: Playwright Legacy Tests
     runs-on: ubuntu-latest
     container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }}

From a77ef8c1569acb6e03ce14cc9dabdcc9f53689f5 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 9 Apr 2026 14:23:12 -0300
Subject: [PATCH 627/656] ci: plumb errata notification API (#10684)

---
 k8s/settings_local.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index b45cbbe260..3a7e524f5a 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -497,3 +497,17 @@ def _multiline_to_list(s):
         "DATATRACKER_SEARCHINDEX_TASK_MAX_RETRIES", "12"
     ),
 }
+
+# Errata system api configuration
+ERRATA_METADATA_NOTIFICATION_API_KEY = os.environ.get(
+    "DATATRACKER_ERRATA_METADATA_NOTIFICATION_API_KEY", None
+)
+if ERRATA_METADATA_NOTIFICATION_API_KEY is not None:
+    ERRATA_METADATA_NOTIFICATION_URL = os.environ.get(
+        "DATATRACKER_ERRATA_METADATA_NOTIFICATION_URL", None
+    )
+    if ERRATA_METADATA_NOTIFICATION_URL is None:
+        raise RuntimeError(
+            "DATATRACKER_ERRATA_METADATA_NOTIFICATION_URL must be set if "
+            "DATATRACKER_ERRATA_METADATA_NOTIFICATION_API_KEY is provided"
+        )

From 4c5d4224b91da63e747c62177f0a8297760ce6ac Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 10 Apr 2026 12:57:20 -0300
Subject: [PATCH 628/656] chore(deps): unpin django-celery-beat (#10679)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 4fe55d3600..ca9a6740e1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,7 +17,7 @@ Django>4.2,<5
 django-admin-rangefilter>=0.13.3
 django-analytical>=3.2.0
 django-bootstrap5>=25.1
-django-celery-beat>=2.7.0,<2.8.0  # pin until https://github.com/celery/django-celery-beat/issues/875 is resolved, then revisit
+django-celery-beat>=2.9.0
 django-celery-results>=2.6.0
 django-csp>=3.7
 django-cors-headers>=4.7.0

From 608a5b73dc6a498162b485e40a9517b9db6adac6 Mon Sep 17 00:00:00 2001
From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com>
Date: Fri, 10 Apr 2026 16:13:25 +0000
Subject: [PATCH 629/656] ci: update base image target version to 20260410T1557

---
 dev/build/Dockerfile  | 2 +-
 dev/build/TARGET_BASE | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile
index 72511b6652..e57fecd5f2 100644
--- a/dev/build/Dockerfile
+++ b/dev/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/ietf-tools/datatracker-app-base:20260408T1908
+FROM ghcr.io/ietf-tools/datatracker-app-base:20260410T1557
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
 ENV DEBIAN_FRONTEND=noninteractive
diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE
index c8ef587199..f430037c09 100644
--- a/dev/build/TARGET_BASE
+++ b/dev/build/TARGET_BASE
@@ -1 +1 @@
-20260408T1908
+20260410T1557

From 73132b34f6dc4088a5a27977f68d8a05cbc38e03 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 13 Apr 2026 14:38:42 -0300
Subject: [PATCH 630/656] fix: resolve procmaterials when uploaded (#10693)

* fix: resolve procmaterials when uploaded

* fix: avoid leaving stale ResolvedMaterial
---
 ietf/meeting/utils.py             | 16 ++++++++++++++--
 ietf/meeting/views_proceedings.py |  4 ++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py
index bdf3d3d3d3..10ae0d3667 100644
--- a/ietf/meeting/utils.py
+++ b/ietf/meeting/utils.py
@@ -1025,9 +1025,18 @@ def resolve_materials_for_one_meeting(meeting: Meeting):
         )
 
 def resolve_uploaded_material(meeting: Meeting, doc: Document):
-    resolved = []
+    resolved: list[ResolvedMaterial] = []
+    remove = ResolvedMaterial.objects.none()
     blob = resolve_one_material(doc, rev=None, ext=None)
-    if blob is not None:
+    if blob is None:
+        # Versionless file does not exist. Remove the versionless ResolvedMaterial
+        # if it existed. This is to avoid leaving behind a stale link to a replaced
+        # version. This comes up e.g. if a ProceedingsMaterial is changed from having
+        # an uploaded file to being an external URL.
+        remove = ResolvedMaterial.objects.filter(
+            name=doc.name, meeting_number=meeting.number
+        )
+    else:
         resolved.append(
             ResolvedMaterial(
                 name=doc.name,
@@ -1047,12 +1056,15 @@ def resolve_uploaded_material(meeting: Meeting, doc: Document):
                 blob=blob.name,
             )
         )
+    # Create the new record(s) 
     ResolvedMaterial.objects.bulk_create(
         resolved,
         update_conflicts=True,
         unique_fields=["name", "meeting_number"],
         update_fields=["bucket", "blob"],
     )
+    # and remove one if necessary (will be a none() queryset if not)
+    remove.delete()
 
 
 def store_blob_for_one_material_file(doc: Document, rev: str, filepath: Path):
diff --git a/ietf/meeting/views_proceedings.py b/ietf/meeting/views_proceedings.py
index d1169bff2d..639efa1da4 100644
--- a/ietf/meeting/views_proceedings.py
+++ b/ietf/meeting/views_proceedings.py
@@ -14,7 +14,7 @@
 from ietf.meeting.models import Meeting, MeetingHost
 from ietf.meeting.helpers import get_meeting
 from ietf.name.models import ProceedingsMaterialTypeName
-from ietf.meeting.utils import handle_upload_file
+from ietf.meeting.utils import handle_upload_file, resolve_uploaded_material
 from ietf.utils.text import xslugify
 
 class UploadProceedingsMaterialForm(FileUploadForm):
@@ -150,7 +150,7 @@ def save_proceedings_material_doc(meeting, material_type, title, request, file=N
 
     if events:
         doc.save_with_history(events)
-
+    resolve_uploaded_material(meeting, doc)
     return doc
 
 

From 3e23f162c4929703a630141a343d820252b29a83 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 13 Apr 2026 13:26:38 -0500
Subject: [PATCH 631/656] chore: git ignore .claude (#10705)

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 84bc800e3b..ccc7a46b08 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 .DS_store
 datatracker.sublime-project
 datatracker.sublime-workspace
+/.claude
 /.coverage
 /.factoryboy_random_state
 /.mypy_cache

From dee75e2e67bcb47b2dc1e556cc34ecd08dfa6806 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 13 Apr 2026 15:37:33 -0300
Subject: [PATCH 632/656] feat: unmask exceptions on blob retrieve error
 (#10704)

These have not been encountered in at least three months, likely longer.

Still remains to unmask exceptions when storing blobs. We are probably
ready to do that, but log noise caused by expected failures makes it
hard to be certain. Soon...
---
 ietf/doc/storage_utils.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py
index 81588c83ec..ffdd4599be 100644
--- a/ietf/doc/storage_utils.py
+++ b/ietf/doc/storage_utils.py
@@ -178,8 +178,7 @@ def retrieve_bytes(kind: str, name: str) -> bytes:
                     content = f.read()
         except Exception as err:
             log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}")
-            if settings.SERVER_MODE == "development":
-                raise
+            raise
     return content
 
 
@@ -192,6 +191,5 @@ def retrieve_str(kind: str, name: str) -> str:
             content = content_bytes.decode("utf-8")
         except Exception as err:
             log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}")
-            if settings.SERVER_MODE == "development":
-                raise
+            raise
     return content

From aa847b4b8ae292e60f9d8f96bc23904c156cda47 Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Mon, 13 Apr 2026 14:15:06 -0500
Subject: [PATCH 633/656] fix: look in target if draft name not found in anchor
 (#10636)

---
 ietf/utils/xmldraft.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ietf/utils/xmldraft.py b/ietf/utils/xmldraft.py
index f555a0a16a..325b8499a9 100644
--- a/ietf/utils/xmldraft.py
+++ b/ietf/utils/xmldraft.py
@@ -102,6 +102,17 @@ def _document_name(self, ref):
             number = int(maybe_number)
             return f"{label}{number}"
 
+        target = ref.get("target")
+        if isinstance(target, str):
+            target = target.lower()
+            if target.startswith("https://datatracker.ietf.org/doc/"):
+                # len("https://datatracker.ietf.org/doc/")==33
+                m = re.match(r"^(draft-[a-z0-9-]*[a-z0-9])([/-]\d{2})?/?$",target[33:])
+                if m:
+                    name = m.group(1)
+                    return name
+
+
         # if we couldn't find a match so far, try the seriesInfo
         series_query = " or ".join(f"@name='{x.upper()}'" for x in series)
         for info in ref.xpath(

From 60ecfa2029f81b6a6b987770a1edd4840a6acf02 Mon Sep 17 00:00:00 2001
From: NGPixel <github@ngpixel.com>
Date: Mon, 13 Apr 2026 16:43:16 -0400
Subject: [PATCH 634/656] chore(k8s): remove interpod affinity

---
 k8s/auth.yaml       | 10 ----------
 k8s/beat.yaml       | 10 ----------
 k8s/celery.yaml     | 10 ----------
 k8s/memcached.yaml  | 10 ----------
 k8s/rabbitmq.yaml   | 10 ----------
 k8s/replicator.yaml | 10 ----------
 6 files changed, 60 deletions(-)

diff --git a/k8s/auth.yaml b/k8s/auth.yaml
index 392e306b54..2bdb064447 100644
--- a/k8s/auth.yaml
+++ b/k8s/auth.yaml
@@ -15,16 +15,6 @@ spec:
       labels:
         app: auth
     spec:
-      affinity:
-        podAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - datatracker
-              topologyKey: "kubernetes.io/hostname"
       securityContext:
         runAsNonRoot: true
       containers:
diff --git a/k8s/beat.yaml b/k8s/beat.yaml
index cc98beecf6..9ab242681c 100644
--- a/k8s/beat.yaml
+++ b/k8s/beat.yaml
@@ -17,16 +17,6 @@ spec:
       labels:
         app: beat
     spec:
-      affinity:
-        podAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - datatracker
-              topologyKey: "kubernetes.io/hostname"
       securityContext:
         runAsNonRoot: true
       containers:
diff --git a/k8s/celery.yaml b/k8s/celery.yaml
index a2799f2a6d..2f4c0fd439 100644
--- a/k8s/celery.yaml
+++ b/k8s/celery.yaml
@@ -17,16 +17,6 @@ spec:
       labels:
         app: celery
     spec:
-      affinity:
-        podAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - datatracker
-              topologyKey: "kubernetes.io/hostname"
       securityContext:
         runAsNonRoot: true
       containers:
diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml
index 8f73f3d0d5..5a4c9f0aed 100644
--- a/k8s/memcached.yaml
+++ b/k8s/memcached.yaml
@@ -13,16 +13,6 @@ spec:
       labels:
         app: memcached
     spec:
-      affinity:
-        podAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - datatracker
-              topologyKey: "kubernetes.io/hostname"
       securityContext:
         runAsNonRoot: true
       containers:
diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml
index 780a399239..346b54c93e 100644
--- a/k8s/rabbitmq.yaml
+++ b/k8s/rabbitmq.yaml
@@ -13,16 +13,6 @@ spec:
       labels:
         app: rabbitmq
     spec:
-      affinity:
-        podAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - datatracker
-              topologyKey: "kubernetes.io/hostname"
       securityContext:
         runAsNonRoot: true
       containers:
diff --git a/k8s/replicator.yaml b/k8s/replicator.yaml
index 9c462bd96b..a28d9e8a16 100644
--- a/k8s/replicator.yaml
+++ b/k8s/replicator.yaml
@@ -17,16 +17,6 @@ spec:
       labels:
         app: replicator
     spec:
-      affinity:
-        podAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchExpressions:
-                  - key: app
-                    operator: In
-                    values:
-                      - datatracker
-              topologyKey: "kubernetes.io/hostname"
       securityContext:
         runAsNonRoot: true
       containers:

From 1689b88859e7c7fc49de4c96c9ac3eec7c239e50 Mon Sep 17 00:00:00 2001
From: Kesara Rathnayake <kesara@fq.nz>
Date: Tue, 14 Apr 2026 16:06:34 +1200
Subject: [PATCH 635/656] fix: Wire bcp, std, fyi index creation (#10706)

* style: Ruff Ruff

* fix: Wire bcp and std index creation tasks

* fix: Use single task and add fyi index creation

* fix: Log index creation errors and continue
---
 ietf/api/views_rpc.py | 21 ++++++-----
 ietf/sync/tasks.py    | 82 +++++++++++++++++++++++++++++++------------
 2 files changed, 72 insertions(+), 31 deletions(-)

diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index 1e96118e58..25982d76bf 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -32,7 +32,9 @@
     EmailPersonSerializer,
     RfcWithAuthorsSerializer,
     DraftWithAuthorsSerializer,
-    NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer,
+    NotificationAckSerializer,
+    RfcPubSerializer,
+    RfcFileSerializer,
     EditableRfcSerializer,
 )
 from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent
@@ -344,9 +346,10 @@ def post(self, request):
 
 class RfcAuthorViewSet(viewsets.ReadOnlyModelViewSet):
     """ViewSet for RfcAuthor model
-    
+
     Router needs to provide rfc_number as a kwarg
     """
+
     api_key_endpoint = "ietf.api.views_rpc"
 
     queryset = RfcAuthor.objects.all()
@@ -407,7 +410,7 @@ class RfcPubFilesView(APIView):
 
     def _fs_destination(self, filename: str | Path) -> Path:
         """Destination for an uploaded RFC file in the filesystem
-        
+
         Strips any path components in filename and returns an absolute Path.
         """
         rfc_path = Path(settings.RFC_PATH)
@@ -419,7 +422,7 @@ def _fs_destination(self, filename: str | Path) -> Path:
 
     def _blob_destination(self, filename: str | Path) -> str:
         """Destination name for an uploaded RFC file in the blob store
-        
+
         Strips any path components in filename and returns an absolute Path.
         """
         filename = Path(filename)  # could potentially have directory components
@@ -472,9 +475,7 @@ def post(self, request):
                         code="files-exist",
                     )
             for possible_existing_blob in possible_rfc_blobs:
-                if exists_in_storage(
-                    kind=blob_kind, name=possible_existing_blob
-                ):
+                if exists_in_storage(kind=blob_kind, name=possible_existing_blob):
                     raise Conflict(
                         "Blob(s) already exist for this RFC",
                         code="blobs-exist",
@@ -523,7 +524,9 @@ def post(self, request):
 
         # Trigger red precomputer
         needs_updating = [rfc.rfc_number]
-        for rel in rfc.relateddocument_set.filter(relationship_id__in=["obs","updates"]):
+        for rel in rfc.relateddocument_set.filter(
+            relationship_id__in=["obs", "updates"]
+        ):
             needs_updating.append(rel.target.rfc_number)
         trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating))
         # Trigger search index update
@@ -540,7 +543,7 @@ class RfcIndexView(APIView):
     @extend_schema(
         operation_id="refresh_rfc_index",
         summary="Refresh rfc-index files",
-        description="Requests creation of rfc-index.xml and rfc-index.txt files",
+        description="Requests creation of various index files.",
         responses={202: None},
         request=None,
     )
diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py
index 4c84dc581e..c48368cccd 100644
--- a/ietf/sync/tasks.py
+++ b/ietf/sync/tasks.py
@@ -18,7 +18,13 @@
 from ietf.sync import iana
 from ietf.sync import rfceditor
 from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue
-from ietf.sync.rfcindex import create_rfc_txt_index, create_rfc_xml_index
+from ietf.sync.rfcindex import (
+    create_bcp_txt_index,
+    create_fyi_txt_index,
+    create_rfc_txt_index,
+    create_rfc_xml_index,
+    create_std_txt_index,
+)
 from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper
 from ietf.utils import log
 from ietf.utils.timezone import date_today
@@ -27,13 +33,13 @@
 @shared_task
 def rfc_editor_index_update_task(full_index=False):
     """Update metadata from the RFC index
-    
+
     Default is to examine only changes in the past 365 days. Call with full_index=True to update
     the full RFC index.
-    
+
     According to comments on the original script, a year's worth took about 20s on production as of
     August 2022
-    
+
     The original rfc-editor-index-update script had a long-disabled provision for running the
     rebuild_reference_relations scripts after the update. That has not been brought over
     at all because it should be implemented as its own task if it is needed.
@@ -51,7 +57,7 @@ def rfc_editor_index_update_task(full_index=False):
             timeout=30,  # seconds
         )
     except requests.Timeout as exc:
-        log.log(f'GET request timed out retrieving RFC editor index: {exc}')
+        log.log(f"GET request timed out retrieving RFC editor index: {exc}")
         return  # failed
     rfc_index_xml = response.text
     index_data = rfceditor.parse_index(io.StringIO(rfc_index_xml))
@@ -61,9 +67,9 @@ def rfc_editor_index_update_task(full_index=False):
             timeout=30,  # seconds
         )
     except requests.Timeout as exc:
-        log.log(f'GET request timed out retrieving RFC editor errata: {exc}')
+        log.log(f"GET request timed out retrieving RFC editor errata: {exc}")
         return  # failed
-    errata_data = response.json()   
+    errata_data = response.json()
     if len(index_data) < rfceditor.MIN_INDEX_RESULTS:
         log.log("Not enough index entries, only %s" % len(index_data))
         return  # failed
@@ -96,15 +102,15 @@ def rfc_editor_queue_updates_task():
     drafts, warnings = parse_queue(io.StringIO(response.text))
     for w in warnings:
         log.log(f"Warning: {w}")
-    
+
     if len(drafts) < MIN_QUEUE_RESULTS:
         log.log("Not enough results, only %s" % len(drafts))
         return  # failed
-    
+
     changed, warnings = update_drafts_from_queue(drafts)
     for w in warnings:
         log.log(f"Warning: {w}")
-    
+
     for c in changed:
         log.log(f"Updated {c}")
 
@@ -120,9 +126,11 @@ def iana_changes_update_task():
     MAX_INTERVAL_ACCEPTED_BY_IANA = datetime.timedelta(hours=23)
 
     start = (
-        timezone.now() 
-        - datetime.timedelta(hours=23) 
-        + datetime.timedelta(seconds=CLOCK_SKEW_COMPENSATION,)
+        timezone.now()
+        - datetime.timedelta(hours=23)
+        + datetime.timedelta(
+            seconds=CLOCK_SKEW_COMPENSATION,
+        )
     )
     end = start + datetime.timedelta(hours=23)
 
@@ -133,7 +141,9 @@ def iana_changes_update_task():
         # requests if necessary
 
         text = iana.fetch_changes_json(
-            settings.IANA_SYNC_CHANGES_URL, t, min(end, t + MAX_INTERVAL_ACCEPTED_BY_IANA)
+            settings.IANA_SYNC_CHANGES_URL,
+            t,
+            min(end, t + MAX_INTERVAL_ACCEPTED_BY_IANA),
         )
         log.log(f"Retrieved the JSON: {text}")
 
@@ -159,9 +169,9 @@ def iana_protocols_update_task():
     # "this needs to be the date where this tool is first deployed" in the original
     # iana-protocols-updates script)"
     rfc_must_published_later_than = datetime.datetime(
-        2012, 
-        11, 
-        26, 
+        2012,
+        11,
+        26,
         tzinfo=datetime.UTC,
     )
 
@@ -171,17 +181,17 @@ def iana_protocols_update_task():
             timeout=30,
         )
     except requests.Timeout as exc:
-        log.log(f'GET request timed out retrieving IANA protocols page: {exc}')
+        log.log(f"GET request timed out retrieving IANA protocols page: {exc}")
         return
 
     rfc_numbers = iana.parse_protocol_page(response.text)
 
     def batched(l, n):
         """Split list l up in batches of max size n.
-        
+
         For Python 3.12 or later, replace this with itertools.batched()
         """
-        return (l[i:i + n] for i in range(0, len(l), n))
+        return (l[i : i + n] for i in range(0, len(l), n))
 
     for batch in batched(rfc_numbers, 100):
         updated = iana.update_rfc_log_from_protocol_page(
@@ -192,6 +202,7 @@ def batched(l, n):
         for d in updated:
             log.log("Added history entry for %s" % d.display_name())
 
+
 @shared_task
 def fix_subseries_docevents_task():
     """Repairs DocEvents related to bugs around removing docs from subseries
@@ -233,6 +244,7 @@ def fix_subseries_docevents_task():
             time=obsoleting_time
         )
 
+
 @shared_task
 def rsync_rfcs_from_rfceditor_task(rfc_numbers: list[int]):
     log.log(f"Rsyncing rfcs from rfc-editor: {rfc_numbers}")
@@ -277,6 +289,32 @@ def load_rfcs_into_blobdb_task(start: int, end: int):
 
 @shared_task
 def create_rfc_index_task():
-    create_rfc_txt_index()
-    create_rfc_xml_index()
+    try:
+        create_rfc_txt_index()
+    except Exception as e:
+        log.log(f"Error: failure in creating rfc-index.txt. {e}")
+        pass
 
+    try:
+        create_rfc_xml_index()
+    except Exception as e:
+        log.log(f"Error: failure in creating rfc-index.xml. {e}")
+        pass
+
+    try:
+        create_bcp_txt_index()
+    except Exception as e:
+        log.log(f"Error: failure in creating bcp-index.txt. {e}")
+        pass
+
+    try:
+        create_std_txt_index()
+    except Exception as e:
+        log.log(f"Error: failure in creating std-index.txt. {e}")
+        pass
+
+    try:
+        create_fyi_txt_index()
+    except Exception as e:
+        log.log(f"Error: failure in creating fyi-index.txt. {e}")
+        pass

From c66655fb5745bfb8657edcae534f96056a7d4ff0 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 13:33:45 -0300
Subject: [PATCH 636/656] feat: debounce rfc index refresh (#10708)

* feat: DirtyBits model + admin

* style: ruff + modernize utils/admin.py

* chore: remove unused code

* feat: rfcindex DirtyBits helpers

* feat: refresh RFC index only when dirty

Renames the task to better match its purpose

* test: update test

* fix: typo

* chore: add (empty) DirtyBits resource

* fix: actually call mark_rfcindex_as_processed
---
 ietf/api/tests_views_rpc.py             | 17 +++++--
 ietf/api/views_rpc.py                   |  4 +-
 ietf/sync/rfcindex.py                   | 50 ++++++++++++++++++
 ietf/sync/tasks.py                      | 68 ++++++++++++++-----------
 ietf/utils/admin.py                     | 65 +++++------------------
 ietf/utils/migrations/0003_dirtybits.py | 37 ++++++++++++++
 ietf/utils/models.py                    | 25 ++++++++-
 ietf/utils/resources.py                 | 12 +++--
 8 files changed, 183 insertions(+), 95 deletions(-)
 create mode 100644 ietf/utils/migrations/0003_dirtybits.py

diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py
index 0db67e126f..180221cffc 100644
--- a/ietf/api/tests_views_rpc.py
+++ b/ietf/api/tests_views_rpc.py
@@ -1,4 +1,5 @@
 # Copyright The IETF Trust 2025, All Rights Reserved
+import datetime
 from io import StringIO
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -10,12 +11,15 @@
 from django.test.utils import override_settings
 from django.urls import reverse as urlreverse
 import mock
+from django.utils import timezone
 
 from ietf.blobdb.models import Blob
 from ietf.doc.factories import IndividualDraftFactory, RfcFactory, WgDraftFactory, WgRfcFactory
 from ietf.doc.models import RelatedDocument, Document
 from ietf.group.factories import RoleFactory, GroupFactory
 from ietf.person.factories import PersonFactory
+from ietf.sync.rfcindex import rfcindex_is_dirty
+from ietf.utils.models import DirtyBits
 from ietf.utils.test_utils import APITestCase, reload_db_objects
 
 
@@ -408,8 +412,13 @@ def _valid_post_data():
             )
 
     @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]})
-    @mock.patch("ietf.api.views_rpc.create_rfc_index_task")
-    def test_refresh_rfc_index(self, mock_task):
+    def test_refresh_rfc_index(self):
+        DirtyBits.objects.create(
+            slug=DirtyBits.Slugs.RFCINDEX,
+            dirty_time=timezone.now() - datetime.timedelta(days=1),
+            processed_time=timezone.now() - datetime.timedelta(hours=12),
+        )
+        self.assertFalse(rfcindex_is_dirty())
         url = urlreverse("ietf.api.purple_api.refresh_rfc_index")
         response = self.client.get(url)
         self.assertEqual(response.status_code, 403)
@@ -417,7 +426,7 @@ def test_refresh_rfc_index(self, mock_task):
         self.assertEqual(response.status_code, 403)
         response = self.client.get(url, headers={"X-Api-Key": "valid-token"})
         self.assertEqual(response.status_code, 405)
-        self.assertFalse(mock_task.delay.called)
+        self.assertFalse(rfcindex_is_dirty())
         response = self.client.post(url, headers={"X-Api-Key": "valid-token"})
         self.assertEqual(response.status_code, 202)
-        self.assertTrue(mock_task.delay.called)
+        self.assertTrue(rfcindex_is_dirty())
diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
index 25982d76bf..6bc45fe3da 100644
--- a/ietf/api/views_rpc.py
+++ b/ietf/api/views_rpc.py
@@ -47,7 +47,7 @@
     update_rfc_searchindex_task,
 )
 from ietf.person.models import Email, Person
-from ietf.sync.tasks import create_rfc_index_task
+from ietf.sync.rfcindex import mark_rfcindex_as_dirty
 
 
 class Conflict(APIException):
@@ -548,5 +548,5 @@ class RfcIndexView(APIView):
         request=None,
     )
     def post(self, request):
-        create_rfc_index_task.delay()
+        mark_rfcindex_as_dirty()
         return Response(status=202)
diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py
index 0ea6fb939f..6864617874 100644
--- a/ietf/sync/rfcindex.py
+++ b/ietf/sync/rfcindex.py
@@ -1,4 +1,5 @@
 # Copyright The IETF Trust 2026, All Rights Reserved
+import datetime
 import json
 from collections import defaultdict
 from collections.abc import Container
@@ -11,6 +12,7 @@
 
 from django.conf import settings
 from django.core.files.base import ContentFile
+from django.db.models import Q
 from lxml import etree
 
 from django.core.files.storage import storages
@@ -22,6 +24,7 @@
 from ietf.doc.models import Document
 from ietf.name.models import StdLevelName
 from ietf.utils.log import log
+from ietf.utils.models import DirtyBits
 
 FORMATS_FOR_INDEX = ["txt", "html", "pdf", "xml", "ps"]
 SS_TXT_MARGIN = 3
@@ -739,3 +742,50 @@ def create_fyi_txt_index():
         },
     )
     save_to_red_bucket("fyi-index.txt", index)
+
+
+## DirtyBits management for the RFC index
+
+RFCINDEX_SLUG = DirtyBits.Slugs.RFCINDEX
+
+
+def mark_rfcindex_as_dirty():
+    _, created = DirtyBits.objects.update_or_create(
+        slug=RFCINDEX_SLUG, defaults={"dirty_time": timezone.now()}
+    )
+    if created:
+        log(f"Created DirtyBits(slug='{RFCINDEX_SLUG}')")
+
+
+def mark_rfcindex_as_processed(when: datetime.datetime):
+    n_updated = DirtyBits.objects.filter(
+        Q(processed_time__isnull=True) | Q(processed_time__lt=when),
+        slug=RFCINDEX_SLUG,
+    ).update(processed_time=when)
+    if n_updated > 0:
+        log(f"processed_time is now {when.isoformat()}")
+    else:
+        log("processed_time not updated, no matching record found")
+
+
+def rfcindex_is_dirty():
+    """Does the rfc index need to be updated?"""
+    dirty_work, created = DirtyBits.objects.get_or_create(
+        slug=RFCINDEX_SLUG, defaults={"dirty_time": timezone.now()}
+    )
+    if created:
+        log(f"Created DirtyBits(slug='{RFCINDEX_SLUG}')")
+    display_processed_time = (
+        dirty_work.processed_time.isoformat()
+        if dirty_work.processed_time is not None
+        else "never"
+    )
+    log(
+        f"DirtyBits(slug='{RFCINDEX_SLUG}'): "
+        f"dirty_time={dirty_work.dirty_time.isoformat()} "
+        f"processed_time={display_processed_time}"
+    )
+    return (
+        dirty_work.processed_time is None
+        or dirty_work.dirty_time >= dirty_work.processed_time
+    )
diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py
index c48368cccd..2805f431bf 100644
--- a/ietf/sync/tasks.py
+++ b/ietf/sync/tasks.py
@@ -24,6 +24,7 @@
     create_rfc_txt_index,
     create_rfc_xml_index,
     create_std_txt_index,
+    rfcindex_is_dirty, mark_rfcindex_as_processed,
 )
 from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper
 from ietf.utils import log
@@ -288,33 +289,40 @@ def load_rfcs_into_blobdb_task(start: int, end: int):
 
 
 @shared_task
-def create_rfc_index_task():
-    try:
-        create_rfc_txt_index()
-    except Exception as e:
-        log.log(f"Error: failure in creating rfc-index.txt. {e}")
-        pass
-
-    try:
-        create_rfc_xml_index()
-    except Exception as e:
-        log.log(f"Error: failure in creating rfc-index.xml. {e}")
-        pass
-
-    try:
-        create_bcp_txt_index()
-    except Exception as e:
-        log.log(f"Error: failure in creating bcp-index.txt. {e}")
-        pass
-
-    try:
-        create_std_txt_index()
-    except Exception as e:
-        log.log(f"Error: failure in creating std-index.txt. {e}")
-        pass
-
-    try:
-        create_fyi_txt_index()
-    except Exception as e:
-        log.log(f"Error: failure in creating fyi-index.txt. {e}")
-        pass
+def refresh_rfc_index_task():
+    if rfcindex_is_dirty():
+        # new_processed_time is the *start* of processing so that any changes after
+        # this point will trigger another refresh
+        new_processed_time = timezone.now()
+
+        try:
+            create_rfc_txt_index()
+        except Exception as e:
+            log.log(f"Error: failure in creating rfc-index.txt. {e}")
+            pass
+
+        try:
+            create_rfc_xml_index()
+        except Exception as e:
+            log.log(f"Error: failure in creating rfc-index.xml. {e}")
+            pass
+
+        try:
+            create_bcp_txt_index()
+        except Exception as e:
+            log.log(f"Error: failure in creating bcp-index.txt. {e}")
+            pass
+
+        try:
+            create_std_txt_index()
+        except Exception as e:
+            log.log(f"Error: failure in creating std-index.txt. {e}")
+            pass
+
+        try:
+            create_fyi_txt_index()
+        except Exception as e:
+            log.log(f"Error: failure in creating fyi-index.txt. {e}")
+            pass
+
+        mark_rfcindex_as_processed(new_processed_time)
diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py
index e6324ad7cd..cb8841cdc6 100644
--- a/ietf/utils/admin.py
+++ b/ietf/utils/admin.py
@@ -1,71 +1,30 @@
-# Copyright The IETF Trust 2011-2020, All Rights Reserved
-# -*- coding: utf-8 -*-
+# Copyright The IETF Trust 2011-2026, All Rights Reserved
 
 
 from django.contrib import admin
-from django.utils.encoding import force_str
-
-def name(obj):
-    if hasattr(obj, 'abbrev'):
-        return obj.abbrev()
-    elif hasattr(obj, 'name'):
-        if callable(obj.name):
-            name = obj.name()
-        else:
-            name = force_str(obj.name)
-        if name:
-            return name
-    return str(obj)
-    
-def admin_link(field, label=None, ordering="", display=name, suffix=""):
-    if not label:
-        label = field.capitalize().replace("_", " ").strip()
-    if ordering == "":
-        ordering = field
-    def _link(self):
-        obj = self
-        for attr in field.split("__"):
-            obj = getattr(obj, attr)
-            if callable(obj):
-                obj = obj()
-        if hasattr(obj, "all"):
-            objects = obj.all()
-        elif callable(obj):
-            objects = obj()
-            if not hasattr(objects, "__iter__"):
-                objects = [ objects ]
-        elif hasattr(obj, "__iter__"):
-            objects = obj
-        else:
-            objects = [ obj ]
-        chunks = []
-        for obj in objects:
-            app = obj._meta.app_label
-            model = obj.__class__.__name__.lower()
-            id = obj.pk
-            chunks += [ '<a href="/admin/%(app)s/%(model)s/%(id)s/%(suffix)s">%(display)s</a>' %
-                {'app':app, "model": model, "id":id, "display": display(obj), "suffix":suffix, } ]
-        return ", ".join(chunks)
-    _link.allow_tags = True
-    _link.short_description = label
-    _link.admin_order_field = ordering
-    return _link
+from .models import DumpInfo, DirtyBits
 
 
 class SaferStackedInline(admin.StackedInline):
     """StackedInline without delete by default"""
+
     can_delete = False  # no delete button
     show_change_link = True  # show a link to the resource (where it can be deleted)
 
 
 class SaferTabularInline(admin.TabularInline):
     """TabularInline without delete by default"""
+
     can_delete = False  # no delete button
     show_change_link = True  # show a link to the resource (where it can be deleted)
 
 
-from .models import DumpInfo
+@admin.register(DumpInfo)
 class DumpInfoAdmin(admin.ModelAdmin):
-    list_display = ['date', 'host', 'tz']
-    list_filter = ['date']
-admin.site.register(DumpInfo, DumpInfoAdmin)
+    list_display = ["date", "host", "tz"]
+    list_filter = ["date"]
+
+
+@admin.register(DirtyBits)
+class DirtyBitsAdmin(admin.ModelAdmin):
+    list_display = ["slug", "dirty_time", "processed_time"]
diff --git a/ietf/utils/migrations/0003_dirtybits.py b/ietf/utils/migrations/0003_dirtybits.py
new file mode 100644
index 0000000000..11f6ed09f6
--- /dev/null
+++ b/ietf/utils/migrations/0003_dirtybits.py
@@ -0,0 +1,37 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("utils", "0002_delete_versioninfo"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DirtyBits",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "slug",
+                    models.CharField(
+                        choices=[("rfcindex", "RFC Index")], max_length=40, unique=True
+                    ),
+                ),
+                ("dirty_time", models.DateTimeField(blank=True, null=True)),
+                ("processed_time", models.DateTimeField(blank=True, null=True)),
+            ],
+            options={
+                "verbose_name_plural": "dirty bits",
+            },
+        ),
+    ]
diff --git a/ietf/utils/models.py b/ietf/utils/models.py
index 21af5766e9..13afbdfe20 100644
--- a/ietf/utils/models.py
+++ b/ietf/utils/models.py
@@ -1,14 +1,35 @@
-# Copyright The IETF Trust 2015-2020, All Rights Reserved
+# Copyright The IETF Trust 2015-2026, All Rights Reserved
 
 import itertools
 
 from django.db import models
 
+
+class DirtyBits(models.Model):
+    """A weak semaphore mechanism for coordination with celery beat tasks
+
+    Web workers will set the "dirty_time" value for a given dirtybit slug.
+    Celery workers will do work if "processed_time" < "dirty_time" and update
+    "processed_time".
+    """
+
+    class Slugs(models.TextChoices):
+        RFCINDEX = "rfcindex", "RFC Index"
+
+    # next line can become `...choices=Slugs)` when we get to Django 5.x
+    slug = models.CharField(max_length=40, blank=False, choices=Slugs.choices, unique=True)
+    dirty_time = models.DateTimeField(null=True, blank=True)
+    processed_time = models.DateTimeField(null=True, blank=True)
+
+    class Meta:
+        verbose_name_plural = "dirty bits"
+
+    
 class DumpInfo(models.Model):
     date = models.DateTimeField()
     host = models.CharField(max_length=128)
     tz   = models.CharField(max_length=32, default='UTC')
-    
+
 class ForeignKey(models.ForeignKey):
     "A local ForeignKey proxy which provides the on_delete value required under Django 2.0."
     def __init__(self, to, on_delete=models.CASCADE, **kwargs):
diff --git a/ietf/utils/resources.py b/ietf/utils/resources.py
index 1252cfef14..63206eb33a 100644
--- a/ietf/utils/resources.py
+++ b/ietf/utils/resources.py
@@ -1,6 +1,4 @@
-# Copyright The IETF Trust 2014-2019, All Rights Reserved
-# -*- coding: utf-8 -*-
-# Autogenerated by the mkresources management command 2014-11-13 05:39
+# Copyright The IETF Trust 2014-2026, All Rights Reserved
 
 
 from ietf.api import ModelResource
@@ -12,7 +10,7 @@
 from django.contrib.contenttypes.models import ContentType
 
 from ietf import api
-from ietf.utils.models import DumpInfo
+from ietf.utils.models import DirtyBits, DumpInfo
 
 
 class UserResource(ModelResource):
@@ -43,3 +41,9 @@ class Meta:
             "host": ALL,
         }
 api.utils.register(DumpInfoResource())
+
+
+class DirtyBitsResource(ModelResource):
+    class Meta:
+        queryset = DirtyBits.objects.none()
+api.utils.register(DirtyBitsResource())

From 98d6bb51481fbba98aefafc4fa3c116e9fc9f57c Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 12:40:03 -0300
Subject: [PATCH 637/656] feat: utils to sync errata tags from errata.json

---
 ietf/sync/errata.py | 108 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 108 insertions(+)
 create mode 100644 ietf/sync/errata.py

diff --git a/ietf/sync/errata.py b/ietf/sync/errata.py
new file mode 100644
index 0000000000..ca826687ec
--- /dev/null
+++ b/ietf/sync/errata.py
@@ -0,0 +1,108 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+import datetime
+import json
+from collections import defaultdict
+from typing import DefaultDict, Literal
+
+from django.core.files.storage import storages
+
+from ietf.doc.models import Document, DocEvent
+from ietf.name.models import DocTagName
+from ietf.person.models import Person
+
+ERRATA_BLOB_NAME = "other/errata.json"  # name of errata.json in the red bucket
+
+def get_errata_last_updated() -> datetime.datetime:
+    """Get timestamp of the last errata.json update
+    
+    May raise FileNotFoundError or other storage/S3 exceptions. Be prepared.
+    """
+    red_bucket = storages["red_bucket"]
+    return red_bucket.get_modified_time(ERRATA_BLOB_NAME)
+
+
+def get_errata_data():
+    red_bucket = storages["red_bucket"]
+    with red_bucket.open(ERRATA_BLOB_NAME, "r") as f:
+        errata_data = json.load(f)
+    return errata_data
+
+
+def errata_map_from_json(errata_data):
+    """Create a dict mapping RFC number to a list of applicable errata records"""
+    errata = defaultdict(list)
+    for item in errata_data:
+        doc_id = item["doc-id"]
+        if doc_id.upper().startswith("RFC"):
+            rfc_number = int(doc_id[3:])
+            errata[rfc_number].append(item)
+    return dict(errata)
+
+
+def update_errata_tags(errata_data):
+    tag_has_errata = DocTagName.objects.get(slug="errata")
+    tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata")
+    system = Person.objects.get(name="(System)")
+
+    errata_map = errata_map_from_json(errata_data)
+    nums_with_errata = [
+        num
+        for num, errata in errata_map.items()
+        if any(er["errata_status_code"] != "Rejected" for er in errata)
+    ]
+    nums_with_verified_errata = [
+        num
+        for num, errata in errata_map.items()
+        if any(er["errata_status_code"] == "Verified" for er in errata)
+    ]
+
+    rfcs_gaining_errata_tag = Document.objects.filter(
+        type_id="rfc", rfc_number__in=nums_with_errata
+    ).exclude(tags=tag_has_errata)
+
+    rfcs_gaining_verified_errata_tag = Document.objects.filter(
+        type_id="rfc", rfc_number__in=nums_with_verified_errata
+    ).exclude(tags=tag_has_verified_errata)
+
+    rfcs_losing_errata_tag = Document.objects.filter(
+        type_id="rfc", tags=tag_has_errata
+    ).exclude(rfc_number__in=nums_with_errata)
+
+    rfcs_losing_verified_errata_tag = Document.objects.filter(
+        type_id="rfc", tags=tag_has_verified_errata
+    ).exclude(rfc_number__in=nums_with_verified_errata)
+
+    # map rfc_number to add/remove lists
+    changes: DefaultDict[Document, dict[str, list[DocTagName]]] = defaultdict(
+        lambda: {"add": [], "remove": []}
+    ) 
+    for rfc in rfcs_gaining_errata_tag:
+        changes[rfc]["add"].append(tag_has_errata)
+    for rfc in rfcs_gaining_verified_errata_tag:    
+        changes[rfc]["add"].append(tag_has_verified_errata)
+    for rfc in rfcs_losing_errata_tag:
+        changes[rfc]["remove"].append(tag_has_errata)
+    for rfc in rfcs_losing_verified_errata_tag:
+        changes[rfc]["remove"].append(tag_has_verified_errata)
+
+    for rfc, changeset in changes.items():
+        change_descs = []
+        for tag in changeset["add"]:
+            rfc.tags.add(tag)
+            change_descs.append(f"added {tag.slug} tag")
+        for tag in changeset["remove"]:
+            rfc.tags.remove(tag)
+            change_descs.append(f"removed {tag.slug} tag")
+        summary = "Update from RFC Editor: " + ", ".join(change_descs)
+        if all(
+            er["errata_status_code"] == "Rejected"
+            for er in errata_map[rfc.rfc_number]
+        ):
+            summary += " (all errata rejected)"
+        DocEvent.objects.create(
+            doc=rfc,
+            rev=rfc.rev,  # expect no rev
+            by=system,
+            type="sync_from_rfc_editor",
+            desc=summary
+        )

From c01380714f0c7d2b0e44dc5a5c9a8f11d3c73cf6 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 13:56:13 -0300
Subject: [PATCH 638/656] feat: DirtyBits for errata processing

---
 ietf/sync/errata.py                           | 57 +++++++++++++++++++
 .../migrations/0004_alter_dirtybits_slug.py   | 21 +++++++
 ietf/utils/models.py                          |  1 +
 3 files changed, 79 insertions(+)
 create mode 100644 ietf/utils/migrations/0004_alter_dirtybits_slug.py

diff --git a/ietf/sync/errata.py b/ietf/sync/errata.py
index ca826687ec..1e8e122eae 100644
--- a/ietf/sync/errata.py
+++ b/ietf/sync/errata.py
@@ -5,10 +5,13 @@
 from typing import DefaultDict, Literal
 
 from django.core.files.storage import storages
+from django.db.models import Q
 
 from ietf.doc.models import Document, DocEvent
 from ietf.name.models import DocTagName
 from ietf.person.models import Person
+from ietf.utils.log import log
+from ietf.utils.models import DirtyBits
 
 ERRATA_BLOB_NAME = "other/errata.json"  # name of errata.json in the red bucket
 
@@ -106,3 +109,57 @@ def update_errata_tags(errata_data):
             type="sync_from_rfc_editor",
             desc=summary
         )
+
+
+## DirtyBits management for the errata tags
+
+ERRATA_SLUG = DirtyBits.Slugs.ERRATA
+
+
+def update_errata_dirty_time() -> DirtyBits | None:
+    try:
+        last_update = get_errata_last_updated()
+    except Exception as err:
+        log(f"Error in get_errata_last_updated: {err}")
+        return None
+    else:
+        dirty_work, created = DirtyBits.objects.update_or_create(
+            slug=ERRATA_SLUG, defaults={"dirty_time": last_update}
+        )
+        if created:
+            log(f"Created DirtyBits(slug='{ERRATA_SLUG}')")
+        return dirty_work
+
+def mark_errata_as_processed(when: datetime.datetime):
+    n_updated = DirtyBits.objects.filter(
+        Q(processed_time__isnull=True) | Q(processed_time__lt=when),
+        slug=ERRATA_SLUG,
+    ).update(processed_time=when)
+    if n_updated > 0:
+        log(f"processed_time is now {when.isoformat()}")
+    else:
+        log("processed_time not updated, no matching record found")
+
+
+def errata_are_dirty():
+    """Does the rfc index need to be updated?"""
+    dirty_work = update_errata_dirty_time()  # creates DirtyBits if needed
+    if dirty_work is None:
+        # A None indicates we could not check the timestamp of errata.json. In that
+        # case, we are not likely to be able to read the blob either, so don't try
+        # to process it. An error was already logged.
+        return False 
+    display_processed_time = (
+        dirty_work.processed_time.isoformat()
+        if dirty_work.processed_time is not None
+        else "never"
+    )
+    log(
+        f"DirtyBits(slug='{ERRATA_SLUG}'): "
+        f"dirty_time={dirty_work.dirty_time.isoformat()} "
+        f"processed_time={display_processed_time}"
+    )
+    return (
+        dirty_work.processed_time is None
+        or dirty_work.dirty_time >= dirty_work.processed_time
+    )
diff --git a/ietf/utils/migrations/0004_alter_dirtybits_slug.py b/ietf/utils/migrations/0004_alter_dirtybits_slug.py
new file mode 100644
index 0000000000..e17ea6cadd
--- /dev/null
+++ b/ietf/utils/migrations/0004_alter_dirtybits_slug.py
@@ -0,0 +1,21 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("utils", "0003_dirtybits"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="dirtybits",
+            name="slug",
+            field=models.CharField(
+                choices=[("rfcindex", "RFC Index"), ("errata", "Errata Tags")],
+                max_length=40,
+                unique=True,
+            ),
+        ),
+    ]
diff --git a/ietf/utils/models.py b/ietf/utils/models.py
index 13afbdfe20..64f7f253f2 100644
--- a/ietf/utils/models.py
+++ b/ietf/utils/models.py
@@ -15,6 +15,7 @@ class DirtyBits(models.Model):
 
     class Slugs(models.TextChoices):
         RFCINDEX = "rfcindex", "RFC Index"
+        ERRATA = "errata", "Errata Tags"
 
     # next line can become `...choices=Slugs)` when we get to Django 5.x
     slug = models.CharField(max_length=40, blank=False, choices=Slugs.choices, unique=True)

From 070c0595c58788b159c113004d749da962cf8449 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 14:03:21 -0300
Subject: [PATCH 639/656] feat: errata update task

---
 ietf/sync/tasks.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py
index 2805f431bf..357075620f 100644
--- a/ietf/sync/tasks.py
+++ b/ietf/sync/tasks.py
@@ -17,6 +17,8 @@
 from ietf.doc.tasks import rebuild_reference_relations_task
 from ietf.sync import iana
 from ietf.sync import rfceditor
+from ietf.sync.errata import errata_are_dirty, mark_errata_as_processed, get_errata_data, \
+    update_errata_tags
 from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue
 from ietf.sync.rfcindex import (
     create_bcp_txt_index,
@@ -288,6 +290,17 @@ def load_rfcs_into_blobdb_task(start: int, end: int):
     load_rfcs_into_blobdb(list(range(start, end + 1)))
 
 
+@shared_task
+def update_errata_from_rfceditor_task():
+    if errata_are_dirty():
+        # new_processed_time is the *start* of processing so that any changes after
+        # this point will trigger another refresh
+        new_processed_time = timezone.now()
+        errata_data = get_errata_data()
+        update_errata_tags(errata_data)
+        mark_errata_as_processed(new_processed_time)
+
+
 @shared_task
 def refresh_rfc_index_task():
     if rfcindex_is_dirty():

From 9f1e5ceb1d90439653920c7080e6e1fb924e762f Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 14:20:10 -0300
Subject: [PATCH 640/656] feat: rebuild rfc indexes after errata update

---
 ietf/sync/tasks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py
index 357075620f..ac0fe173ca 100644
--- a/ietf/sync/tasks.py
+++ b/ietf/sync/tasks.py
@@ -26,7 +26,7 @@
     create_rfc_txt_index,
     create_rfc_xml_index,
     create_std_txt_index,
-    rfcindex_is_dirty, mark_rfcindex_as_processed,
+    rfcindex_is_dirty, mark_rfcindex_as_processed, mark_rfcindex_as_dirty,
 )
 from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper
 from ietf.utils import log
@@ -299,6 +299,7 @@ def update_errata_from_rfceditor_task():
         errata_data = get_errata_data()
         update_errata_tags(errata_data)
         mark_errata_as_processed(new_processed_time)
+        mark_rfcindex_as_dirty()  # ensure any changes are reflected in the indexes
 
 
 @shared_task

From 4dd9f2a82569b85b6d2e74e418a43b7382edb8af Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 14:32:32 -0300
Subject: [PATCH 641/656] chore: setting for ERRATA_JSON_BLOB_NAME

---
 ietf/sync/errata.py   | 27 +++++++++++++++++----------
 k8s/settings_local.py |  5 +++++
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/ietf/sync/errata.py b/ietf/sync/errata.py
index 1e8e122eae..0d154bf86e 100644
--- a/ietf/sync/errata.py
+++ b/ietf/sync/errata.py
@@ -4,6 +4,7 @@
 from collections import defaultdict
 from typing import DefaultDict, Literal
 
+from django.conf import settings
 from django.core.files.storage import storages
 from django.db.models import Q
 
@@ -13,20 +14,26 @@
 from ietf.utils.log import log
 from ietf.utils.models import DirtyBits
 
-ERRATA_BLOB_NAME = "other/errata.json"  # name of errata.json in the red bucket
+
+DEFAULT_ERRATA_JSON_BLOB_NAME = "other/errata.json"
+
 
 def get_errata_last_updated() -> datetime.datetime:
     """Get timestamp of the last errata.json update
-    
+
     May raise FileNotFoundError or other storage/S3 exceptions. Be prepared.
     """
     red_bucket = storages["red_bucket"]
-    return red_bucket.get_modified_time(ERRATA_BLOB_NAME)
+    return red_bucket.get_modified_time(
+        getattr(settings, "ERRATA_JSON_BLOB_NAME", DEFAULT_ERRATA_JSON_BLOB_NAME)
+    )
 
 
 def get_errata_data():
     red_bucket = storages["red_bucket"]
-    with red_bucket.open(ERRATA_BLOB_NAME, "r") as f:
+    with red_bucket.open(
+        getattr(settings, "ERRATA_JSON_BLOB_NAME", DEFAULT_ERRATA_JSON_BLOB_NAME), "r"
+    ) as f:
         errata_data = json.load(f)
     return errata_data
 
@@ -78,10 +85,10 @@ def update_errata_tags(errata_data):
     # map rfc_number to add/remove lists
     changes: DefaultDict[Document, dict[str, list[DocTagName]]] = defaultdict(
         lambda: {"add": [], "remove": []}
-    ) 
+    )
     for rfc in rfcs_gaining_errata_tag:
         changes[rfc]["add"].append(tag_has_errata)
-    for rfc in rfcs_gaining_verified_errata_tag:    
+    for rfc in rfcs_gaining_verified_errata_tag:
         changes[rfc]["add"].append(tag_has_verified_errata)
     for rfc in rfcs_losing_errata_tag:
         changes[rfc]["remove"].append(tag_has_errata)
@@ -98,8 +105,7 @@ def update_errata_tags(errata_data):
             change_descs.append(f"removed {tag.slug} tag")
         summary = "Update from RFC Editor: " + ", ".join(change_descs)
         if all(
-            er["errata_status_code"] == "Rejected"
-            for er in errata_map[rfc.rfc_number]
+            er["errata_status_code"] == "Rejected" for er in errata_map[rfc.rfc_number]
         ):
             summary += " (all errata rejected)"
         DocEvent.objects.create(
@@ -107,7 +113,7 @@ def update_errata_tags(errata_data):
             rev=rfc.rev,  # expect no rev
             by=system,
             type="sync_from_rfc_editor",
-            desc=summary
+            desc=summary,
         )
 
 
@@ -130,6 +136,7 @@ def update_errata_dirty_time() -> DirtyBits | None:
             log(f"Created DirtyBits(slug='{ERRATA_SLUG}')")
         return dirty_work
 
+
 def mark_errata_as_processed(when: datetime.datetime):
     n_updated = DirtyBits.objects.filter(
         Q(processed_time__isnull=True) | Q(processed_time__lt=when),
@@ -148,7 +155,7 @@ def errata_are_dirty():
         # A None indicates we could not check the timestamp of errata.json. In that
         # case, we are not likely to be able to read the blob either, so don't try
         # to process it. An error was already logged.
-        return False 
+        return False
     display_processed_time = (
         dirty_work.processed_time.isoformat()
         if dirty_work.processed_time is not None
diff --git a/k8s/settings_local.py b/k8s/settings_local.py
index 3a7e524f5a..251f11234f 100644
--- a/k8s/settings_local.py
+++ b/k8s/settings_local.py
@@ -511,3 +511,8 @@ def _multiline_to_list(s):
             "DATATRACKER_ERRATA_METADATA_NOTIFICATION_URL must be set if "
             "DATATRACKER_ERRATA_METADATA_NOTIFICATION_API_KEY is provided"
         )
+
+# name (with path) of errata.json in the red bucket
+ERRATA_JSON_BLOB_NAME = os.environ.get(
+    "DATATRACKER_ERRATA_JSON_BLOB_NAME", "other/errata.json"
+)

From 1b2aced58e6fa43ff5c9579964ac4684fb87824f Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 15:00:35 -0300
Subject: [PATCH 642/656] fix: transaction

---
 ietf/sync/errata.py | 43 ++++++++++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 19 deletions(-)

diff --git a/ietf/sync/errata.py b/ietf/sync/errata.py
index 0d154bf86e..51531a6190 100644
--- a/ietf/sync/errata.py
+++ b/ietf/sync/errata.py
@@ -6,6 +6,7 @@
 
 from django.conf import settings
 from django.core.files.storage import storages
+from django.db import transaction
 from django.db.models import Q
 
 from ietf.doc.models import Document, DocEvent
@@ -96,25 +97,29 @@ def update_errata_tags(errata_data):
         changes[rfc]["remove"].append(tag_has_verified_errata)
 
     for rfc, changeset in changes.items():
-        change_descs = []
-        for tag in changeset["add"]:
-            rfc.tags.add(tag)
-            change_descs.append(f"added {tag.slug} tag")
-        for tag in changeset["remove"]:
-            rfc.tags.remove(tag)
-            change_descs.append(f"removed {tag.slug} tag")
-        summary = "Update from RFC Editor: " + ", ".join(change_descs)
-        if all(
-            er["errata_status_code"] == "Rejected" for er in errata_map[rfc.rfc_number]
-        ):
-            summary += " (all errata rejected)"
-        DocEvent.objects.create(
-            doc=rfc,
-            rev=rfc.rev,  # expect no rev
-            by=system,
-            type="sync_from_rfc_editor",
-            desc=summary,
-        )
+        # Update in a transaction per RFC to keep tags and DocEvents consistent.
+        # With this in place, an interrupted task will be cleanly completed on the
+        # next run.
+        with transaction.atomic():
+            change_descs = []
+            for tag in changeset["add"]:
+                rfc.tags.add(tag)
+                change_descs.append(f"added {tag.slug} tag")
+            for tag in changeset["remove"]:
+                rfc.tags.remove(tag)
+                change_descs.append(f"removed {tag.slug} tag")
+            summary = "Update from RFC Editor: " + ", ".join(change_descs)
+            if all(
+                er["errata_status_code"] == "Rejected" for er in errata_map[rfc.rfc_number]
+            ):
+                summary += " (all errata rejected)"
+            DocEvent.objects.create(
+                doc=rfc,
+                rev=rfc.rev,  # expect no rev
+                by=system,
+                type="sync_from_rfc_editor",
+                desc=summary,
+            )
 
 
 ## DirtyBits management for the errata tags

From 5bdbdf1ea961542f9aa36e9286222cea37598c41 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 15:14:13 -0300
Subject: [PATCH 643/656] refactor: extract method from task

---
 ietf/sync/errata.py |  5 +++++
 ietf/sync/tasks.py  | 10 ++++++----
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/ietf/sync/errata.py b/ietf/sync/errata.py
index 51531a6190..81a31e7484 100644
--- a/ietf/sync/errata.py
+++ b/ietf/sync/errata.py
@@ -175,3 +175,8 @@ def errata_are_dirty():
         dirty_work.processed_time is None
         or dirty_work.dirty_time >= dirty_work.processed_time
     )
+
+
+def update_errata_from_rfceditor():
+    errata_data = get_errata_data()
+    update_errata_tags(errata_data)
diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py
index ac0fe173ca..34b2efeb5c 100644
--- a/ietf/sync/tasks.py
+++ b/ietf/sync/tasks.py
@@ -17,8 +17,11 @@
 from ietf.doc.tasks import rebuild_reference_relations_task
 from ietf.sync import iana
 from ietf.sync import rfceditor
-from ietf.sync.errata import errata_are_dirty, mark_errata_as_processed, get_errata_data, \
-    update_errata_tags
+from ietf.sync.errata import (
+    errata_are_dirty,
+    mark_errata_as_processed,
+    update_errata_from_rfceditor,
+)
 from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue
 from ietf.sync.rfcindex import (
     create_bcp_txt_index,
@@ -296,8 +299,7 @@ def update_errata_from_rfceditor_task():
         # new_processed_time is the *start* of processing so that any changes after
         # this point will trigger another refresh
         new_processed_time = timezone.now()
-        errata_data = get_errata_data()
-        update_errata_tags(errata_data)
+        update_errata_from_rfceditor()
         mark_errata_as_processed(new_processed_time)
         mark_rfcindex_as_dirty()  # ensure any changes are reflected in the indexes
 

From 2c385a0f95a991ff547152dba2f700e7a7c8e3ec Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 15:15:32 -0300
Subject: [PATCH 644/656] test: test_update_errata_from_rfceditor_task

---
 ietf/sync/tests.py | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index 21d6cb5cd5..6cbd91d59f 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -1,5 +1,4 @@
-# Copyright The IETF Trust 2012-2020, All Rights Reserved
-# -*- coding: utf-8 -*-
+# Copyright The IETF Trust 2012-2026, All Rights Reserved
 
 
 import os
@@ -33,6 +32,7 @@
 from ietf.person.factories import PersonFactory
 from ietf.person.models import Person
 from ietf.sync import iana, rfceditor, tasks
+from ietf.sync.tasks import update_errata_from_rfceditor_task
 from ietf.utils.mail import outbox, empty_outbox
 from ietf.utils.test_utils import login_testing_unauthorized
 from ietf.utils.test_utils import TestCase
@@ -1215,3 +1215,28 @@ def test_load_rfcs_into_blobdb_task(
         self.assertEqual(mock_kwargs, {})
 
 
+    @mock.patch("ietf.sync.tasks.update_errata_from_rfceditor")
+    @mock.patch("ietf.sync.tasks.mark_rfcindex_as_dirty")
+    @mock.patch("ietf.sync.tasks.mark_errata_as_processed")
+    @mock.patch("ietf.sync.tasks.errata_are_dirty")
+    def test_update_errata_from_rfceditor_task(
+        self,
+        mock_errata_are_dirty,
+        mock_mark_errata_processed,
+        mock_mark_rfcindex_dirty,
+        mock_update,
+    ):
+        mock_errata_are_dirty.return_value = False
+        update_errata_from_rfceditor_task()
+        self.assertTrue(mock_errata_are_dirty.called)
+        self.assertFalse(mock_mark_errata_processed.called)
+        self.assertFalse(mock_mark_rfcindex_dirty.called)
+        self.assertFalse(mock_update.called)
+
+        mock_errata_are_dirty.reset_mock()
+        mock_errata_are_dirty.return_value = True
+        update_errata_from_rfceditor_task()
+        self.assertTrue(mock_errata_are_dirty.called)
+        self.assertTrue(mock_mark_errata_processed.called)
+        self.assertTrue(mock_mark_rfcindex_dirty.called)
+        self.assertTrue(mock_update.called)

From 3a4e248570e7373e0de8c2281af2df6d485919e1 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 15:24:12 -0300
Subject: [PATCH 645/656] refactor: slight reorg in errata.py

---
 ietf/sync/errata.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/ietf/sync/errata.py b/ietf/sync/errata.py
index 81a31e7484..8a4c13c3a9 100644
--- a/ietf/sync/errata.py
+++ b/ietf/sync/errata.py
@@ -122,6 +122,11 @@ def update_errata_tags(errata_data):
             )
 
 
+def update_errata_from_rfceditor():
+    errata_data = get_errata_data()
+    update_errata_tags(errata_data)
+
+
 ## DirtyBits management for the errata tags
 
 ERRATA_SLUG = DirtyBits.Slugs.ERRATA
@@ -175,8 +180,3 @@ def errata_are_dirty():
         dirty_work.processed_time is None
         or dirty_work.dirty_time >= dirty_work.processed_time
     )
-
-
-def update_errata_from_rfceditor():
-    errata_data = get_errata_data()
-    update_errata_tags(errata_data)

From a568046bbed4656005a64ce97e8f1e58b46ede9b Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 19:43:01 -0300
Subject: [PATCH 646/656] test: test errata module

---
 ietf/sync/tests.py | 193 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 192 insertions(+), 1 deletion(-)

diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index 6cbd91d59f..eb70bcf9e9 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -12,6 +12,8 @@
 from dataclasses import dataclass
 
 from django.conf import settings
+from django.core.files.base import ContentFile
+from django.core.files.storage import storages
 from django.urls import reverse as urlreverse
 from django.utils import timezone
 from django.test.utils import override_settings
@@ -24,7 +26,7 @@
     RfcFactory,
     DocumentAuthorFactory,
     DocEventFactory,
-    BcpFactory,
+    BcpFactory, WgRfcFactory,
 )
 from ietf.doc.models import Document, DocEvent, DeletedEvent, DocTagName, RelatedDocument, State, StateDocEvent
 from ietf.doc.utils import add_state_change_event
@@ -32,8 +34,12 @@
 from ietf.person.factories import PersonFactory
 from ietf.person.models import Person
 from ietf.sync import iana, rfceditor, tasks
+from ietf.sync.errata import update_errata_from_rfceditor, get_errata_last_updated, \
+    get_errata_data, errata_map_from_json, update_errata_dirty_time, \
+    mark_errata_as_processed, update_errata_tags
 from ietf.sync.tasks import update_errata_from_rfceditor_task
 from ietf.utils.mail import outbox, empty_outbox
+from ietf.utils.models import DirtyBits
 from ietf.utils.test_utils import login_testing_unauthorized
 from ietf.utils.test_utils import TestCase
 from ietf.utils.timezone import date_today, RPC_TZINFO
@@ -882,6 +888,191 @@ def test_rfceditor_undo(self):
         self.assertTrue(StateDocEvent.objects.filter(desc="First", doc=draft))
 
 
+class ErrataTests(TestCase):
+    @override_settings(ERRATA_JSON_BLOB_NAME="myblob.json")
+    def test_get_errata_last_update(self):
+        red_bucket = storages["red_bucket"]  # InMemoryStorage in test
+        red_bucket.save("myblob.json", ContentFile("file"))
+        self.assertEqual(
+            get_errata_last_updated(), red_bucket.get_modified_time("myblob.json")
+        )
+    
+    @override_settings(ERRATA_JSON_BLOB_NAME="myblob.json")
+    def test_get_errata_data(self):
+        red_bucket = storages["red_bucket"]  # InMemoryStorage in test
+        red_bucket.save("myblob.json", ContentFile('[{"value": 3}]'))
+        self.assertEqual(
+            get_errata_data(),
+            [{"value": 3}],
+        )
+
+    def test_errata_map_from_json(self):
+        input_data = [
+            {
+                "doc-id": "not-an-rfc",
+                "errata_status_code": "Verified",
+            },
+            {
+                "doc-id": "rfc01234",
+                "errata_status_code": "Reported",
+            },
+            {
+                "doc-id": "RFC1001",
+                "errata_status_code": "Verified"
+            },
+            {
+                "doc-id": "RfC1234",
+                "errata_status_code": "Verified",
+            },
+        ]
+        expected_output = {1001: [input_data[2]], 1234: [input_data[1], input_data[3]]}
+        self.assertDictEqual(errata_map_from_json(input_data), expected_output)
+
+    @mock.patch("ietf.sync.errata.update_errata_tags")
+    @mock.patch("ietf.sync.errata.get_errata_data")
+    def test_update_errata_from_rfceditor(self, mock_get_data, mock_update):
+        fake_data = object()
+        mock_get_data.return_value = fake_data
+        update_errata_from_rfceditor()
+        self.assertTrue(mock_get_data.called)
+        self.assertTrue(mock_update.called)
+        self.assertEqual(mock_update.call_args, mock.call(fake_data))
+
+    def test_update_errata_tags(self):
+        tag_has_errata = DocTagName.objects.get(slug="errata")
+        tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata")
+
+        rfcs = WgRfcFactory.create_batch(10)
+        rfcs[0].tags.set([tag_has_errata])
+        rfcs[1].tags.set([tag_has_errata, tag_has_verified_errata])
+        rfcs[2].tags.set([tag_has_errata])
+        rfcs[3].tags.set([tag_has_errata, tag_has_verified_errata])
+        rfcs[4].tags.set([tag_has_errata])
+        rfcs[5].tags.set([tag_has_errata, tag_has_verified_errata])
+
+        # Only contains the fields we care about, not the full JSON
+        errata_data = [
+            # rfcs[0] had errata and should keep it
+            {"doc-id": rfcs[0].name, "errata_status_code": "Held for Document Update"},
+            {"doc-id": rfcs[0].name, "errata_status_code": "Rejected"},
+            # rfcs[1] had errata+verified-errata and should keep both
+            {"doc-id": rfcs[1].name, "errata_status_code": "Verified"},
+            # rfcs[2] had errata and should gain verified-errata
+            {"doc-id": rfcs[2].name, "errata_status_code": "Verified"},
+            # rfcs[3] had errata+verified errata and should lose both
+            {"doc-id": rfcs[3].name, "errata_status_code": "Rejected"},
+            # rfcs[4] had errata and should gain verified-errata
+            {"doc-id": rfcs[4].name, "errata_status_code": "Verified"},
+            {"doc-id": rfcs[4].name, "errata_status_code": "Reported"},
+            # rfcs[5] had errata+verified-errata and should lose verified-errata
+            {"doc-id": rfcs[5].name, "errata_status_code": "Reported"},
+            # rfcs[6] had none and should gain errata
+            {"doc-id": rfcs[6].name, "errata_status_code": "Reported"},
+            # rfcs[7] had none and should gain errata+verified-errata
+            {"doc-id": rfcs[7].name, "errata_status_code": "Verified"},
+            # rfcs[8] had none and it should stay that way
+            {"doc-id": rfcs[8].name, "errata_status_code": "Rejected"},
+            # rfcs[9] had none and it should stay that way (no entry at all)
+        ]
+        update_errata_tags(errata_data)
+
+        self.assertCountEqual(rfcs[0].tags.all(), [tag_has_errata])
+        self.assertIsNone(rfcs[0].docevent_set.first())  # no change
+
+        self.assertCountEqual(
+            rfcs[1].tags.all(), [tag_has_errata, tag_has_verified_errata]
+        )
+        self.assertIsNone(rfcs[1].docevent_set.first())  # no change
+
+        self.assertCountEqual(
+            rfcs[2].tags.all(), [tag_has_errata, tag_has_verified_errata]
+        )
+        self.assertEqual(rfcs[2].docevent_set.count(), 1)
+        self.assertIn(": added verified-errata tag", rfcs[2].docevent_set.first().desc)
+
+        self.assertCountEqual(rfcs[3].tags.all(), [])
+        self.assertEqual(rfcs[3].docevent_set.count(), 1)
+        self.assertIn(
+            ": removed errata tag, removed verified-errata tag (all errata rejected)",
+            rfcs[3].docevent_set.first().desc,
+        )
+
+        self.assertCountEqual(
+            rfcs[4].tags.all(), [tag_has_errata, tag_has_verified_errata]
+        )
+        self.assertEqual(rfcs[4].docevent_set.count(), 1)
+        self.assertIn(": added verified-errata tag", rfcs[4].docevent_set.first().desc)
+
+        self.assertCountEqual(rfcs[5].tags.all(), [tag_has_errata])
+        self.assertEqual(rfcs[5].docevent_set.count(), 1)
+        self.assertIn(
+            ": removed verified-errata tag", rfcs[5].docevent_set.first().desc
+        )
+
+        self.assertCountEqual(rfcs[6].tags.all(), [tag_has_errata])
+        self.assertEqual(rfcs[6].docevent_set.count(), 1)
+        self.assertIn(": added errata tag", rfcs[6].docevent_set.first().desc)
+
+        self.assertCountEqual(
+            rfcs[7].tags.all(), [tag_has_errata, tag_has_verified_errata]
+        )
+        self.assertEqual(rfcs[7].docevent_set.count(), 1)
+        self.assertIn(
+            ": added errata tag, added verified-errata tag",
+            rfcs[7].docevent_set.first().desc,
+        )
+
+        self.assertCountEqual(rfcs[8].tags.all(), [])
+        self.assertIsNone(rfcs[8].docevent_set.first())  # no change
+
+        self.assertCountEqual(rfcs[9].tags.all(), [])
+        self.assertIsNone(rfcs[9].docevent_set.first())  # no change
+
+    @override_settings(ERRATA_JSON_BLOB_NAME="myblob.json")
+    @mock.patch("ietf.sync.errata.get_errata_last_updated")
+    def test_update_errata_dirty_time(self, mock_last_updated):
+        ERRATA_SLUG = DirtyBits.Slugs.ERRATA
+        
+        # No time available
+        mock_last_updated.side_effect = FileNotFoundError
+        self.assertIsNone(DirtyBits.objects.filter(slug=ERRATA_SLUG).first())
+        self.assertIsNone(update_errata_dirty_time())  # no blob yet
+        self.assertIsNone(DirtyBits.objects.filter(slug=ERRATA_SLUG).first())
+
+        # Now set a time
+        first_timestamp = timezone.now() - datetime.timedelta(hours=3)
+        mock_last_updated.return_value = first_timestamp
+        mock_last_updated.side_effect = None
+        result = update_errata_dirty_time()
+        self.assertTrue(isinstance(result, DirtyBits))
+        result.refresh_from_db()
+        self.assertEqual(result.slug, ERRATA_SLUG)
+        self.assertEqual(result.processed_time, None)
+        self.assertEqual(result.dirty_time, first_timestamp)
+        
+        # Update the time
+        second_timestamp = timezone.now()
+        mock_last_updated.return_value = second_timestamp
+        second_result = update_errata_dirty_time()
+        self.assertEqual(result.pk, second_result.pk)  # should be the same record
+        result.refresh_from_db()
+        self.assertEqual(result.slug, ERRATA_SLUG)
+        self.assertEqual(result.processed_time, None)
+        self.assertEqual(result.dirty_time, second_timestamp)
+
+    def test_mark_errata_as_processed(self):
+        ERRATA_SLUG = DirtyBits.Slugs.ERRATA
+        first_timestamp = timezone.now()
+        mark_errata_as_processed(first_timestamp)  # no DirtyBits is not an error
+        self.assertIsNone(DirtyBits.objects.filter(slug=ERRATA_SLUG).first())
+        dbits = DirtyBits.objects.create(slug=ERRATA_SLUG, dirty_time=first_timestamp)
+        second_timestamp = timezone.now()
+        mark_errata_as_processed(second_timestamp)
+        dbits.refresh_from_db()
+        self.assertEqual(dbits.dirty_time, first_timestamp)
+        self.assertEqual(dbits.processed_time, second_timestamp)
+        
+
 class TaskTests(TestCase):
     @override_settings(
         RFC_EDITOR_INDEX_URL="https://rfc-editor.example.com/index/",

From 96e227b89b349e67c65e6f7149577974ed3021dd Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 19:43:17 -0300
Subject: [PATCH 647/656] fix: guard map lookup

---
 ietf/sync/errata.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ietf/sync/errata.py b/ietf/sync/errata.py
index 8a4c13c3a9..9f71cf111b 100644
--- a/ietf/sync/errata.py
+++ b/ietf/sync/errata.py
@@ -109,8 +109,9 @@ def update_errata_tags(errata_data):
                 rfc.tags.remove(tag)
                 change_descs.append(f"removed {tag.slug} tag")
             summary = "Update from RFC Editor: " + ", ".join(change_descs)
-            if all(
-                er["errata_status_code"] == "Rejected" for er in errata_map[rfc.rfc_number]
+            if rfc.rfc_number in errata_map and all(
+                er["errata_status_code"] == "Rejected"
+                for er in errata_map[rfc.rfc_number]
             ):
                 summary += " (all errata rejected)"
             DocEvent.objects.create(

From a34917937cafd0637a22e60e06e1dbcad72dcc53 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Tue, 14 Apr 2026 20:04:06 -0300
Subject: [PATCH 648/656] fix: lint / typing

---
 ietf/sync/errata.py |  9 +++++----
 ietf/sync/tests.py  | 22 ++++++++++++++++++----
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/ietf/sync/errata.py b/ietf/sync/errata.py
index 9f71cf111b..113d987291 100644
--- a/ietf/sync/errata.py
+++ b/ietf/sync/errata.py
@@ -2,7 +2,7 @@
 import datetime
 import json
 from collections import defaultdict
-from typing import DefaultDict, Literal
+from typing import DefaultDict
 
 from django.conf import settings
 from django.core.files.storage import storages
@@ -18,6 +18,7 @@
 
 DEFAULT_ERRATA_JSON_BLOB_NAME = "other/errata.json"
 
+type ErrataJsonEntry = dict[str, str]
 
 def get_errata_last_updated() -> datetime.datetime:
     """Get timestamp of the last errata.json update
@@ -30,7 +31,7 @@ def get_errata_last_updated() -> datetime.datetime:
     )
 
 
-def get_errata_data():
+def get_errata_data() -> list[ErrataJsonEntry]:
     red_bucket = storages["red_bucket"]
     with red_bucket.open(
         getattr(settings, "ERRATA_JSON_BLOB_NAME", DEFAULT_ERRATA_JSON_BLOB_NAME), "r"
@@ -39,7 +40,7 @@ def get_errata_data():
     return errata_data
 
 
-def errata_map_from_json(errata_data):
+def errata_map_from_json(errata_data: list[ErrataJsonEntry]):
     """Create a dict mapping RFC number to a list of applicable errata records"""
     errata = defaultdict(list)
     for item in errata_data:
@@ -50,7 +51,7 @@ def errata_map_from_json(errata_data):
     return dict(errata)
 
 
-def update_errata_tags(errata_data):
+def update_errata_tags(errata_data: list[ErrataJsonEntry]):
     tag_has_errata = DocTagName.objects.get(slug="errata")
     tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata")
     system = Person.objects.get(name="(System)")
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index eb70bcf9e9..e83b6a5e0a 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -28,15 +28,29 @@
     DocEventFactory,
     BcpFactory, WgRfcFactory,
 )
-from ietf.doc.models import Document, DocEvent, DeletedEvent, DocTagName, RelatedDocument, State, StateDocEvent
+from ietf.doc.models import (
+    Document,
+    DocEvent,
+    DeletedEvent,
+    DocTagName,
+    RelatedDocument,
+    State,
+    StateDocEvent,
+)
 from ietf.doc.utils import add_state_change_event
 from ietf.group.factories import GroupFactory
 from ietf.person.factories import PersonFactory
 from ietf.person.models import Person
 from ietf.sync import iana, rfceditor, tasks
-from ietf.sync.errata import update_errata_from_rfceditor, get_errata_last_updated, \
-    get_errata_data, errata_map_from_json, update_errata_dirty_time, \
-    mark_errata_as_processed, update_errata_tags
+from ietf.sync.errata import (
+    update_errata_from_rfceditor,
+    get_errata_last_updated,
+    get_errata_data,
+    errata_map_from_json,
+    update_errata_dirty_time,
+    mark_errata_as_processed,
+    update_errata_tags,
+)
 from ietf.sync.tasks import update_errata_from_rfceditor_task
 from ietf.utils.mail import outbox, empty_outbox
 from ietf.utils.models import DirtyBits

From 15b2abd937f5d6e2b71667501309a965fdd9dc9d Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 16 Apr 2026 12:12:58 -0500
Subject: [PATCH 649/656] feat: serve notprepped rfc xml from datatracker
 (#10692)

* chore: move v3 RFC boundary number to settings

* feat: button for downloading unprepped RFCXML

* chore: ruff

* chore: refactor test to avoid setup overhead

* feat: serve notprepped bytes from blobdb

* fix: typo

* chore; improve test class name

* feat: wrapper explaining notprepped xml
---
 ietf/doc/feeds.py                          |   3 +-
 ietf/doc/tests_unprepped.py                | 118 +++++++++++++++++++++
 ietf/doc/urls.py                           |   2 +
 ietf/doc/views_doc.py                      |  31 +++++-
 ietf/settings.py                           |   4 +-
 ietf/sync/rfcindex.py                      |   2 +-
 ietf/templates/doc/document_rfc.html       |   9 ++
 ietf/templates/doc/notprepped_wrapper.html |  26 +++++
 8 files changed, 190 insertions(+), 5 deletions(-)
 create mode 100644 ietf/doc/tests_unprepped.py
 create mode 100644 ietf/templates/doc/notprepped_wrapper.html

diff --git a/ietf/doc/feeds.py b/ietf/doc/feeds.py
index afe96cf0df..0269906fcf 100644
--- a/ietf/doc/feeds.py
+++ b/ietf/doc/feeds.py
@@ -5,6 +5,7 @@
 import datetime
 import unicodedata
 
+from django.conf import settings
 from django.contrib.syndication.views import Feed, FeedDoesNotExist
 from django.utils.feedgenerator import Atom1Feed, Rss201rev2Feed
 from django.urls import reverse as urlreverse
@@ -223,7 +224,7 @@ def item_extra_kwargs(self, item):
         extra.update({"dcterms_accessRights": "gratis"})
         extra.update({"dcterms_format": "text/html"})
         media_contents = []
-        if item.rfc_number < 8650:
+        if item.rfc_number < settings.FIRST_V3_RFC:
             if item.rfc_number not in [8, 9, 51, 418, 500, 530, 589]:
                 for fmt, media_type in [("txt", "text/plain"), ("html", "text/html")]:
                     media_contents.append(
diff --git a/ietf/doc/tests_unprepped.py b/ietf/doc/tests_unprepped.py
new file mode 100644
index 0000000000..f88af8e81a
--- /dev/null
+++ b/ietf/doc/tests_unprepped.py
@@ -0,0 +1,118 @@
+# Copyright The IETF Trust 2026, All Rights Reserved
+
+from django.conf import settings
+from django.utils import timezone
+from django.urls import reverse as urlreverse
+
+from pyquery import PyQuery
+
+from ietf.doc.factories import WgRfcFactory
+from ietf.doc.models import StoredObject
+from ietf.doc.storage_utils import store_bytes
+from ietf.utils.test_utils import TestCase
+
+
+class UnpreppedRfcXmlTests(TestCase):
+    def test_editor_source_button_visibility(self):
+        pre_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC - 1)
+        first_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC)
+        post_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC + 1)
+
+        for rfc, expect_button in [(pre_v3, False), (first_v3, True), (post_v3, True)]:
+            r = self.client.get(
+                urlreverse(
+                    "ietf.doc.views_doc.document_main", kwargs=dict(name=rfc.name)
+                )
+            )
+            self.assertEqual(r.status_code, 200)
+            buttons = PyQuery(r.content)('a.btn:contains("Get editor source")')
+            if expect_button:
+                self.assertEqual(len(buttons), 1, msg=f"rfc_number={rfc.rfc_number}")
+                expected_href = urlreverse(
+                    "ietf.doc.views_doc.rfcxml_notprepped_wrapper",
+                    kwargs=dict(number=rfc.rfc_number),
+                )
+                self.assertEqual(
+                    buttons.attr("href"),
+                    expected_href,
+                    msg=f"rfc_number={rfc.rfc_number}",
+                )
+            else:
+                self.assertEqual(len(buttons), 0, msg=f"rfc_number={rfc.rfc_number}")
+
+    def test_rfcxml_notprepped(self):
+        number = settings.FIRST_V3_RFC
+        stored_name = f"notprepped/rfc{number}.notprepped.xml"
+        url = f"/doc/rfc{number}/notprepped/"
+
+        # 404 for pre-v3 RFC numbers (no document needed)
+        r = self.client.get(f"/doc/rfc{number - 1}/notprepped/")
+        self.assertEqual(r.status_code, 404)
+
+        # 404 when no RFC document exists in the database
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
+
+        # 404 when RFC document exists but has no StoredObject
+        WgRfcFactory(rfc_number=number)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
+
+        # 404 when StoredObject exists but backing storage is missing (FileNotFoundError)
+        now = timezone.now()
+        StoredObject.objects.create(
+            store="rfc",
+            name=stored_name,
+            sha384="a" * 96,
+            len=0,
+            store_created=now,
+            created=now,
+            modified=now,
+        )
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
+
+        # 200 with correct content-type and body when object is fully stored
+        xml_content = b"<rfc>test</rfc>"
+        store_bytes("rfc", stored_name, xml_content, allow_overwrite=True)
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r["Content-Type"], "application/xml")
+        self.assertEqual(r.content, xml_content)
+
+    def test_rfcxml_notprepped_wrapper(self):
+        number = settings.FIRST_V3_RFC
+
+        # 404 for pre-v3 RFC numbers (no document needed)
+        r = self.client.get(
+            urlreverse(
+                "ietf.doc.views_doc.rfcxml_notprepped_wrapper",
+                kwargs=dict(number=number - 1),
+            )
+        )
+        self.assertEqual(r.status_code, 404)
+
+        # 404 when no RFC document exists in the database
+        r = self.client.get(
+            urlreverse(
+                "ietf.doc.views_doc.rfcxml_notprepped_wrapper",
+                kwargs=dict(number=number),
+            )
+        )
+        self.assertEqual(r.status_code, 404)
+
+        # 200 with rendered template when RFC document exists
+        rfc = WgRfcFactory(rfc_number=number)
+        r = self.client.get(
+            urlreverse(
+                "ietf.doc.views_doc.rfcxml_notprepped_wrapper",
+                kwargs=dict(number=number),
+            )
+        )
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+        self.assertIn(str(rfc.rfc_number), q("h1").text())
+        download_url = urlreverse(
+            "ietf.doc.views_doc.rfcxml_notprepped", kwargs=dict(number=number)
+        )
+        self.assertEqual(len(q(f'a.btn[href="{download_url}"]')), 1)
diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py
index 61e94b2231..0c13503b78 100644
--- a/ietf/doc/urls.py
+++ b/ietf/doc/urls.py
@@ -99,6 +99,8 @@
 
     url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main),
     url(r'^%(name)s(?:/%(rev)s)?/bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex),
+    url(r'^rfc(?P<number>[0-9]+)/notprepped/$' , views_doc.rfcxml_notprepped),
+    url(r'^rfc(?P<number>[0-9]+)/notprepped-wrapper/$', views_doc.rfcxml_notprepped_wrapper),
     url(r'^%(name)s(?:/%(rev)s)?/idnits2-state/$' % settings.URL_REGEXPS, views_doc.idnits2_state),
     url(r'^bibxml3/reference.I-D.%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml_ref),
     url(r'^bibxml3/%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml),
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index c1f6352ac3..a23185333e 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -1,4 +1,4 @@
-# Copyright The IETF Trust 2009-2024, All Rights Reserved
+# Copyright The IETF Trust 2009-2026, All Rights Reserved
 # -*- coding: utf-8 -*-
 #
 # Parts Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies).
@@ -57,7 +57,7 @@
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import ( Document, DocHistory, DocEvent, BallotDocEvent, BallotType,
-    ConsensusDocEvent, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent,
+    ConsensusDocEvent, NewRevisionDocEvent, StoredObject, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent,
     IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS, DocumentActionHolder, DocumentAuthor,
     RelatedDocument, RelatedDocHistory)
 from ietf.doc.tasks import investigate_fragment_task
@@ -86,6 +86,7 @@
 from ietf.review.models import ReviewAssignment
 from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
+from ietf.doc.storage_utils import retrieve_bytes
 from ietf.utils import markup_txt, log, markdown
 from ietf.utils.draft import get_status_from_draft_text
 from ietf.utils.meetecho import MeetechoAPIError, SlidesManager
@@ -2356,3 +2357,29 @@ def investigate(request):
             "results": results,
         },
     )
+
+def rfcxml_notprepped(request, number):
+    number = int(number)
+    if number < settings.FIRST_V3_RFC:
+        raise Http404
+    rfc = Document.objects.filter(type="rfc", rfc_number=number).first()
+    if rfc is None:
+        raise Http404
+    name = f"notprepped/rfc{number}.notprepped.xml"
+    if not StoredObject.objects.filter(name=name).exists():
+        raise Http404
+    try:
+        bytes = retrieve_bytes("rfc", name)
+    except FileNotFoundError:
+        raise Http404
+    return HttpResponse(bytes, content_type="application/xml")
+
+
+def rfcxml_notprepped_wrapper(request, number):
+    number = int(number)
+    if number < settings.FIRST_V3_RFC:
+        raise Http404
+    rfc = Document.objects.filter(type="rfc", rfc_number=number).first()
+    if rfc is None:
+        raise Http404
+    return render(request, "doc/notprepped_wrapper.html", context={"rfc": rfc})
diff --git a/ietf/settings.py b/ietf/settings.py
index 40a4cb5c56..3aa45a453c 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -235,7 +235,9 @@
 
 AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', )
 
-FILE_UPLOAD_PERMISSIONS = 0o644          
+FILE_UPLOAD_PERMISSIONS = 0o644
+
+FIRST_V3_RFC = 8650
 
 
 #
diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py
index 6864617874..d1a0ed432f 100644
--- a/ietf/sync/rfcindex.py
+++ b/ietf/sync/rfcindex.py
@@ -153,7 +153,7 @@ def get_publication_std_levels() -> dict[int, StdLevelName]:
 
 
 def format_ordering(rfc_number):
-    if rfc_number < 8650:
+    if rfc_number < settings.FIRST_V3_RFC:
         ordering = ["txt", "ps", "pdf", "html", "xml"]
     else:
         ordering = ["html", "txt", "ps", "pdf", "xml"]
diff --git a/ietf/templates/doc/document_rfc.html b/ietf/templates/doc/document_rfc.html
index 7612ef8910..d4b309a964 100644
--- a/ietf/templates/doc/document_rfc.html
+++ b/ietf/templates/doc/document_rfc.html
@@ -124,6 +124,15 @@
                 </i>
                 Referenced by
             </a>
+            {% if doc.rfc_number >= settings.FIRST_V3_RFC %}
+            <a class="btn btn-primary btn-sm"
+              href="{% url 'ietf.doc.views_doc.rfcxml_notprepped_wrapper' number=doc.rfc_number %}"
+              rel="nofollow">
+              <i class="bi bi-vector-pen">
+              </i>
+              Get editor source
+            </a>
+            {% endif %}
             <a class="btn btn-primary btn-sm"
                 href="https://mailarchive.ietf.org/arch/search?q=%22{{ doc.name }}%22"
                 rel="nofollow"
diff --git a/ietf/templates/doc/notprepped_wrapper.html b/ietf/templates/doc/notprepped_wrapper.html
new file mode 100644
index 0000000000..078db7bc51
--- /dev/null
+++ b/ietf/templates/doc/notprepped_wrapper.html
@@ -0,0 +1,26 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2026, All Rights Reserved #}
+{% load origin %}
+{% block title %}RFC {{ rfc.rfc_number }} — Not-prepped XML{% endblock %}
+{% block content %}
+    {% origin %}
+    <h1>RFC {{ rfc.rfc_number }} — Not-prepped XML</h1>
+    <p>
+        The <em>not-prepped</em> XML 
+        is the RFC XML v3 source for an RFC at the moment in the publication process
+        just before the prep tool was used to expand default
+        values, generate section numbers, resolve cross-references, and embed
+        boilerplate.  
+    </p><p>   
+        It is useful for authors who want to begin a new draft based on
+        the RFC's text, such as when creating a bis-draft, and for tools that process 
+        author-facing RFC XML.
+    </p>
+    <p>
+        <a class="btn btn-primary"
+           href="{% url 'ietf.doc.views_doc.rfcxml_notprepped' number=rfc.rfc_number %}">
+            <i class="bi bi-download"></i>
+            Download not-prepped XML for RFC {{ rfc.rfc_number }}
+        </a>
+    </p>
+{% endblock %}

From 20480d6242254693b3dbcbf9b73380b5b3c838cb Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Thu, 16 Apr 2026 13:59:23 -0500
Subject: [PATCH 650/656] fix: force notprepped downloads (#10719)

---
 ietf/doc/{tests_unprepped.py => tests_notprepped.py} | 10 +++++++---
 ietf/doc/views_doc.py                                |  5 +++--
 2 files changed, 10 insertions(+), 5 deletions(-)
 rename ietf/doc/{tests_unprepped.py => tests_notprepped.py} (92%)

diff --git a/ietf/doc/tests_unprepped.py b/ietf/doc/tests_notprepped.py
similarity index 92%
rename from ietf/doc/tests_unprepped.py
rename to ietf/doc/tests_notprepped.py
index f88af8e81a..f417aa7931 100644
--- a/ietf/doc/tests_unprepped.py
+++ b/ietf/doc/tests_notprepped.py
@@ -12,7 +12,7 @@
 from ietf.utils.test_utils import TestCase
 
 
-class UnpreppedRfcXmlTests(TestCase):
+class NotpreppedRfcXmlTests(TestCase):
     def test_editor_source_button_visibility(self):
         pre_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC - 1)
         first_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC)
@@ -72,13 +72,17 @@ def test_rfcxml_notprepped(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 404)
 
-        # 200 with correct content-type and body when object is fully stored
+        # 200 with correct content-type, attachment disposition, and body when object is fully stored
         xml_content = b"<rfc>test</rfc>"
         store_bytes("rfc", stored_name, xml_content, allow_overwrite=True)
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r["Content-Type"], "application/xml")
-        self.assertEqual(r.content, xml_content)
+        self.assertEqual(
+            r["Content-Disposition"],
+            f'attachment; filename="rfc{number}.notprepped.xml"',
+        )
+        self.assertEqual(b"".join(r.streaming_content), xml_content)
 
     def test_rfcxml_notprepped_wrapper(self):
         number = settings.FIRST_V3_RFC
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index a23185333e..5b57a62074 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -43,9 +43,10 @@
 
 from celery.result import AsyncResult
 from django.core.cache import caches
+from django.core.files.base import ContentFile
 from django.core.exceptions import PermissionDenied
 from django.db.models import Max
-from django.http import HttpResponse, Http404, HttpResponseBadRequest, JsonResponse
+from django.http import FileResponse, HttpResponse, Http404, HttpResponseBadRequest, JsonResponse
 from django.shortcuts import render, get_object_or_404, redirect
 from django.template.loader import render_to_string
 from django.urls import reverse as urlreverse
@@ -2372,7 +2373,7 @@ def rfcxml_notprepped(request, number):
         bytes = retrieve_bytes("rfc", name)
     except FileNotFoundError:
         raise Http404
-    return HttpResponse(bytes, content_type="application/xml")
+    return FileResponse(ContentFile(bytes, name=f"rfc{number}.notprepped.xml"), as_attachment=True)
 
 
 def rfcxml_notprepped_wrapper(request, number):

From 9cecc36bc7e42ecc5cd196d96f4bd0eaf03b5e69 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 17 Apr 2026 00:25:02 -0300
Subject: [PATCH 651/656] feat: rebuild_searchindex task (#10723)

* refactor: DRY

* chore: typesense docker container (commented out)

* feat: batched RFC search index import

* feat: rebuild_searchindex task

* feat: logging / error reporting

* refactor: _task suffix for task name

* test: tests for searchindex utils + tasks

* fix: only create collection if dropped

* fix: typing / lint
---
 docker-compose.yml              |  12 ++
 ietf/doc/tasks.py               |  11 ++
 ietf/doc/tests_tasks.py         |  43 ++++++
 ietf/utils/searchindex.py       | 239 ++++++++++++++++++++++++++++++--
 ietf/utils/tests_searchindex.py | 152 +++++++++++++++-----
 5 files changed, 410 insertions(+), 47 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 4c3f2f6b8e..073d04b896 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -132,6 +132,18 @@ services:
         volumes:
             - blobdb-data:/var/lib/postgresql/data
 
+#    typesense:
+#        image: typesense/typesense:30.1
+#        restart: on-failure
+#        ports:
+#            - "8108:8108"
+#        volumes:
+#            - ./typesense-data:/data
+#        command:
+#            - '--data-dir=/data'
+#            - '--api-key=typesense-api-key'
+#            - '--enable-cors'
+
 #  Celery Beat is a periodic task runner. It is not normally needed for development,
 #  but can be enabled by uncommenting the following.
 #
diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py
index 19edb39014..273242e35f 100644
--- a/ietf/doc/tasks.py
+++ b/ietf/doc/tasks.py
@@ -209,3 +209,14 @@ def update_rfc_searchindex_task(self, rfc_number: int):
                 countdown=searchindex_settings["TASK_RETRY_DELAY"],
                 max_retries=searchindex_settings["TASK_MAX_RETRIES"],
             )
+
+
+@shared_task
+def rebuild_searchindex_task(*, batchsize=40, drop_collection=False):
+    if drop_collection:
+        searchindex.delete_collection()
+        searchindex.create_collection()
+    searchindex.update_or_create_rfc_entries(
+        Document.objects.filter(type_id="rfc").order_by("-rfc_number"),
+        batchsize=batchsize,
+    )
diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py
index 728d21f131..2e2d65463f 100644
--- a/ietf/doc/tests_tasks.py
+++ b/ietf/doc/tests_tasks.py
@@ -24,6 +24,7 @@
     generate_idnits2_rfc_status_task,
     investigate_fragment_task,
     notify_expirations_task,
+    rebuild_searchindex_task,
     update_rfc_searchindex_task,
 )
 
@@ -144,6 +145,48 @@ def test_update_rfc_searchindex_task(
             with self.assertRaises(Retry):
                 update_rfc_searchindex_task(rfc_number=rfc.rfc_number)
 
+    @mock.patch("ietf.doc.tasks.searchindex.update_or_create_rfc_entries")
+    @mock.patch("ietf.doc.tasks.searchindex.create_collection")
+    @mock.patch("ietf.doc.tasks.searchindex.delete_collection")
+    def test_rebuild_searchindex_task(self, mock_delete, mock_create, mock_update):
+        rfcs = WgRfcFactory.create_batch(10)
+        rebuild_searchindex_task()
+        self.assertFalse(mock_delete.called)
+        self.assertFalse(mock_create.called)
+        self.assertTrue(mock_update.called)
+        self.assertQuerysetEqual(
+            mock_update.call_args.args[0],
+            sorted(rfcs, key=lambda doc: -doc.rfc_number),
+            ordered=True,
+        )
+
+        mock_delete.reset_mock()
+        mock_create.reset_mock()
+        mock_update.reset_mock()
+        rebuild_searchindex_task(drop_collection=True)
+        self.assertTrue(mock_delete.called)
+        self.assertTrue(mock_create.called)
+        self.assertTrue(mock_update.called)
+        self.assertQuerysetEqual(
+            mock_update.call_args.args[0],
+            sorted(rfcs, key=lambda doc: -doc.rfc_number),
+            ordered=True,
+        )
+
+        mock_delete.reset_mock()
+        mock_create.reset_mock()
+        mock_update.reset_mock()
+        rebuild_searchindex_task(drop_collection=True, batchsize=3)
+        self.assertTrue(mock_delete.called)
+        self.assertTrue(mock_create.called)
+        self.assertTrue(mock_update.called)
+        self.assertQuerysetEqual(
+            mock_update.call_args.args[0],
+            sorted(rfcs, key=lambda doc: -doc.rfc_number),
+            ordered=True,
+        )
+        self.assertEqual(mock_update.call_args.kwargs["batchsize"], 3)
+
 
 class Idnits2SupportTests(TestCase):
     settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [
diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py
index e4427b88b5..a47e6d2f12 100644
--- a/ietf/utils/searchindex.py
+++ b/ietf/utils/searchindex.py
@@ -2,12 +2,15 @@
 """Search indexing utilities"""
 
 import re
+from itertools import batched
 from math import floor
+from typing import Iterable
 
 import httpx  # just for exceptions
 import typesense
 import typesense.exceptions
 from django.conf import settings
+from typesense.types.document import DocumentSchema
 
 from ietf.doc.models import Document, StoredObject
 from ietf.doc.storage_utils import retrieve_str
@@ -42,6 +45,24 @@ def enabled():
     return _settings["TYPESENSE_API_URL"] != ""
 
 
+def get_typesense_client() -> typesense.Client:
+    _settings = get_settings()
+    client = typesense.Client(
+        {
+            "api_key": _settings["TYPESENSE_API_KEY"],
+            "nodes": [_settings["TYPESENSE_API_URL"]],
+        }
+    )
+    return client
+
+
+def get_collection_name() -> str:
+    _settings = get_settings()
+    collection_name = _settings["TYPESENSE_COLLECTION_NAME"]
+    assert isinstance(collection_name, str)
+    return collection_name
+
+
 def _sanitize_text(content):
     """Sanitize content or abstract text for search"""
     # REs (with approximate names)
@@ -62,7 +83,7 @@ def _sanitize_text(content):
     return content.strip()
 
 
-def update_or_create_rfc_entry(rfc: Document):
+def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema:
     assert rfc.type_id == "rfc"
     assert rfc.rfc_number is not None
 
@@ -75,8 +96,8 @@ def update_or_create_rfc_entry(rfc: Document):
             f"Indexing as {subseries[0].name}"
         )
     subseries = subseries[0] if len(subseries) > 0 else None
-    obsoleted_by = rfc.relations_that("obs")
-    updated_by = rfc.relations_that("updates")
+    obsoleted_by = rfc.related_that("obs")
+    updated_by = rfc.related_that("updates")
 
     stored_txt = (
         StoredObject.objects.exclude_deleted()
@@ -91,8 +112,8 @@ def update_or_create_rfc_entry(rfc: Document):
         except Exception as err:
             log(f"Unable to retrieve {stored_txt} from storage: {err}")
 
-    ts_id = f"doc-{rfc.pk}"
     ts_document = {
+        "id": f"doc-{rfc.pk}",
         "rfcNumber": rfc.rfc_number,
         "rfc": str(rfc.rfc_number),
         "filename": rfc.name,
@@ -143,13 +164,205 @@ def update_or_create_rfc_entry(rfc: Document):
         ts_document["adName"] = rfc.ad.name
     if content != "":
         ts_document["content"] = _sanitize_text(content)
-    _settings = get_settings()
-    client = typesense.Client(
+    return ts_document
+
+
+def update_or_create_rfc_entry(rfc: Document):
+    """Update/create index entries for one RFC"""
+    ts_document = typesense_doc_from_rfc(rfc)
+    client = get_typesense_client()
+    client.collections[get_collection_name()].documents.upsert(ts_document)
+
+
+def update_or_create_rfc_entries(
+    rfcs: Iterable[Document], batchsize: int | None = None
+):
+    """Update/create index entries for RFCs in bulk
+
+    If batchsize is set, computes index data in batches of batchsize and adds to the
+    index. Will make a total of (len(rfcs) // batchsize) + 1 API calls.
+
+    N.b. that typesense has a server-side batch size that defaults to 40, which should
+    "almost never be changed from the default." This does not change that. Further,
+    the python client library's import_ method has a batch_size parameter that does
+    client-side batching. We don't use that, either.
+    """
+    success_count = 0
+    fail_count = 0
+    client = get_typesense_client()
+    batches = [rfcs] if batchsize is None else batched(rfcs, batchsize)
+    for batch in batches:
+        tdoc_batch = [typesense_doc_from_rfc(rfc) for rfc in batch]
+        results = client.collections[get_collection_name()].documents.import_(
+            tdoc_batch, {"action": "upsert"}
+        )
+        for tdoc, result in zip(tdoc_batch, results):
+            if result["success"]:
+                success_count += 1
+            else:
+                fail_count += 1
+                log(f"Failed to index RFC {tdoc['rfcNumber']}: {result['error']}")
+    log(f"Added {success_count} RFCs to the index, failed to add {fail_count}")
+
+
+DOCS_SCHEMA = {
+    "enable_nested_fields": True,
+    "default_sorting_field": "ranking",
+    "fields": [
+        # RFC number in integer form, for sorting asc/desc in search results
+        # Omit field for drafts
         {
-            "api_key": _settings["TYPESENSE_API_KEY"],
-            "nodes": [_settings["TYPESENSE_API_URL"]],
-        }
-    )
-    client.collections[_settings["TYPESENSE_COLLECTION_NAME"]].documents.upsert(
-        {"id": ts_id} | ts_document
-    )
+            "name": "rfcNumber",
+            "type": "int32",
+            "facet": False,
+            "optional": True,
+            "sort": True,
+        },
+        # RFC number in string form, for direct matching with ranking
+        # Omit field for drafts
+        {"name": "rfc", "type": "string", "facet": False, "optional": True},
+        # For drafts that correspond to an RFC, insert the RFC number
+        # Omit field for rfcs or if not relevant
+        {"name": "ref", "type": "string", "facet": False, "optional": True},
+        # Filename of the document (without the extension, e.g. "rfc1234"
+        # or "draft-ietf-abc-def-02")
+        {"name": "filename", "type": "string", "facet": False, "infix": True},
+        # Title of the draft / rfc
+        {"name": "title", "type": "string", "facet": False},
+        # Abstract of the draft / rfc
+        {"name": "abstract", "type": "string", "facet": False},
+        # A list of search keywords if relevant, set to empty array otherwise
+        {"name": "keywords", "type": "string[]", "facet": True},
+        # Type of the document
+        # Accepted values: "draft" or "rfc"
+        {"name": "type", "type": "string", "facet": True},
+        # State(s) of the document (e.g. "Published", "Adopted by a WG", etc.)
+        # Use the full name, not the slug
+        {"name": "state", "type": "string[]", "facet": True, "optional": True},
+        # Status (Standard Level Name)
+        # Object with properties "slug" and "name"
+        # e.g.: { slug: "std", "name": "Internet Standard" }
+        {"name": "status", "type": "object", "facet": True, "optional": True},
+        # The subseries it is part of. (e.g. "BCP")
+        # Omit otherwise.
+        {
+            "name": "subseries.acronym",
+            "type": "string",
+            "facet": True,
+            "optional": True,
+        },
+        # The subseries number it is part of. (e.g. 123)
+        # Omit otherwise.
+        {
+            "name": "subseries.number",
+            "type": "int32",
+            "facet": True,
+            "sort": True,
+            "optional": True,
+        },
+        # The total of RFCs in the subseries
+        # Omit if not part of a subseries
+        {
+            "name": "subseries.total",
+            "type": "int32",
+            "facet": False,
+            "sort": False,
+            "optional": True,
+        },
+        # Date of the document, in unix epoch seconds (can be negative for < 1970)
+        {"name": "date", "type": "int64", "facet": False},
+        # Expiration date of the document, in unix epoch seconds (can be negative
+        # for < 1970). Omit field for RFCs
+        {"name": "expires", "type": "int64", "facet": False, "optional": True},
+        # Publication date of the RFC, in unix epoch seconds (can be negative
+        # for < 1970). Omit field for drafts
+        {
+            "name": "publicationDate",
+            "type": "int64",
+            "facet": True,
+            "optional": True,
+        },
+        # Working Group
+        # Object with properties "acronym", "name" and "full"
+        # e.g.:
+        # {
+        #     "acronym": "ntp",
+        #     "name": "Network Time Protocols",
+        #     "full": "ntp - Network Time Protocols",
+        # }
+        {"name": "group", "type": "object", "facet": True, "optional": True},
+        # Area
+        # Object with properties "acronym", "name" and "full"
+        # e.g.:
+        # {
+        #     "acronym": "mpls",
+        #     "name": "Multiprotocol Label Switching",
+        #     "full": "mpls - Multiprotocol Label Switching",
+        # }
+        {"name": "area", "type": "object", "facet": True, "optional": True},
+        # Stream
+        # Object with properties "slug" and "name"
+        # e.g.: { slug: "ietf", "name": "IETF" }
+        {"name": "stream", "type": "object", "facet": True, "optional": True},
+        # List of authors
+        # Array of objects with properties "name" and "affiliation"
+        # e.g.:
+        # [
+        #     {"name": "John Doe", "affiliation": "ACME Inc."},
+        #     {"name": "Ada Lovelace", "affiliation": "Babbage Corps."},
+        # ]
+        {"name": "authors", "type": "object[]", "facet": True, "optional": True},
+        # Area Director Name (e.g. "Leonardo DaVinci")
+        {"name": "adName", "type": "string", "facet": True, "optional": True},
+        # Whether the document should be hidden by default in search results or not.
+        {"name": "flags.hiddenDefault", "type": "bool", "facet": True},
+        # Whether the document is obsoleted by another document or not.
+        {"name": "flags.obsoleted", "type": "bool", "facet": True},
+        # Whether the document is updated by another document or not.
+        {"name": "flags.updated", "type": "bool", "facet": True},
+        # List of documents that obsolete this document.
+        # Array of strings. Use RFC number for RFCs. (e.g. ["123", "456"])
+        # Omit if none. Must be provided if "flags.obsoleted" is set to True.
+        {
+            "name": "obsoletedBy",
+            "type": "string[]",
+            "facet": False,
+            "optional": True,
+        },
+        # List of documents that update this document.
+        # Array of strings. Use RFC number for RFCs. (e.g. ["123", "456"])
+        # Omit if none. Must be provided if "flags.updated" is set to True.
+        {"name": "updatedBy", "type": "string[]", "facet": False, "optional": True},
+        # Sanitized content of the document.
+        # Make sure to remove newlines, double whitespaces, symbols and tags.
+        {
+            "name": "content",
+            "type": "string",
+            "facet": False,
+            "optional": True,
+            "store": False,
+        },
+        # Ranking value to use when no explicit sorting is used during search
+        # Set to the RFC number for RFCs and the revision number for drafts
+        # This ensures newer RFCs get listed first in the default search results
+        # (without a query)
+        {"name": "ranking", "type": "int32", "facet": False},
+    ],
+}
+
+
+def create_collection():
+    collection_name = get_collection_name()
+    log(f"Creating '{collection_name}' collection")
+    client = get_typesense_client()
+    client.collections.create({"name": get_collection_name()} | DOCS_SCHEMA)
+
+
+def delete_collection():
+    collection_name = get_collection_name()
+    log(f"Deleting '{collection_name}' collection")
+    client = get_typesense_client()
+    try:
+        client.collections[collection_name].delete()
+    except typesense.exceptions.ObjectNotFound:
+        pass
diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py
index 8740716c85..0bff96ec7d 100644
--- a/ietf/utils/tests_searchindex.py
+++ b/ietf/utils/tests_searchindex.py
@@ -1,6 +1,7 @@
 # Copyright The IETF Trust 2026, All Rights Reserved
 from unittest import mock
 
+import typesense.exceptions
 from django.conf import settings
 from django.test.utils import override_settings
 
@@ -51,42 +52,29 @@ def test_sanitize_text(self):
             "TYPESENSE_COLLECTION_NAME": "frogs",
         }
     )
-    @mock.patch("ietf.utils.searchindex.typesense.Client")
-    def test_update_or_create_rfc_entry(self, mock_ts_client_constructor):
+    def test_typesense_doc_from_rfc(self):
         not_rfc = WgDraftFactory()
         assert isinstance(not_rfc, Document)
         with self.assertRaises(AssertionError):
-            searchindex.update_or_create_rfc_entry(not_rfc)
-        self.assertFalse(mock_ts_client_constructor.called)
+            searchindex.typesense_doc_from_rfc(not_rfc)
 
         invalid_rfc = WgRfcFactory(name="rfc1000000", rfc_number=None)
         assert isinstance(invalid_rfc, Document)
         with self.assertRaises(AssertionError):
-            searchindex.update_or_create_rfc_entry(invalid_rfc)
-        self.assertFalse(mock_ts_client_constructor.called)
+            searchindex.typesense_doc_from_rfc(invalid_rfc)
 
         rfc = PublishedRfcDocEventFactory().doc
         assert isinstance(rfc, Document)
-        searchindex.update_or_create_rfc_entry(rfc)
-        self.assertTrue(mock_ts_client_constructor.called)
-        # walk the tree down to the method we expected to be called...
-        mock_upsert = mock_ts_client_constructor.return_value.collections[
-            "frogs"
-        ].documents.upsert  # matches value in override_settings above
-        self.assertTrue(mock_upsert.called)
-        upserted_dict = mock_upsert.call_args[0][0]
+        result = searchindex.typesense_doc_from_rfc(rfc)
         # Check a few values, not exhaustive
-        self.assertEqual(upserted_dict["id"], f"doc-{rfc.pk}")
-        self.assertEqual(upserted_dict["rfcNumber"], rfc.rfc_number)
-        self.assertEqual(
-            upserted_dict["abstract"], searchindex._sanitize_text(rfc.abstract)
-        )
-        self.assertNotIn("adName", upserted_dict)
-        self.assertNotIn("content", upserted_dict)  # no blob
-        self.assertNotIn("subseries", upserted_dict)
+        self.assertEqual(result["id"], f"doc-{rfc.pk}")
+        self.assertEqual(result["rfcNumber"], rfc.rfc_number)
+        self.assertEqual(result["abstract"], searchindex._sanitize_text(rfc.abstract))
+        self.assertNotIn("adName", result)
+        self.assertNotIn("content", result)  # no blob
+        self.assertNotIn("subseries", result)
 
         # repeat, this time with contents, an AD, and subseries docs
-        mock_upsert.reset_mock()
         store_str(
             kind="rfc",
             name=f"txt/{rfc.name}.txt",
@@ -99,17 +87,15 @@ def test_update_or_create_rfc_entry(self, mock_ts_client_constructor):
         # (the typesense schema does not support this for real at the moment)
         BcpFactory(contains=[rfc], name="bcp1234")
         StdFactory(contains=[rfc], name="std1234")
-        searchindex.update_or_create_rfc_entry(rfc)
-        self.assertTrue(mock_upsert.called)
-        upserted_dict = mock_upsert.call_args[0][0]
+        result = searchindex.typesense_doc_from_rfc(rfc)
         # Check a few values, not exhaustive
         self.assertEqual(
-            upserted_dict["content"],
+            result["content"],
             searchindex._sanitize_text("The contents of this RFC"),
         )
-        self.assertEqual(upserted_dict["adName"], "Alfred D. Rector")
-        self.assertIn("subseries", upserted_dict)
-        ss_dict = upserted_dict["subseries"]
+        self.assertEqual(result["adName"], "Alfred D. Rector")
+        self.assertIn("subseries", result)
+        ss_dict = result["subseries"]
         # We should get one of the two subseries docs, but neither is more correct
         # than the other...
         self.assertTrue(
@@ -119,10 +105,108 @@ def test_update_or_create_rfc_entry(self, mock_ts_client_constructor):
             )
         )
 
-        # Finally, delete the contents blob and make sure things don't blow up 
-        mock_upsert.reset_mock()
+        # Finally, delete the contents blob and make sure things don't blow up
         Blob.objects.get(bucket="rfc", name=f"txt/{rfc.name}.txt").delete()
+        result = searchindex.typesense_doc_from_rfc(rfc)
+        self.assertNotIn("content", result)
+
+    @override_settings(
+        SEARCHINDEX_CONFIG={
+            "TYPESENSE_API_URL": "http://ts.example.com",
+            "TYPESENSE_API_KEY": "test-api-key",
+            "TYPESENSE_COLLECTION_NAME": "frogs",
+        }
+    )
+    @mock.patch("ietf.utils.searchindex.typesense_doc_from_rfc")
+    @mock.patch("ietf.utils.searchindex.typesense.Client")
+    def test_update_or_create_rfc_entry(
+        self, mock_ts_client_constructor, mock_tdoc_from_rfc
+    ):
+        fake_tdoc = object()
+        mock_tdoc_from_rfc.return_value = fake_tdoc
+        rfc = WgRfcFactory()
+        assert isinstance(rfc, Document)
         searchindex.update_or_create_rfc_entry(rfc)
+        self.assertTrue(mock_ts_client_constructor.called)
+        # walk the tree down to the method we expected to be called...
+        mock_upsert = mock_ts_client_constructor.return_value.collections[
+            "frogs"  # matches value in override_settings above
+        ].documents.upsert
         self.assertTrue(mock_upsert.called)
-        upserted_dict = mock_upsert.call_args[0][0]
-        self.assertNotIn("content", upserted_dict)
+        self.assertEqual(mock_upsert.call_args, mock.call(fake_tdoc))
+
+    @override_settings(
+        SEARCHINDEX_CONFIG={
+            "TYPESENSE_API_URL": "http://ts.example.com",
+            "TYPESENSE_API_KEY": "test-api-key",
+            "TYPESENSE_COLLECTION_NAME": "frogs",
+        }
+    )
+    @mock.patch("ietf.utils.searchindex.typesense_doc_from_rfc")
+    @mock.patch("ietf.utils.searchindex.typesense.Client")
+    def test_update_or_create_rfc_entries(
+        self, mock_ts_client_constructor, mock_tdoc_from_rfc
+    ):
+        fake_tdoc = object()
+        mock_tdoc_from_rfc.return_value = fake_tdoc
+        rfc = WgRfcFactory()
+        assert isinstance(rfc, Document)
+        searchindex.update_or_create_rfc_entries([rfc] * 50)  # list of docs...
+        self.assertEqual(mock_ts_client_constructor.call_count, 1)
+        # walk the tree down to the method we expected to be called...
+        mock_import_ = mock_ts_client_constructor.return_value.collections[
+            "frogs"  # matches value in override_settings above
+        ].documents.import_
+        self.assertEqual(mock_import_.call_count, 1)
+        self.assertEqual(
+            mock_import_.call_args, mock.call([fake_tdoc] * 50, {"action": "upsert"})
+        )
+
+        mock_import_.reset_mock()
+        searchindex.update_or_create_rfc_entries([rfc] * 50, batchsize=20)
+        self.assertEqual(mock_ts_client_constructor.call_count, 2)  # one more
+        # walk the tree down to the method we expected to be called...
+        mock_import_ = mock_ts_client_constructor.return_value.collections[
+            "frogs"  # matches value in override_settings above
+        ].documents.import_
+        self.assertEqual(mock_import_.call_count, 3)
+        self.assertEqual(
+            mock_import_.call_args_list,
+            [
+                mock.call([fake_tdoc] * 20, {"action": "upsert"}),
+                mock.call([fake_tdoc] * 20, {"action": "upsert"}),
+                mock.call([fake_tdoc] * 10, {"action": "upsert"}),
+            ],
+        )
+
+    @override_settings(
+        SEARCHINDEX_CONFIG={
+            "TYPESENSE_API_URL": "http://ts.example.com",
+            "TYPESENSE_API_KEY": "test-api-key",
+            "TYPESENSE_COLLECTION_NAME": "frogs",
+        }
+    )
+    @mock.patch("ietf.utils.searchindex.typesense.Client")
+    def test_create_collection(self, mock_ts_client_constructor):
+        searchindex.create_collection()
+        self.assertEqual(mock_ts_client_constructor.call_count, 1)
+        mock_collections = mock_ts_client_constructor.return_value.collections
+        self.assertTrue(mock_collections.create.called)
+        self.assertEqual(mock_collections.create.call_args[0][0]["name"], "frogs")
+
+    @override_settings(
+        SEARCHINDEX_CONFIG={
+            "TYPESENSE_API_URL": "http://ts.example.com",
+            "TYPESENSE_API_KEY": "test-api-key",
+            "TYPESENSE_COLLECTION_NAME": "frogs",
+        }
+    )
+    @mock.patch("ietf.utils.searchindex.typesense.Client")
+    def test_delete_collection(self, mock_ts_client_constructor):
+        searchindex.delete_collection()
+        self.assertEqual(mock_ts_client_constructor.call_count, 1)
+        mock_collections = mock_ts_client_constructor.return_value.collections
+        self.assertTrue(mock_collections["frogs"].delete.called)
+
+        mock_collections["frogs"].side_effect = typesense.exceptions.ObjectNotFound
+        searchindex.delete_collection()  # should ignore the exception

From c4cb8b91fc9434a3bb3419acfac2dd3b30cb4a6c Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 17 Apr 2026 07:33:51 -0300
Subject: [PATCH 652/656] fix: add pages to typesense schema (#10726)

---
 ietf/utils/searchindex.py       | 4 ++++
 ietf/utils/tests_searchindex.py | 1 +
 2 files changed, 5 insertions(+)

diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py
index a47e6d2f12..87951abb60 100644
--- a/ietf/utils/searchindex.py
+++ b/ietf/utils/searchindex.py
@@ -86,6 +86,7 @@ def _sanitize_text(content):
 def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema:
     assert rfc.type_id == "rfc"
     assert rfc.rfc_number is not None
+    assert rfc.pages is not None
 
     keywords: list[str] = rfc.keywords  # help type checking
 
@@ -119,6 +120,7 @@ def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema:
         "filename": rfc.name,
         "title": rfc.title,
         "abstract": _sanitize_text(rfc.abstract),
+        "pages": rfc.pages,
         "keywords": keywords,
         "type": "rfc",
         "state": [state.name for state in rfc.states.all()],
@@ -231,6 +233,8 @@ def update_or_create_rfc_entries(
         {"name": "title", "type": "string", "facet": False},
         # Abstract of the draft / rfc
         {"name": "abstract", "type": "string", "facet": False},
+        # Number of pages
+        {"name": "pages", "type": "int32", "facet": False},
         # A list of search keywords if relevant, set to empty array otherwise
         {"name": "keywords", "type": "string[]", "facet": True},
         # Type of the document
diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py
index 0bff96ec7d..e9fbf52020 100644
--- a/ietf/utils/tests_searchindex.py
+++ b/ietf/utils/tests_searchindex.py
@@ -70,6 +70,7 @@ def test_typesense_doc_from_rfc(self):
         self.assertEqual(result["id"], f"doc-{rfc.pk}")
         self.assertEqual(result["rfcNumber"], rfc.rfc_number)
         self.assertEqual(result["abstract"], searchindex._sanitize_text(rfc.abstract))
+        self.assertEqual(result["pages"], rfc.pages)
         self.assertNotIn("adName", result)
         self.assertNotIn("content", result)  # no blob
         self.assertNotIn("subseries", result)

From 629ffb13480201e25fc5d941cfcea9de123562f9 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 17 Apr 2026 15:04:23 -0300
Subject: [PATCH 653/656] fix: decode non-utf-8 blob content (#10729)

* refactor: decode_document_content() utility method

* fix: fall back to latin-1 in retrieve_str()

* refactor: match structure with retrieve_bytes()

* refactor: separate tests_text.py module

* test: test_decode_document_content + ruff

* fix: revert misguided refactor

* test: assert to guarantee test is valid
---
 ietf/doc/models.py        | 15 ++-------
 ietf/doc/storage_utils.py | 47 +++++++++++++-------------
 ietf/utils/tests.py       | 19 -----------
 ietf/utils/tests_text.py  | 71 +++++++++++++++++++++++++++++++++++++++
 ietf/utils/text.py        | 18 ++++++++++
 5 files changed, 114 insertions(+), 56 deletions(-)
 create mode 100644 ietf/utils/tests_text.py

diff --git a/ietf/doc/models.py b/ietf/doc/models.py
index 972f0a34e8..cc79b73831 100644
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@@ -52,6 +52,7 @@
 from ietf.person.utils import get_active_balloters
 from ietf.utils import log
 from ietf.utils.decorators import memoize
+from ietf.utils.text import decode_document_content
 from ietf.utils.validators import validate_no_control_chars
 from ietf.utils.mail import formataddr
 from ietf.utils.models import ForeignKey
@@ -640,19 +641,7 @@ def text(self, size = -1):
         except IOError as e:
             log.log(f"Error reading text for {path}: {e}")
             return None
-        text = None
-        try:
-            text = raw.decode('utf-8')
-        except UnicodeDecodeError:
-            for back in range(1,4):
-                try:
-                    text = raw[:-back].decode('utf-8')
-                    break
-                except UnicodeDecodeError:
-                    pass
-            if text is None:
-                text = raw.decode('latin-1')
-        return text
+        return decode_document_content(raw)
 
     def text_or_error(self):
         return self.text() or "Error; cannot read '%s'"%self.get_base_name()
diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py
index ffdd4599be..9c18bb8a8a 100644
--- a/ietf/doc/storage_utils.py
+++ b/ietf/doc/storage_utils.py
@@ -10,6 +10,7 @@
 from django.core.files.storage import storages, Storage
 
 from ietf.utils.log import log
+from ietf.utils.text import decode_document_content
 
 
 class StorageUtilsError(Exception):
@@ -164,32 +165,30 @@ def store_str(
 
 def retrieve_bytes(kind: str, name: str) -> bytes:
     from ietf.doc.storage import maybe_log_timing
-    content = b""
-    if settings.ENABLE_BLOBSTORAGE:
-        try:
-            store = _get_storage(kind)
-            with store.open(name) as f:
-                with maybe_log_timing(
-                    hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing,
-                    "read",
-                    bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "",
-                    name=name,
-                ):
-                    content = f.read()
-        except Exception as err:
-            log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}")
-            raise
+    if not settings.ENABLE_BLOBSTORAGE:
+        return b""
+    try:
+        store = _get_storage(kind)
+        with store.open(name) as f:
+            with maybe_log_timing(
+                hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing,
+                "read",
+                bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "",
+                name=name,
+            ):
+                content = f.read()
+    except Exception as err:
+        log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}")
+        raise
     return content
 
 
 def retrieve_str(kind: str, name: str) -> str:
-    content = ""
-    if settings.ENABLE_BLOBSTORAGE:
-        try:
-            content_bytes = retrieve_bytes(kind, name)
-            # TODO-BLOBSTORE: try to decode all the different ways doc.text() does
-            content = content_bytes.decode("utf-8")
-        except Exception as err:
-            log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}")
-            raise
+    if not settings.ENABLE_BLOBSTORAGE:
+        return ""
+    try:
+        content = decode_document_content(retrieve_bytes(kind, name))
+    except Exception as err:
+        log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}")
+        raise
     return content
diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py
index 3288309095..99c33f34b3 100644
--- a/ietf/utils/tests.py
+++ b/ietf/utils/tests.py
@@ -60,7 +60,6 @@
     set_url_coverage,
 )
 from ietf.utils.test_utils import TestCase, unicontent
-from ietf.utils.text import parse_unicode
 from ietf.utils.timezone import timezone_not_near_midnight
 from ietf.utils.xmldraft import XMLDraft, InvalidMetadataError, capture_xml2rfc_output
 
@@ -864,24 +863,6 @@ def test_assertion(self):
         assertion('False')
         settings.SERVER_MODE = 'test'
 
-class TestRFC2047Strings(TestCase):
-    def test_parse_unicode(self):
-        names = (
-            ('=?utf-8?b?4Yuz4YuK4Ym1IOGJoOGJgOGIiA==?=', 'ዳዊት በቀለ'),
-            ('=?utf-8?b?5Li9IOmDnA==?=', '丽 郜'),
-            ('=?utf-8?b?4KSV4KSu4KWN4KSs4KWL4KScIOCkoeCkvuCksA==?=', 'कम्बोज डार'),
-            ('=?utf-8?b?zpfPgc6szrrOu861zrnOsSDOm865z4zOvc+Ezrc=?=', 'Ηράκλεια Λιόντη'),
-            ('=?utf-8?b?15nXqdeo15DXnCDXqNeV15bXoNek15zXkw==?=', 'ישראל רוזנפלד'),
-            ('=?utf-8?b?5Li95Y2OIOeahw==?=', '丽华 皇'),
-            ('=?utf-8?b?77ul77qu766V77qzIO+tlu+7ru+vvu+6ju+7pw==?=', 'ﻥﺮﮕﺳ ﭖﻮﯾﺎﻧ'),
-            ('=?utf-8?b?77uh77uu77qz77uu76++IO+6su+7tO+7p++6jSDvurDvu6Pvuo7vu6jvr74=?=', 'ﻡﻮﺳﻮﯾ ﺲﻴﻧﺍ ﺰﻣﺎﻨﯾ'),
-            ('=?utf-8?b?ScOxaWdvIFNhbsOnIEliw6HDsWV6IGRlIGxhIFBlw7Fh?=', 'Iñigo Sanç Ibáñez de la Peña'),
-            ('Mart van Oostendorp', 'Mart van Oostendorp'),
-            ('', ''),
-            )
-        for encoded_str, unicode in names: 
-            self.assertEqual(unicode, parse_unicode(encoded_str))
-
 class TestAndroidSiteManifest(TestCase):
     def test_manifest(self):
         r = self.client.get(urlreverse('site.webmanifest'))
diff --git a/ietf/utils/tests_text.py b/ietf/utils/tests_text.py
new file mode 100644
index 0000000000..51aa2eff13
--- /dev/null
+++ b/ietf/utils/tests_text.py
@@ -0,0 +1,71 @@
+# Copyright The IETF Trust 2021-2026, All Rights Reserved
+from ietf.utils.test_utils import TestCase
+from ietf.utils.text import parse_unicode, decode_document_content
+
+
+class TestDecoders(TestCase):
+    def test_parse_unicode(self):
+        names = (
+            ("=?utf-8?b?4Yuz4YuK4Ym1IOGJoOGJgOGIiA==?=", "ዳዊት በቀለ"),
+            ("=?utf-8?b?5Li9IOmDnA==?=", "丽 郜"),
+            ("=?utf-8?b?4KSV4KSu4KWN4KSs4KWL4KScIOCkoeCkvuCksA==?=", "कम्बोज डार"),
+            ("=?utf-8?b?zpfPgc6szrrOu861zrnOsSDOm865z4zOvc+Ezrc=?=", "Ηράκλεια Λιόντη"),
+            ("=?utf-8?b?15nXqdeo15DXnCDXqNeV15bXoNek15zXkw==?=", "ישראל רוזנפלד"),
+            ("=?utf-8?b?5Li95Y2OIOeahw==?=", "丽华 皇"),
+            ("=?utf-8?b?77ul77qu766V77qzIO+tlu+7ru+vvu+6ju+7pw==?=", "ﻥﺮﮕﺳ ﭖﻮﯾﺎﻧ"),
+            (
+                "=?utf-8?b?77uh77uu77qz77uu76++IO+6su+7tO+7p++6jSDvurDvu6Pvuo7vu6jvr74=?=",
+                "ﻡﻮﺳﻮﯾ ﺲﻴﻧﺍ ﺰﻣﺎﻨﯾ",
+            ),
+            (
+                "=?utf-8?b?ScOxaWdvIFNhbsOnIEliw6HDsWV6IGRlIGxhIFBlw7Fh?=",
+                "Iñigo Sanç Ibáñez de la Peña",
+            ),
+            ("Mart van Oostendorp", "Mart van Oostendorp"),
+            ("", ""),
+        )
+        for encoded_str, unicode in names:
+            self.assertEqual(unicode, parse_unicode(encoded_str))
+
+    def test_decode_document_content(self):
+        utf8_bytes = "𒀭𒊩𒌆𒄈𒋢".encode("utf-8")  # ends with 4-byte character
+        latin1_bytes = "àéîøü".encode("latin-1")
+        other_bytes = "àéîøü".encode("macintosh")  # different from its latin-1 encoding
+        assert other_bytes.decode("macintosh") != other_bytes.decode("latin-1"),\
+            "test broken: other_bytes must decode differently as latin-1"
+
+        # simplest case
+        self.assertEqual(
+            decode_document_content(utf8_bytes),
+            utf8_bytes.decode(),
+        )
+        # losing 1-4 bytes from the end leave the last character incomplete; the
+        # decoder should decode all but that last character
+        self.assertEqual(
+            decode_document_content(utf8_bytes[:-1]),
+            utf8_bytes.decode()[:-1],
+        )
+        self.assertEqual(
+            decode_document_content(utf8_bytes[:-2]),
+            utf8_bytes.decode()[:-1],
+        )
+        self.assertEqual(
+            decode_document_content(utf8_bytes[:-3]),
+            utf8_bytes.decode()[:-1],
+        )
+        self.assertEqual(
+            decode_document_content(utf8_bytes[:-4]),
+            utf8_bytes.decode()[:-1],
+        )
+
+        # latin-1 is also simple
+        self.assertEqual(
+            decode_document_content(latin1_bytes),
+            latin1_bytes.decode("latin-1"),
+        )
+
+        # other character sets are just treated as latin1 (bug? feature? you decide)
+        self.assertEqual(
+            decode_document_content(other_bytes),
+            other_bytes.decode("latin-1"),
+        )
diff --git a/ietf/utils/text.py b/ietf/utils/text.py
index 590ec3fd30..2763056e1a 100644
--- a/ietf/utils/text.py
+++ b/ietf/utils/text.py
@@ -263,3 +263,21 @@ def parse_unicode(text):
     else:
         text = decoded_string
     return text
+
+
+def decode_document_content(content: bytes) -> str:
+    """Decode document contents as utf-8 or latin1
+    
+    Method was developed in DocumentInfo.text() where it gave acceptable results
+    for existing documents / RFCs.
+    """
+    try:
+        return content.decode("utf-8")
+    except UnicodeDecodeError:
+        pass
+    for back in range(1, 4):
+        try:
+            return content[:-back].decode("utf-8")
+        except UnicodeDecodeError:
+            pass
+    return content.decode("latin-1")  # everything is legal in latin-1

From 63a69945ab11b1c3b3ec490fb260073c90eed0bc Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Fri, 17 Apr 2026 16:24:18 -0500
Subject: [PATCH 654/656] test: Squash some transient test error vectors
 (#10730)

* test: enforce queryset order assumed by test

* test: match html escaping in test

* test: search more specifically for tokens to avoid mis-reading them when they occur in faker data
---
 ietf/group/tests_review.py             | 30 +++++++++++++-------------
 ietf/meeting/tests_session_requests.py |  2 +-
 ietf/meeting/tests_views.py            |  7 +++---
 3 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/ietf/group/tests_review.py b/ietf/group/tests_review.py
index 89c755bb26..bb9b79a416 100644
--- a/ietf/group/tests_review.py
+++ b/ietf/group/tests_review.py
@@ -888,10 +888,10 @@ def test_requests_history_filter_page(self):
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, review_req.doc.name)
         self.assertContains(r, review_req2.doc.name)
-        self.assertContains(r, 'Assigned')
-        self.assertContains(r, 'Accepted')
-        self.assertContains(r, 'Completed')
-        self.assertContains(r, 'Ready')
+        self.assertContains(r, 'data-text="Assigned"')
+        self.assertContains(r, 'data-text="Accepted"')
+        self.assertContains(r, 'data-text="Completed"')
+        self.assertContains(r, 'data-text="Ready"')
         self.assertContains(r, escape(assignment.reviewer.person.name))
         self.assertContains(r, escape(assignment2.reviewer.person.name))
 
@@ -907,10 +907,10 @@ def test_requests_history_filter_page(self):
             self.assertEqual(r.status_code, 200)
             self.assertContains(r, review_req.doc.name)
             self.assertNotContains(r, review_req2.doc.name)
-            self.assertContains(r, 'Assigned')
-            self.assertNotContains(r, 'Accepted')
-            self.assertNotContains(r, 'Completed')
-            self.assertNotContains(r, 'Ready')
+            self.assertContains(r, 'data-text="Assigned"')
+            self.assertNotContains(r, 'data-text="Accepted"')
+            self.assertNotContains(r, 'data-text="Completed"')
+            self.assertNotContains(r, 'data-text="Ready"')
             self.assertContains(r, escape(assignment.reviewer.person.name))
             self.assertNotContains(r, escape(assignment2.reviewer.person.name))
 
@@ -926,10 +926,10 @@ def test_requests_history_filter_page(self):
             self.assertEqual(r.status_code, 200)
             self.assertNotContains(r, review_req.doc.name)
             self.assertContains(r, review_req2.doc.name)
-            self.assertNotContains(r, 'Assigned')
-            self.assertContains(r, 'Accepted')
-            self.assertContains(r, 'Completed')
-            self.assertContains(r, 'Ready')
+            self.assertNotContains(r, 'data-text="Assigned"')
+            self.assertContains(r, 'data-text="Accepted"')
+            self.assertContains(r, 'data-text="Completed"')
+            self.assertContains(r, 'data-text="Ready"')
             self.assertNotContains(r, escape(assignment.reviewer.person.name))
             self.assertContains(r, escape(assignment2.reviewer.person.name))
 
@@ -940,9 +940,9 @@ def test_requests_history_filter_page(self):
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         self.assertNotContains(r, review_req.doc.name)
-        self.assertNotContains(r, 'Assigned')
-        self.assertNotContains(r, 'Accepted')
-        self.assertNotContains(r, 'Completed')
+        self.assertNotContains(r, 'data-text="Assigned"')
+        self.assertNotContains(r, 'data-text="Accepted"')
+        self.assertNotContains(r, 'data-text="Completed"')
 
     def test_requests_history_invalid_filter_parameters(self):
         # First assignment as assigned
diff --git a/ietf/meeting/tests_session_requests.py b/ietf/meeting/tests_session_requests.py
index 0cb092d2f8..42dbee5f23 100644
--- a/ietf/meeting/tests_session_requests.py
+++ b/ietf/meeting/tests_session_requests.py
@@ -236,7 +236,7 @@ def test_edit(self):
         self.assertRedirects(r, redirect_url)
 
         # Check whether updates were stored in the database
-        sessions = Session.objects.filter(meeting=meeting, group=mars)
+        sessions = Session.objects.filter(meeting=meeting, group=mars).order_by("id")
         self.assertEqual(len(sessions), 2)
         session = sessions[0]
         self.assertFalse(session.constraints().filter(name='time_relation'))
diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py
index 258ffe554c..17988e50be 100644
--- a/ietf/meeting/tests_views.py
+++ b/ietf/meeting/tests_views.py
@@ -33,6 +33,7 @@
 from django.http import QueryDict, FileResponse
 from django.template import Context, Template
 from django.utils import timezone
+from django.utils.html import escape
 from django.utils.safestring import mark_safe
 from django.utils.text import slugify
 
@@ -9491,7 +9492,7 @@ def test_session_attendance(self):
         self.assertEqual(r.status_code, 200)  
         self.assertContains(r, '3 attendees')
         for person in persons:
-            self.assertContains(r, person.plain_name())
+            self.assertContains(r, escape(person.plain_name()))
 
         # Test for the "I was there" button.
         def _test_button(person, expected):
@@ -9511,14 +9512,14 @@ def _test_button(person, expected):
         # attempt to POST anyway is ignored
         r = self.client.post(attendance_url)
         self.assertEqual(r.status_code, 200)
-        self.assertNotContains(r, persons[3].plain_name())
+        self.assertNotContains(r, escape(persons[3].plain_name()))
         self.assertEqual(session.attended_set.count(), 3)
         # button is shown, and POST is accepted
         meeting.importantdate_set.update(name_id='revsub',date=date_today() + datetime.timedelta(days=20))
         _test_button(persons[3], True)
         r = self.client.post(attendance_url)
         self.assertEqual(r.status_code, 200)
-        self.assertContains(r, persons[3].plain_name())
+        self.assertContains(r, escape(persons[3].plain_name()))
         self.assertEqual(session.attended_set.count(), 4)
 
         # When the meeting is finalized, a bluesheet file is generated,

From dc49dc8362812893cad560feecc55efcea1553dc Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 20 Apr 2026 14:29:41 -0300
Subject: [PATCH 655/656] chore: beat termination grace period -> 10 s (#10741)

---
 k8s/beat.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k8s/beat.yaml b/k8s/beat.yaml
index 9ab242681c..b4291c7e31 100644
--- a/k8s/beat.yaml
+++ b/k8s/beat.yaml
@@ -59,4 +59,4 @@ spec:
             name: files-cfgmap
       dnsPolicy: ClusterFirst
       restartPolicy: Always
-      terminationGracePeriodSeconds: 600
+      terminationGracePeriodSeconds: 10

From 4d69329ef86054fa5bfb9da9acd0c966ab013d8f Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 20 Apr 2026 23:59:36 -0300
Subject: [PATCH 656/656] chore: remove blobdb profiling logs (#10732)

These are not useful any more, blobdb is fast
---
 ietf/doc/storage.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ietf/doc/storage.py b/ietf/doc/storage.py
index 375620ccaf..ee1e76c4fa 100644
--- a/ietf/doc/storage.py
+++ b/ietf/doc/storage.py
@@ -114,7 +114,6 @@ def _get_write_parameters(self, name, content=None):
 
 
 class StoredObjectBlobdbStorage(BlobdbStorage):
-    ietf_log_blob_timing = True
     warn_if_missing = True  # TODO-BLOBSTORE make this configurable (or remove it)
 
     def _save_stored_object(self, name, content) -> StoredObject: