From 711313d983d1d5ec9afe8c014a76959e04c04632 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 19 Aug 2025 12:08:11 -0300 Subject: [PATCH 001/313] chore: postgresql-client-17 for base.Dockerfile (#9378) --- docker/base.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index f364456c7af..57aac8ee561 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -69,7 +69,7 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends nodejs \ pgloader \ pigz \ - postgresql-client-14 \ + postgresql-client-17 \ pv \ python3-ipython \ ripgrep \ From 811216838823507b911635967a012bc41f43edd5 Mon Sep 17 00:00:00 2001 From: rjsparks <10996692+rjsparks@users.noreply.github.com> Date: Tue, 19 Aug 2025 15:21:46 +0000 Subject: [PATCH 002/313] ci: update base image target version to 20250819T1508 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 0ecf9566efd..756b7021b7a 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250807T1514 +FROM ghcr.io/ietf-tools/datatracker-app-base:20250819T1508 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 327fb48da42..2e0f1519dab 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250807T1514 +20250819T1508 From 566cf50b39fa4b0e7882a12b8ce977c29192009f Mon Sep 17 00:00:00 2001 From: "A. Nour" Date: Tue, 19 Aug 2025 17:52:56 +0100 Subject: [PATCH 003/313] feat: add auto-generated IPR email notifications on WG Call for Adoption or WG Last Call (#9322) * feat(documents): add auto-generated IPR email notifications on WG Call for Adoption or WG Last Call * fix: edit call durations in mails and call logic under new_state in views * fix: calc end_date as 7 * call_duration * feat(mailtrigger): added new mailtrigger for wg-lc and rfc stream states * test: add mailtrigger test fixtures and new tests * fix: use two action-oriented mailtrigger names The two actions have the same recipients to start with, but that may change over time. Mailtrigger names should describe "what happened to trigger this email?". Changed the utility names to match the actions. * fix: send from whomever issued the call Using a list name as the From will not work - the mail infrastructure blocks such mail when it is submitted. * chore: revert ietf/doc/tests_draft.py * fix: trigger call for adoption email from manage adoption view * fix: changed template names to match functions * fix: match the subject requested in the issue * fix: Initial tests * fix: pass duration to the email message generator * fix: only issue the c-adopt and wg-lc email for ietf-stream docs * chore: remove stray whitespace --------- Co-authored-by: Robert Sparks --- ietf/doc/mails.py | 55 +++++ ietf/doc/tests_draft.py | 205 +++++++++++++++++- ietf/doc/views_draft.py | 17 +- ..._call_for_adoption_and_last_call_issued.py | 43 ++++ ietf/name/fixtures/names.json | 28 +++ .../doc/mail/wg_call_for_adoption_issued.txt | 21 ++ .../doc/mail/wg_last_call_issued.txt | 22 ++ 7 files changed, 385 insertions(+), 6 deletions(-) create mode 100644 ietf/mailtrigger/migrations/0006_call_for_adoption_and_last_call_issued.py create mode 100644 ietf/templates/doc/mail/wg_call_for_adoption_issued.txt create mode 100644 ietf/templates/doc/mail/wg_last_call_issued.txt diff --git a/ietf/doc/mails.py b/ietf/doc/mails.py index ddecbb6b54b..f20d398c3cb 100644 --- a/ietf/doc/mails.py +++ b/ietf/doc/mails.py @@ -103,6 +103,61 @@ def email_stream_changed(request, doc, old_stream, new_stream, text=""): dict(text=text, url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url()), cc=cc) + +def email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=None): + if cfa_duration_weeks is None: + cfa_duration_weeks=2 + (to, cc) = gather_address_lists("doc_wg_call_for_adoption_issued", doc=doc) + frm = request.user.person.formatted_email() + + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=7 * cfa_duration_weeks) + + subject = f"Call for adoption: {doc.name}-{doc.rev} (Ends {end_date})" + + send_mail( + request, + to, + frm, + subject, + "doc/mail/wg_call_for_adoption_issued.txt", + dict( + doc=doc, + subject=subject, + url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(), + end_date=end_date, + cfa_duration_weeks=cfa_duration_weeks, + wg_list=doc.group.list_email, + ), + cc=cc, + ) + + +def email_wg_last_call_issued(request, doc, wglc_duration_weeks=None): + if wglc_duration_weeks is None: + wglc_duration_weeks = 2 + (to, cc) = gather_address_lists("doc_wg_last_call_issued", doc=doc) + frm = request.user.person.formatted_email() + + + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=7 * wglc_duration_weeks) + subject = f"WG Last Call: {doc.name}-{doc.rev} (Ends {end_date})" + + send_mail( + request, + to, + frm, + subject, + "doc/mail/wg_last_call_issued.txt", + dict( + doc=doc, + subject=subject, + url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(), + end_date=end_date, + wglc_duration_weeks=wglc_duration_weeks, + wg_list=doc.group.list_email, + ), + cc=cc, + ) def email_pulled_from_rfc_queue(request, doc, comment, prev_state, next_state): extra=extra_automation_headers(doc) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index 576feb05821..ab7eaba7684 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -1707,11 +1707,12 @@ def test_adopt_document(self): self.assertEqual(draft.group, chair_role.group) self.assertEqual(draft.stream_id, stream_state_type_slug[type_id][13:]) # trim off "draft-stream-" self.assertEqual(draft.docevent_set.count() - events_before, 5) - self.assertEqual(len(outbox), 1) - self.assertTrue("Call For Adoption" in outbox[-1]["Subject"]) - self.assertTrue(f"{chair_role.group.acronym}-chairs@" in outbox[-1]['To']) - self.assertTrue(f"{draft.name}@" in outbox[-1]['To']) - self.assertTrue(f"{chair_role.group.acronym}@" in outbox[-1]['To']) + self.assertEqual(len(outbox), 2) + self.assertTrue("Call For Adoption" in outbox[0]["Subject"]) + self.assertTrue(f"{chair_role.group.acronym}-chairs@" in outbox[0]['To']) + self.assertTrue(f"{draft.name}@" in outbox[0]['To']) + self.assertTrue(f"{chair_role.group.acronym}@" in outbox[0]['To']) + # contents of outbox[1] are tested elsewhere # adopt empty_outbox() @@ -2001,6 +2002,200 @@ def test_set_state(self): self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string()) self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string()) + def test_wg_call_for_adoption_issued(self): + role = RoleFactory( + name_id="chair", + group__acronym="mars", + group__list_email="mars-wg@ietf.org", + person__user__username="marschairman", + person__name="WG Cháir Man", + ) + # First test the usual workflow through the manage adoption view + draft = IndividualDraftFactory() + url = urlreverse( + "ietf.doc.views_draft.adopt_draft", kwargs=dict(name=draft.name) + ) + login_testing_unauthorized(self, "marschairman", url) + empty_outbox() + call_issued = State.objects.get(type="draft-stream-ietf", slug="c-adopt") + r = self.client.post( + url, + dict( + comment="some comment", + group=role.group.pk, + newstate=call_issued.pk, + weeks="10", + ), + ) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 2) + self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) + self.assertIn("Call for adoption", outbox[1]["Subject"]) + body = get_payload_text(outbox[1]) + self.assertIn("disclosure obligations", body) + self.assertIn("starts a 10-week", body) + # Test not entering a duration on the form + draft = IndividualDraftFactory() + url = urlreverse( + "ietf.doc.views_draft.adopt_draft", kwargs=dict(name=draft.name) + ) + empty_outbox() + call_issued = State.objects.get(type="draft-stream-ietf", slug="c-adopt") + r = self.client.post( + url, + dict( + comment="some comment", + group=role.group.pk, + newstate=call_issued.pk, + ), + ) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 2) + self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) + self.assertIn("Call for adoption", outbox[1]["Subject"]) + body = get_payload_text(outbox[1]) + self.assertIn("disclosure obligations", body) + self.assertIn("starts a 2-week", body) + + # Test the less usual workflow of issuing a call for adoption + # of a document that's already in the ietf stream + draft = WgDraftFactory(group=role.group) + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=draft.name, state_type="draft-stream-ietf"), + ) + old_state = draft.get_state("draft-stream-%s" % draft.stream_id) + new_state = State.objects.get( + used=True, type="draft-stream-%s" % draft.stream_id, slug="c-adopt" + ) + self.assertNotEqual(old_state, new_state) + empty_outbox() + r = self.client.post( + url, + dict( + new_state=new_state.pk, + comment="some comment", + weeks="10", + tags=[ + t.pk + for t in draft.tags.filter( + slug__in=get_tags_for_stream_id(draft.stream_id) + ) + ], + ), + ) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 2) + self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) + self.assertIn("Call for adoption", outbox[1]["Subject"]) + body = get_payload_text(outbox[1]) + self.assertIn("disclosure obligations", body) + self.assertIn("starts a 10-week", body) + draft = WgDraftFactory(group=role.group) + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=draft.name, state_type="draft-stream-ietf"), + ) + old_state = draft.get_state("draft-stream-%s" % draft.stream_id) + new_state = State.objects.get( + used=True, type="draft-stream-%s" % draft.stream_id, slug="c-adopt" + ) + self.assertNotEqual(old_state, new_state) + empty_outbox() + r = self.client.post( + url, + dict( + new_state=new_state.pk, + comment="some comment", + tags=[ + t.pk + for t in draft.tags.filter( + slug__in=get_tags_for_stream_id(draft.stream_id) + ) + ], + ), + ) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 2) + self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) + self.assertIn("Call for adoption", outbox[1]["Subject"]) + body = get_payload_text(outbox[1]) + self.assertIn("disclosure obligations", body) + self.assertIn("starts a 2-week", body) + + def test_wg_last_call_issued(self): + role = RoleFactory( + name_id="chair", + group__acronym="mars", + group__list_email="mars-wg@ietf.org", + person__user__username="marschairman", + person__name="WG Cháir Man", + ) + draft = WgDraftFactory(group=role.group) + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=draft.name, state_type="draft-stream-ietf"), + ) + login_testing_unauthorized(self, "marschairman", url) + old_state = draft.get_state("draft-stream-%s" % draft.stream_id) + new_state = State.objects.get( + used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-lc" + ) + self.assertNotEqual(old_state, new_state) + empty_outbox() + r = self.client.post( + url, + dict( + new_state=new_state.pk, + comment="some comment", + weeks="10", + tags=[ + t.pk + for t in draft.tags.filter( + slug__in=get_tags_for_stream_id(draft.stream_id) + ) + ], + ), + ) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 2) + self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) + self.assertIn("WG Last Call", outbox[1]["Subject"]) + body = get_payload_text(outbox[1]) + self.assertIn("disclosure obligations", body) + self.assertIn("starts a 10-week", body) + draft = WgDraftFactory(group=role.group) + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=draft.name, state_type="draft-stream-ietf"), + ) + old_state = draft.get_state("draft-stream-%s" % draft.stream_id) + new_state = State.objects.get( + used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-lc" + ) + self.assertNotEqual(old_state, new_state) + empty_outbox() + r = self.client.post( + url, + dict( + new_state=new_state.pk, + comment="some comment", + tags=[ + t.pk + for t in draft.tags.filter( + slug__in=get_tags_for_stream_id(draft.stream_id) + ) + ], + ), + ) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 2) + self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) + self.assertIn("WG Last Call", outbox[1]["Subject"]) + body = get_payload_text(outbox[1]) + self.assertIn("disclosure obligations", body) + self.assertIn("starts a 2-week", body) + def test_pubreq_validation(self): role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') RoleFactory(name_id='delegate',group=role.group,person__user__email='marsdelegate@ietf.org') diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py index c80537afb37..16d04ee66aa 100644 --- a/ietf/doc/views_draft.py +++ b/ietf/doc/views_draft.py @@ -28,6 +28,7 @@ IanaExpertDocEvent, IESG_SUBSTATE_TAGS) from ietf.doc.mails import ( email_pulled_from_rfc_queue, email_resurrect_requested, email_resurrection_completed, email_state_changed, email_stream_changed, + email_wg_call_for_adoption_issued, email_wg_last_call_issued, email_stream_state_changed, email_stream_tags_changed, extra_automation_headers, generate_publication_request, email_adopted, email_intended_status_changed, email_iesg_processing_document, email_ad_approved_doc, @@ -1568,8 +1569,15 @@ def adopt_draft(request, name): update_reminder(doc, "stream-s", e, due_date) + # The following call name is very misleading - the view allows + # setting states that are _not_ the adopted state. email_adopted(request, doc, prev_state, new_state, by, comment) + # Currently only the IETF stream uses the c-adopt state - guard against other + # streams starting to use it asthe IPR rules for those streams will be different. + if doc.stream_id == "ietf" and new_state.slug == "c-adopt": + email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=form.cleaned_data["weeks"]) + # comment if comment: e = DocEvent(type="added_comment", doc=doc, rev=doc.rev, by=by) @@ -1754,13 +1762,20 @@ def change_stream_state(request, name, state_type): events.append(e) due_date = None - if form.cleaned_data["weeks"] != None: + if form.cleaned_data["weeks"] is not None: due_date = datetime_today(DEADLINE_TZINFO) + datetime.timedelta(weeks=form.cleaned_data["weeks"]) update_reminder(doc, "stream-s", e, due_date) email_stream_state_changed(request, doc, prev_state, new_state, by, comment) + if doc.stream_id == "ietf": + if new_state.slug == "c-adopt": + email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=form.cleaned_data["weeks"]) + + if new_state.slug == "wg-lc": + email_wg_last_call_issued(request, doc, wglc_duration_weeks=form.cleaned_data["weeks"]) + # tags existing_tags = set(doc.tags.all()) new_tags = set(form.cleaned_data["tags"]) diff --git a/ietf/mailtrigger/migrations/0006_call_for_adoption_and_last_call_issued.py b/ietf/mailtrigger/migrations/0006_call_for_adoption_and_last_call_issued.py new file mode 100644 index 00000000000..7adad150eba --- /dev/null +++ b/ietf/mailtrigger/migrations/0006_call_for_adoption_and_last_call_issued.py @@ -0,0 +1,43 @@ +# Copyright The IETF Trust 2023, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + MailTrigger = apps.get_model("mailtrigger", "MailTrigger") + Recipient = apps.get_model("mailtrigger", "Recipient") + recipients = list( + Recipient.objects.filter( + slug__in=( + "doc_group_mail_list", + "doc_authors", + "doc_group_chairs", + "doc_shepherd", + ) + ) + ) + call_for_adoption = MailTrigger.objects.create( + slug="doc_wg_call_for_adoption_issued", + desc="Recipients when a working group call for adoption is issued", + ) + call_for_adoption.to.add(*recipients) + wg_last_call = MailTrigger.objects.create( + slug="doc_wg_last_call_issued", + desc="Recipients when a working group last call is issued", + ) + wg_last_call.to.add(*recipients) + + +def reverse(apps, schema_editor): + MailTrigger = apps.get_model("mailtrigger", "MailTrigger") + MailTrigger.objects.filter( + slug_in=("doc_wg_call_for_adoption_issued", "doc_wg_last_call_issued") + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("mailtrigger", "0005_rfc_recipients"), + ] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 0724cbb4b59..c94e15a4591 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -5116,6 +5116,34 @@ "model": "mailtrigger.mailtrigger", "pk": "doc_telechat_details_changed" }, + { + "fields": { + "cc": [], + "desc": "Recipients when a working group call for adoption is issued", + "to": [ + "doc_authors", + "doc_group_chairs", + "doc_group_mail_list", + "doc_shepherd" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "doc_wg_call_for_adoption_issued" + }, + { + "fields": { + "cc": [], + "desc": "Recipients when a working group last call is issued", + "to": [ + "doc_authors", + "doc_group_chairs", + "doc_group_mail_list", + "doc_shepherd" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "doc_wg_last_call_issued" + }, { "fields": { "cc": [], diff --git a/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt b/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt new file mode 100644 index 00000000000..c4a2401bc29 --- /dev/null +++ b/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt @@ -0,0 +1,21 @@ +{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %} +Subject: {{ subject }} + +This message starts a {{ cfa_duration_weeks }}-week Call for Adoption for this document. + +Abstract: +{{ doc.abstract }} + +File can be retrieved from: +{{ url }} + +Please reply to this message keeping {{ wg_list }} in copy by indicating whether you support or not the adoption of this draft as a WG document. Comments to motivate your preference are highly appreciated. + +Authors, and WG participants in general, are reminded of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [2]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3]. + +Thank you. +[1] https://datatracker.ietf.org/doc/bcp78/ +[2] https://datatracker.ietf.org/doc/bcp79/ +[3] https://datatracker.ietf.org/doc/rfc6701/ +{% endfilter %} +{% endautoescape %} diff --git a/ietf/templates/doc/mail/wg_last_call_issued.txt b/ietf/templates/doc/mail/wg_last_call_issued.txt new file mode 100644 index 00000000000..35b1e149d7c --- /dev/null +++ b/ietf/templates/doc/mail/wg_last_call_issued.txt @@ -0,0 +1,22 @@ +{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %} +Subject: {{ subject }} + +This message starts a {{ wglc_duration_weeks }}-week WG Last Call for this document. + +Abstract: +{{ doc.abstract }} + +File can be retrieved from: +{{ url }} + +Please review and indicate your support or objection to proceed with the publication of this document by replying to this email keeping {{ wg_list }} in copy. Objections should be motivated and suggestions to resolve them are highly appreciated. + +Authors, and WG participants in general, are reminded again of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [1]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3]. + +Thank you. + +[1] https://datatracker.ietf.org/doc/bcp78/ +[2] https://datatracker.ietf.org/doc/bcp79/ +[3] https://datatracker.ietf.org/doc/rfc6701/ +{% endfilter %} +{% endautoescape %} From 1e451fbac105b18a3539c406b8380890ee701ef8 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 19 Aug 2025 18:12:38 -0300 Subject: [PATCH 004/313] ci: replace DOCKER_BUILD_NO_SUMMARY (#9383) Option is deprecated; replace with DOCKER_BUILD_SUMMARY: false --- .github/workflows/build-base-app.yml | 2 +- .github/workflows/build-devblobstore.yml | 2 +- .github/workflows/build-mq-broker.yml | 2 +- .github/workflows/dev-assets-sync-nightly.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 479cd7cadf6..ef8a17f6b43 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -43,7 +43,7 @@ jobs: - name: Docker Build & Push uses: docker/build-push-action@v6 env: - DOCKER_BUILD_NO_SUMMARY: true + DOCKER_BUILD_SUMMARY: false with: context: . file: docker/base.Dockerfile diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml index 2d8214b4481..f49a11af19c 100644 --- a/.github/workflows/build-devblobstore.yml +++ b/.github/workflows/build-devblobstore.yml @@ -35,7 +35,7 @@ jobs: - name: Docker Build & Push uses: docker/build-push-action@v6 env: - DOCKER_BUILD_NO_SUMMARY: true + DOCKER_BUILD_SUMMARY: false with: context: . file: docker/devblobstore.Dockerfile diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index 85c27c23cc3..4de861dbcd3 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -42,7 +42,7 @@ jobs: - name: Docker Build & Push uses: docker/build-push-action@v6 env: - DOCKER_BUILD_NO_SUMMARY: true + DOCKER_BUILD_SUMMARY: false with: context: . file: dev/mq/Dockerfile diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index a7fe67f012e..19933bddfdd 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -41,7 +41,7 @@ jobs: - name: Docker Build & Push uses: docker/build-push-action@v6 env: - DOCKER_BUILD_NO_SUMMARY: true + DOCKER_BUILD_SUMMARY: false with: context: . file: dev/shared-assets-sync/Dockerfile From b7da3d7a779f310f765f205119624229d9a860d7 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 21 Aug 2025 10:25:49 -0300 Subject: [PATCH 005/313] fix: escape linkify filter input (#9389) * fix: escape linkify filter input * test: exercise linkify * chore: lint --- ietf/utils/templatetags/tests.py | 66 ++++++++++++++++++++++++++ ietf/utils/templatetags/textfilters.py | 10 ++-- ietf/utils/text.py | 6 +++ 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/ietf/utils/templatetags/tests.py b/ietf/utils/templatetags/tests.py index a93bf2d94d6..859319be3dc 100644 --- a/ietf/utils/templatetags/tests.py +++ b/ietf/utils/templatetags/tests.py @@ -3,6 +3,7 @@ from django.template import Context, Origin, Template from django.test import override_settings +from ietf.utils.templatetags.textfilters import linkify from ietf.utils.test_utils import TestCase import debug # pyflakes: ignore @@ -39,3 +40,68 @@ def test_origin_outside_base_dir(self): output = template.render(Context()) self.assertNotIn(component, output, 'Full path components should not be revealed in html') + + +class TextfiltersTests(TestCase): + def test_linkify(self): + # Cases with autoescape = True (the default) + self.assertEqual( + linkify("plain string"), + "plain string", + ) + self.assertEqual( + linkify("https://www.ietf.org"), + 'https://www.ietf.org', + ) + self.assertEqual( + linkify('IETF'), + ( + '<a href="https://www.ietf.org">IETF</a>' + ), + ) + self.assertEqual( + linkify("somebody@example.com"), + 'somebody@example.com', + ) + self.assertEqual( + linkify("Some Body "), + ( + 'Some Body <' + 'somebody@example.com>' + ), + ) + self.assertEqual( + linkify(""), + "<script>alert('h4x0r3d');</script>", + ) + + # Cases with autoescape = False (these are dangerous and assume the caller + # has sanitized already) + self.assertEqual( + linkify("plain string", autoescape=False), + "plain string", + ) + self.assertEqual( + linkify("https://www.ietf.org", autoescape=False), + 'https://www.ietf.org', + ) + self.assertEqual( + linkify('IETF', autoescape=False), + 'IETF', + ) + self.assertEqual( + linkify("somebody@example.com", autoescape=False), + 'somebody@example.com', + ) + # bleach.Linkifier translates the < -> < and > -> > on this one + self.assertEqual( + linkify("Some Body ", autoescape=False), + ( + 'Some Body <' + 'somebody@example.com>' + ), + ) + self.assertEqual( + linkify("", autoescape=False), + "", + ) diff --git a/ietf/utils/templatetags/textfilters.py b/ietf/utils/templatetags/textfilters.py index 3b240740e04..e3bfbe0c562 100644 --- a/ietf/utils/templatetags/textfilters.py +++ b/ietf/utils/templatetags/textfilters.py @@ -7,6 +7,7 @@ from django import template from django.conf import settings from django.template.defaultfilters import stringfilter +from django.utils.html import conditional_escape from django.utils.safestring import mark_safe import debug # pyflakes:ignore @@ -71,10 +72,13 @@ def texescape_filter(value): "A TeX escape filter" return texescape(value) -@register.filter +@register.filter(needs_autoescape=True) @stringfilter -def linkify(value): - text = mark_safe(_linkify(value)) +def linkify(value, autoescape=True): + if autoescape: + # Escape unless the input was already a SafeString + value = conditional_escape(value) + text = mark_safe(_linkify(value)) # _linkify is a safe operation return text @register.filter diff --git a/ietf/utils/text.py b/ietf/utils/text.py index 4e5d5b6cd5a..590ec3fd30a 100644 --- a/ietf/utils/text.py +++ b/ietf/utils/text.py @@ -60,6 +60,12 @@ def check_url_validity(attrs, new=False): def linkify(text): + """Convert URL-ish substrings into HTML links + + This does no sanitization whatsoever. Caller must sanitize the input or output as + contextually appropriate. Do not call `mark_safe()` on the output if the input is + user-provided unless it has been sanitized or escaped. + """ return _bleach_linker.linkify(text) From 450ffd8e9e586fbfbc9c583a01ee4770de4f62d9 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 21 Aug 2025 10:26:38 -0300 Subject: [PATCH 006/313] chore(deps): bump pytz (#9385) * chore(deps): bump pytz * chore(deps): fix copy/paste error --- ...meeting_country_alter_meeting_time_zone.py | 693 ++++++++++++++++++ requirements.txt | 4 +- 2 files changed, 695 insertions(+), 2 deletions(-) create mode 100644 ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py diff --git a/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py b/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py new file mode 100644 index 00000000000..8f5db261127 --- /dev/null +++ b/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py @@ -0,0 +1,693 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("meeting", "0015_alter_meeting_time_zone"), + ] + + operations = [ + migrations.AlterField( + model_name="meeting", + name="country", + field=models.CharField( + blank=True, + choices=[ + ("", "---------"), + ("AF", "Afghanistan"), + ("AL", "Albania"), + ("DZ", "Algeria"), + ("AD", "Andorra"), + ("AO", "Angola"), + ("AI", "Anguilla"), + ("AQ", "Antarctica"), + ("AG", "Antigua & Barbuda"), + ("AR", "Argentina"), + ("AM", "Armenia"), + ("AW", "Aruba"), + ("AU", "Australia"), + ("AT", "Austria"), + ("AZ", "Azerbaijan"), + ("BS", "Bahamas"), + ("BH", "Bahrain"), + ("BD", "Bangladesh"), + ("BB", "Barbados"), + ("BY", "Belarus"), + ("BE", "Belgium"), + ("BZ", "Belize"), + ("BJ", "Benin"), + ("BM", "Bermuda"), + ("BT", "Bhutan"), + ("BO", "Bolivia"), + ("BA", "Bosnia & Herzegovina"), + ("BW", "Botswana"), + ("BV", "Bouvet Island"), + ("BR", "Brazil"), + ("GB", "Britain (UK)"), + ("IO", "British Indian Ocean Territory"), + ("BN", "Brunei"), + ("BG", "Bulgaria"), + ("BF", "Burkina Faso"), + ("BI", "Burundi"), + ("KH", "Cambodia"), + ("CM", "Cameroon"), + ("CA", "Canada"), + ("CV", "Cape Verde"), + ("BQ", "Caribbean NL"), + ("KY", "Cayman Islands"), + ("CF", "Central African Rep."), + ("TD", "Chad"), + ("CL", "Chile"), + ("CN", "China"), + ("CX", "Christmas Island"), + ("CC", "Cocos (Keeling) Islands"), + ("CO", "Colombia"), + ("KM", "Comoros"), + ("CD", "Congo (Dem. Rep.)"), + ("CG", "Congo (Rep.)"), + ("CK", "Cook Islands"), + ("CR", "Costa Rica"), + ("HR", "Croatia"), + ("CU", "Cuba"), + ("CW", "Curaçao"), + ("CY", "Cyprus"), + ("CZ", "Czech Republic"), + ("CI", "Côte d'Ivoire"), + ("DK", "Denmark"), + ("DJ", "Djibouti"), + ("DM", "Dominica"), + ("DO", "Dominican Republic"), + ("TL", "East Timor"), + ("EC", "Ecuador"), + ("EG", "Egypt"), + ("SV", "El Salvador"), + ("GQ", "Equatorial Guinea"), + ("ER", "Eritrea"), + ("EE", "Estonia"), + ("SZ", "Eswatini (Swaziland)"), + ("ET", "Ethiopia"), + ("FK", "Falkland Islands"), + ("FO", "Faroe Islands"), + ("FJ", "Fiji"), + ("FI", "Finland"), + ("FR", "France"), + ("GF", "French Guiana"), + ("PF", "French Polynesia"), + ("TF", "French S. Terr."), + ("GA", "Gabon"), + ("GM", "Gambia"), + ("GE", "Georgia"), + ("DE", "Germany"), + ("GH", "Ghana"), + ("GI", "Gibraltar"), + ("GR", "Greece"), + ("GL", "Greenland"), + ("GD", "Grenada"), + ("GP", "Guadeloupe"), + ("GU", "Guam"), + ("GT", "Guatemala"), + ("GG", "Guernsey"), + ("GN", "Guinea"), + ("GW", "Guinea-Bissau"), + ("GY", "Guyana"), + ("HT", "Haiti"), + ("HM", "Heard Island & McDonald Islands"), + ("HN", "Honduras"), + ("HK", "Hong Kong"), + ("HU", "Hungary"), + ("IS", "Iceland"), + ("IN", "India"), + ("ID", "Indonesia"), + ("IR", "Iran"), + ("IQ", "Iraq"), + ("IE", "Ireland"), + ("IM", "Isle of Man"), + ("IL", "Israel"), + ("IT", "Italy"), + ("JM", "Jamaica"), + ("JP", "Japan"), + ("JE", "Jersey"), + ("JO", "Jordan"), + ("KZ", "Kazakhstan"), + ("KE", "Kenya"), + ("KI", "Kiribati"), + ("KP", "Korea (North)"), + ("KR", "Korea (South)"), + ("KW", "Kuwait"), + ("KG", "Kyrgyzstan"), + ("LA", "Laos"), + ("LV", "Latvia"), + ("LB", "Lebanon"), + ("LS", "Lesotho"), + ("LR", "Liberia"), + ("LY", "Libya"), + ("LI", "Liechtenstein"), + ("LT", "Lithuania"), + ("LU", "Luxembourg"), + ("MO", "Macau"), + ("MG", "Madagascar"), + ("MW", "Malawi"), + ("MY", "Malaysia"), + ("MV", "Maldives"), + ("ML", "Mali"), + ("MT", "Malta"), + ("MH", "Marshall Islands"), + ("MQ", "Martinique"), + ("MR", "Mauritania"), + ("MU", "Mauritius"), + ("YT", "Mayotte"), + ("MX", "Mexico"), + ("FM", "Micronesia"), + ("MD", "Moldova"), + ("MC", "Monaco"), + ("MN", "Mongolia"), + ("ME", "Montenegro"), + ("MS", "Montserrat"), + ("MA", "Morocco"), + ("MZ", "Mozambique"), + ("MM", "Myanmar (Burma)"), + ("NA", "Namibia"), + ("NR", "Nauru"), + ("NP", "Nepal"), + ("NL", "Netherlands"), + ("NC", "New Caledonia"), + ("NZ", "New Zealand"), + ("NI", "Nicaragua"), + ("NE", "Niger"), + ("NG", "Nigeria"), + ("NU", "Niue"), + ("NF", "Norfolk Island"), + ("MK", "North Macedonia"), + ("MP", "Northern Mariana Islands"), + ("NO", "Norway"), + ("OM", "Oman"), + ("PK", "Pakistan"), + ("PW", "Palau"), + ("PS", "Palestine"), + ("PA", "Panama"), + ("PG", "Papua New Guinea"), + ("PY", "Paraguay"), + ("PE", "Peru"), + ("PH", "Philippines"), + ("PN", "Pitcairn"), + ("PL", "Poland"), + ("PT", "Portugal"), + ("PR", "Puerto Rico"), + ("QA", "Qatar"), + ("RO", "Romania"), + ("RU", "Russia"), + ("RW", "Rwanda"), + ("RE", "Réunion"), + ("AS", "Samoa (American)"), + ("WS", "Samoa (western)"), + ("SM", "San Marino"), + ("ST", "Sao Tome & Principe"), + ("SA", "Saudi Arabia"), + ("SN", "Senegal"), + ("RS", "Serbia"), + ("SC", "Seychelles"), + ("SL", "Sierra Leone"), + ("SG", "Singapore"), + ("SK", "Slovakia"), + ("SI", "Slovenia"), + ("SB", "Solomon Islands"), + ("SO", "Somalia"), + ("ZA", "South Africa"), + ("GS", "South Georgia & the South Sandwich Islands"), + ("SS", "South Sudan"), + ("ES", "Spain"), + ("LK", "Sri Lanka"), + ("BL", "St Barthelemy"), + ("SH", "St Helena"), + ("KN", "St Kitts & Nevis"), + ("LC", "St Lucia"), + ("SX", "St Maarten (Dutch)"), + ("MF", "St Martin (French)"), + ("PM", "St Pierre & Miquelon"), + ("VC", "St Vincent"), + ("SD", "Sudan"), + ("SR", "Suriname"), + ("SJ", "Svalbard & Jan Mayen"), + ("SE", "Sweden"), + ("CH", "Switzerland"), + ("SY", "Syria"), + ("TW", "Taiwan"), + ("TJ", "Tajikistan"), + ("TZ", "Tanzania"), + ("TH", "Thailand"), + ("TG", "Togo"), + ("TK", "Tokelau"), + ("TO", "Tonga"), + ("TT", "Trinidad & Tobago"), + ("TN", "Tunisia"), + ("TR", "Turkey"), + ("TM", "Turkmenistan"), + ("TC", "Turks & Caicos Is"), + ("TV", "Tuvalu"), + ("UM", "US minor outlying islands"), + ("UG", "Uganda"), + ("UA", "Ukraine"), + ("AE", "United Arab Emirates"), + ("US", "United States"), + ("UY", "Uruguay"), + ("UZ", "Uzbekistan"), + ("VU", "Vanuatu"), + ("VA", "Vatican City"), + ("VE", "Venezuela"), + ("VN", "Vietnam"), + ("VG", "Virgin Islands (UK)"), + ("VI", "Virgin Islands (US)"), + ("WF", "Wallis & Futuna"), + ("EH", "Western Sahara"), + ("YE", "Yemen"), + ("ZM", "Zambia"), + ("ZW", "Zimbabwe"), + ("AX", "Åland Islands"), + ], + max_length=2, + ), + ), + migrations.AlterField( + model_name="meeting", + name="time_zone", + field=models.CharField( + choices=[ + ("", "---------"), + ("Africa/Abidjan", "Africa/Abidjan"), + ("Africa/Accra", "Africa/Accra"), + ("Africa/Addis_Ababa", "Africa/Addis_Ababa"), + ("Africa/Algiers", "Africa/Algiers"), + ("Africa/Asmara", "Africa/Asmara"), + ("Africa/Bamako", "Africa/Bamako"), + ("Africa/Bangui", "Africa/Bangui"), + ("Africa/Banjul", "Africa/Banjul"), + ("Africa/Bissau", "Africa/Bissau"), + ("Africa/Blantyre", "Africa/Blantyre"), + ("Africa/Brazzaville", "Africa/Brazzaville"), + ("Africa/Bujumbura", "Africa/Bujumbura"), + ("Africa/Cairo", "Africa/Cairo"), + ("Africa/Casablanca", "Africa/Casablanca"), + ("Africa/Ceuta", "Africa/Ceuta"), + ("Africa/Conakry", "Africa/Conakry"), + ("Africa/Dakar", "Africa/Dakar"), + ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"), + ("Africa/Djibouti", "Africa/Djibouti"), + ("Africa/Douala", "Africa/Douala"), + ("Africa/El_Aaiun", "Africa/El_Aaiun"), + ("Africa/Freetown", "Africa/Freetown"), + ("Africa/Gaborone", "Africa/Gaborone"), + ("Africa/Harare", "Africa/Harare"), + ("Africa/Johannesburg", "Africa/Johannesburg"), + ("Africa/Juba", "Africa/Juba"), + ("Africa/Kampala", "Africa/Kampala"), + ("Africa/Khartoum", "Africa/Khartoum"), + ("Africa/Kigali", "Africa/Kigali"), + ("Africa/Kinshasa", "Africa/Kinshasa"), + ("Africa/Lagos", "Africa/Lagos"), + ("Africa/Libreville", "Africa/Libreville"), + ("Africa/Lome", "Africa/Lome"), + ("Africa/Luanda", "Africa/Luanda"), + ("Africa/Lubumbashi", "Africa/Lubumbashi"), + ("Africa/Lusaka", "Africa/Lusaka"), + ("Africa/Malabo", "Africa/Malabo"), + ("Africa/Maputo", "Africa/Maputo"), + ("Africa/Maseru", "Africa/Maseru"), + ("Africa/Mbabane", "Africa/Mbabane"), + ("Africa/Mogadishu", "Africa/Mogadishu"), + ("Africa/Monrovia", "Africa/Monrovia"), + ("Africa/Nairobi", "Africa/Nairobi"), + ("Africa/Ndjamena", "Africa/Ndjamena"), + ("Africa/Niamey", "Africa/Niamey"), + ("Africa/Nouakchott", "Africa/Nouakchott"), + ("Africa/Ouagadougou", "Africa/Ouagadougou"), + ("Africa/Porto-Novo", "Africa/Porto-Novo"), + ("Africa/Sao_Tome", "Africa/Sao_Tome"), + ("Africa/Tripoli", "Africa/Tripoli"), + ("Africa/Tunis", "Africa/Tunis"), + ("Africa/Windhoek", "Africa/Windhoek"), + ("America/Adak", "America/Adak"), + ("America/Anchorage", "America/Anchorage"), + ("America/Anguilla", "America/Anguilla"), + ("America/Antigua", "America/Antigua"), + ("America/Araguaina", "America/Araguaina"), + ( + "America/Argentina/Buenos_Aires", + "America/Argentina/Buenos_Aires", + ), + ("America/Argentina/Catamarca", "America/Argentina/Catamarca"), + ("America/Argentina/Cordoba", "America/Argentina/Cordoba"), + ("America/Argentina/Jujuy", "America/Argentina/Jujuy"), + ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"), + ("America/Argentina/Mendoza", "America/Argentina/Mendoza"), + ( + "America/Argentina/Rio_Gallegos", + "America/Argentina/Rio_Gallegos", + ), + ("America/Argentina/Salta", "America/Argentina/Salta"), + ("America/Argentina/San_Juan", "America/Argentina/San_Juan"), + ("America/Argentina/San_Luis", "America/Argentina/San_Luis"), + ("America/Argentina/Tucuman", "America/Argentina/Tucuman"), + ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"), + ("America/Aruba", "America/Aruba"), + ("America/Asuncion", "America/Asuncion"), + ("America/Atikokan", "America/Atikokan"), + ("America/Bahia", "America/Bahia"), + ("America/Bahia_Banderas", "America/Bahia_Banderas"), + ("America/Barbados", "America/Barbados"), + ("America/Belem", "America/Belem"), + ("America/Belize", "America/Belize"), + ("America/Blanc-Sablon", "America/Blanc-Sablon"), + ("America/Boa_Vista", "America/Boa_Vista"), + ("America/Bogota", "America/Bogota"), + ("America/Boise", "America/Boise"), + ("America/Cambridge_Bay", "America/Cambridge_Bay"), + ("America/Campo_Grande", "America/Campo_Grande"), + ("America/Cancun", "America/Cancun"), + ("America/Caracas", "America/Caracas"), + ("America/Cayenne", "America/Cayenne"), + ("America/Cayman", "America/Cayman"), + ("America/Chicago", "America/Chicago"), + ("America/Chihuahua", "America/Chihuahua"), + ("America/Ciudad_Juarez", "America/Ciudad_Juarez"), + ("America/Costa_Rica", "America/Costa_Rica"), + ("America/Coyhaique", "America/Coyhaique"), + ("America/Creston", "America/Creston"), + ("America/Cuiaba", "America/Cuiaba"), + ("America/Curacao", "America/Curacao"), + ("America/Danmarkshavn", "America/Danmarkshavn"), + ("America/Dawson", "America/Dawson"), + ("America/Dawson_Creek", "America/Dawson_Creek"), + ("America/Denver", "America/Denver"), + ("America/Detroit", "America/Detroit"), + ("America/Dominica", "America/Dominica"), + ("America/Edmonton", "America/Edmonton"), + ("America/Eirunepe", "America/Eirunepe"), + ("America/El_Salvador", "America/El_Salvador"), + ("America/Fort_Nelson", "America/Fort_Nelson"), + ("America/Fortaleza", "America/Fortaleza"), + ("America/Glace_Bay", "America/Glace_Bay"), + ("America/Goose_Bay", "America/Goose_Bay"), + ("America/Grand_Turk", "America/Grand_Turk"), + ("America/Grenada", "America/Grenada"), + ("America/Guadeloupe", "America/Guadeloupe"), + ("America/Guatemala", "America/Guatemala"), + ("America/Guayaquil", "America/Guayaquil"), + ("America/Guyana", "America/Guyana"), + ("America/Halifax", "America/Halifax"), + ("America/Havana", "America/Havana"), + ("America/Hermosillo", "America/Hermosillo"), + ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"), + ("America/Indiana/Knox", "America/Indiana/Knox"), + ("America/Indiana/Marengo", "America/Indiana/Marengo"), + ("America/Indiana/Petersburg", "America/Indiana/Petersburg"), + ("America/Indiana/Tell_City", "America/Indiana/Tell_City"), + ("America/Indiana/Vevay", "America/Indiana/Vevay"), + ("America/Indiana/Vincennes", "America/Indiana/Vincennes"), + ("America/Indiana/Winamac", "America/Indiana/Winamac"), + ("America/Inuvik", "America/Inuvik"), + ("America/Iqaluit", "America/Iqaluit"), + ("America/Jamaica", "America/Jamaica"), + ("America/Juneau", "America/Juneau"), + ("America/Kentucky/Louisville", "America/Kentucky/Louisville"), + ("America/Kentucky/Monticello", "America/Kentucky/Monticello"), + ("America/La_Paz", "America/La_Paz"), + ("America/Lima", "America/Lima"), + ("America/Los_Angeles", "America/Los_Angeles"), + ("America/Maceio", "America/Maceio"), + ("America/Managua", "America/Managua"), + ("America/Manaus", "America/Manaus"), + ("America/Martinique", "America/Martinique"), + ("America/Matamoros", "America/Matamoros"), + ("America/Mazatlan", "America/Mazatlan"), + ("America/Menominee", "America/Menominee"), + ("America/Merida", "America/Merida"), + ("America/Metlakatla", "America/Metlakatla"), + ("America/Mexico_City", "America/Mexico_City"), + ("America/Miquelon", "America/Miquelon"), + ("America/Moncton", "America/Moncton"), + ("America/Monterrey", "America/Monterrey"), + ("America/Montevideo", "America/Montevideo"), + ("America/Montserrat", "America/Montserrat"), + ("America/Nassau", "America/Nassau"), + ("America/New_York", "America/New_York"), + ("America/Nome", "America/Nome"), + ("America/Noronha", "America/Noronha"), + ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"), + ("America/North_Dakota/Center", "America/North_Dakota/Center"), + ( + "America/North_Dakota/New_Salem", + "America/North_Dakota/New_Salem", + ), + ("America/Nuuk", "America/Nuuk"), + ("America/Ojinaga", "America/Ojinaga"), + ("America/Panama", "America/Panama"), + ("America/Paramaribo", "America/Paramaribo"), + ("America/Phoenix", "America/Phoenix"), + ("America/Port-au-Prince", "America/Port-au-Prince"), + ("America/Port_of_Spain", "America/Port_of_Spain"), + ("America/Porto_Velho", "America/Porto_Velho"), + ("America/Puerto_Rico", "America/Puerto_Rico"), + ("America/Punta_Arenas", "America/Punta_Arenas"), + ("America/Rankin_Inlet", "America/Rankin_Inlet"), + ("America/Recife", "America/Recife"), + ("America/Regina", "America/Regina"), + ("America/Resolute", "America/Resolute"), + ("America/Rio_Branco", "America/Rio_Branco"), + ("America/Santarem", "America/Santarem"), + ("America/Santiago", "America/Santiago"), + ("America/Santo_Domingo", "America/Santo_Domingo"), + ("America/Sao_Paulo", "America/Sao_Paulo"), + ("America/Scoresbysund", "America/Scoresbysund"), + ("America/Sitka", "America/Sitka"), + ("America/St_Johns", "America/St_Johns"), + ("America/St_Kitts", "America/St_Kitts"), + ("America/St_Lucia", "America/St_Lucia"), + ("America/St_Thomas", "America/St_Thomas"), + ("America/St_Vincent", "America/St_Vincent"), + ("America/Swift_Current", "America/Swift_Current"), + ("America/Tegucigalpa", "America/Tegucigalpa"), + ("America/Thule", "America/Thule"), + ("America/Tijuana", "America/Tijuana"), + ("America/Toronto", "America/Toronto"), + ("America/Tortola", "America/Tortola"), + ("America/Vancouver", "America/Vancouver"), + ("America/Whitehorse", "America/Whitehorse"), + ("America/Winnipeg", "America/Winnipeg"), + ("America/Yakutat", "America/Yakutat"), + ("Antarctica/Casey", "Antarctica/Casey"), + ("Antarctica/Davis", "Antarctica/Davis"), + ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"), + ("Antarctica/Macquarie", "Antarctica/Macquarie"), + ("Antarctica/Mawson", "Antarctica/Mawson"), + ("Antarctica/McMurdo", "Antarctica/McMurdo"), + ("Antarctica/Palmer", "Antarctica/Palmer"), + ("Antarctica/Rothera", "Antarctica/Rothera"), + ("Antarctica/Syowa", "Antarctica/Syowa"), + ("Antarctica/Troll", "Antarctica/Troll"), + ("Antarctica/Vostok", "Antarctica/Vostok"), + ("Asia/Aden", "Asia/Aden"), + ("Asia/Almaty", "Asia/Almaty"), + ("Asia/Amman", "Asia/Amman"), + ("Asia/Anadyr", "Asia/Anadyr"), + ("Asia/Aqtau", "Asia/Aqtau"), + ("Asia/Aqtobe", "Asia/Aqtobe"), + ("Asia/Ashgabat", "Asia/Ashgabat"), + ("Asia/Atyrau", "Asia/Atyrau"), + ("Asia/Baghdad", "Asia/Baghdad"), + ("Asia/Bahrain", "Asia/Bahrain"), + ("Asia/Baku", "Asia/Baku"), + ("Asia/Bangkok", "Asia/Bangkok"), + ("Asia/Barnaul", "Asia/Barnaul"), + ("Asia/Beirut", "Asia/Beirut"), + ("Asia/Bishkek", "Asia/Bishkek"), + ("Asia/Brunei", "Asia/Brunei"), + ("Asia/Chita", "Asia/Chita"), + ("Asia/Colombo", "Asia/Colombo"), + ("Asia/Damascus", "Asia/Damascus"), + ("Asia/Dhaka", "Asia/Dhaka"), + ("Asia/Dili", "Asia/Dili"), + ("Asia/Dubai", "Asia/Dubai"), + ("Asia/Dushanbe", "Asia/Dushanbe"), + ("Asia/Famagusta", "Asia/Famagusta"), + ("Asia/Gaza", "Asia/Gaza"), + ("Asia/Hebron", "Asia/Hebron"), + ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"), + ("Asia/Hong_Kong", "Asia/Hong_Kong"), + ("Asia/Hovd", "Asia/Hovd"), + ("Asia/Irkutsk", "Asia/Irkutsk"), + ("Asia/Jakarta", "Asia/Jakarta"), + ("Asia/Jayapura", "Asia/Jayapura"), + ("Asia/Jerusalem", "Asia/Jerusalem"), + ("Asia/Kabul", "Asia/Kabul"), + ("Asia/Kamchatka", "Asia/Kamchatka"), + ("Asia/Karachi", "Asia/Karachi"), + ("Asia/Kathmandu", "Asia/Kathmandu"), + ("Asia/Khandyga", "Asia/Khandyga"), + ("Asia/Kolkata", "Asia/Kolkata"), + ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"), + ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"), + ("Asia/Kuching", "Asia/Kuching"), + ("Asia/Kuwait", "Asia/Kuwait"), + ("Asia/Macau", "Asia/Macau"), + ("Asia/Magadan", "Asia/Magadan"), + ("Asia/Makassar", "Asia/Makassar"), + ("Asia/Manila", "Asia/Manila"), + ("Asia/Muscat", "Asia/Muscat"), + ("Asia/Nicosia", "Asia/Nicosia"), + ("Asia/Novokuznetsk", "Asia/Novokuznetsk"), + ("Asia/Novosibirsk", "Asia/Novosibirsk"), + ("Asia/Omsk", "Asia/Omsk"), + ("Asia/Oral", "Asia/Oral"), + ("Asia/Phnom_Penh", "Asia/Phnom_Penh"), + ("Asia/Pontianak", "Asia/Pontianak"), + ("Asia/Pyongyang", "Asia/Pyongyang"), + ("Asia/Qatar", "Asia/Qatar"), + ("Asia/Qostanay", "Asia/Qostanay"), + ("Asia/Qyzylorda", "Asia/Qyzylorda"), + ("Asia/Riyadh", "Asia/Riyadh"), + ("Asia/Sakhalin", "Asia/Sakhalin"), + ("Asia/Samarkand", "Asia/Samarkand"), + ("Asia/Seoul", "Asia/Seoul"), + ("Asia/Shanghai", "Asia/Shanghai"), + ("Asia/Singapore", "Asia/Singapore"), + ("Asia/Srednekolymsk", "Asia/Srednekolymsk"), + ("Asia/Taipei", "Asia/Taipei"), + ("Asia/Tashkent", "Asia/Tashkent"), + ("Asia/Tbilisi", "Asia/Tbilisi"), + ("Asia/Tehran", "Asia/Tehran"), + ("Asia/Thimphu", "Asia/Thimphu"), + ("Asia/Tokyo", "Asia/Tokyo"), + ("Asia/Tomsk", "Asia/Tomsk"), + ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"), + ("Asia/Urumqi", "Asia/Urumqi"), + ("Asia/Ust-Nera", "Asia/Ust-Nera"), + ("Asia/Vientiane", "Asia/Vientiane"), + ("Asia/Vladivostok", "Asia/Vladivostok"), + ("Asia/Yakutsk", "Asia/Yakutsk"), + ("Asia/Yangon", "Asia/Yangon"), + ("Asia/Yekaterinburg", "Asia/Yekaterinburg"), + ("Asia/Yerevan", "Asia/Yerevan"), + ("Atlantic/Azores", "Atlantic/Azores"), + ("Atlantic/Bermuda", "Atlantic/Bermuda"), + ("Atlantic/Canary", "Atlantic/Canary"), + ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"), + ("Atlantic/Faroe", "Atlantic/Faroe"), + ("Atlantic/Madeira", "Atlantic/Madeira"), + ("Atlantic/Reykjavik", "Atlantic/Reykjavik"), + ("Atlantic/South_Georgia", "Atlantic/South_Georgia"), + ("Atlantic/St_Helena", "Atlantic/St_Helena"), + ("Atlantic/Stanley", "Atlantic/Stanley"), + ("Australia/Adelaide", "Australia/Adelaide"), + ("Australia/Brisbane", "Australia/Brisbane"), + ("Australia/Broken_Hill", "Australia/Broken_Hill"), + ("Australia/Darwin", "Australia/Darwin"), + ("Australia/Eucla", "Australia/Eucla"), + ("Australia/Hobart", "Australia/Hobart"), + ("Australia/Lindeman", "Australia/Lindeman"), + ("Australia/Lord_Howe", "Australia/Lord_Howe"), + ("Australia/Melbourne", "Australia/Melbourne"), + ("Australia/Perth", "Australia/Perth"), + ("Australia/Sydney", "Australia/Sydney"), + ("Europe/Amsterdam", "Europe/Amsterdam"), + ("Europe/Andorra", "Europe/Andorra"), + ("Europe/Astrakhan", "Europe/Astrakhan"), + ("Europe/Athens", "Europe/Athens"), + ("Europe/Belgrade", "Europe/Belgrade"), + ("Europe/Berlin", "Europe/Berlin"), + ("Europe/Brussels", "Europe/Brussels"), + ("Europe/Bucharest", "Europe/Bucharest"), + ("Europe/Budapest", "Europe/Budapest"), + ("Europe/Chisinau", "Europe/Chisinau"), + ("Europe/Copenhagen", "Europe/Copenhagen"), + ("Europe/Dublin", "Europe/Dublin"), + ("Europe/Gibraltar", "Europe/Gibraltar"), + ("Europe/Helsinki", "Europe/Helsinki"), + ("Europe/Istanbul", "Europe/Istanbul"), + ("Europe/Kaliningrad", "Europe/Kaliningrad"), + ("Europe/Kirov", "Europe/Kirov"), + ("Europe/Kyiv", "Europe/Kyiv"), + ("Europe/Lisbon", "Europe/Lisbon"), + ("Europe/London", "Europe/London"), + ("Europe/Luxembourg", "Europe/Luxembourg"), + ("Europe/Madrid", "Europe/Madrid"), + ("Europe/Malta", "Europe/Malta"), + ("Europe/Minsk", "Europe/Minsk"), + ("Europe/Monaco", "Europe/Monaco"), + ("Europe/Moscow", "Europe/Moscow"), + ("Europe/Oslo", "Europe/Oslo"), + ("Europe/Paris", "Europe/Paris"), + ("Europe/Prague", "Europe/Prague"), + ("Europe/Riga", "Europe/Riga"), + ("Europe/Rome", "Europe/Rome"), + ("Europe/Samara", "Europe/Samara"), + ("Europe/Saratov", "Europe/Saratov"), + ("Europe/Simferopol", "Europe/Simferopol"), + ("Europe/Sofia", "Europe/Sofia"), + ("Europe/Stockholm", "Europe/Stockholm"), + ("Europe/Tallinn", "Europe/Tallinn"), + ("Europe/Tirane", "Europe/Tirane"), + ("Europe/Ulyanovsk", "Europe/Ulyanovsk"), + ("Europe/Vaduz", "Europe/Vaduz"), + ("Europe/Vienna", "Europe/Vienna"), + ("Europe/Vilnius", "Europe/Vilnius"), + ("Europe/Volgograd", "Europe/Volgograd"), + ("Europe/Warsaw", "Europe/Warsaw"), + ("Europe/Zurich", "Europe/Zurich"), + ("Indian/Antananarivo", "Indian/Antananarivo"), + ("Indian/Chagos", "Indian/Chagos"), + ("Indian/Christmas", "Indian/Christmas"), + ("Indian/Cocos", "Indian/Cocos"), + ("Indian/Comoro", "Indian/Comoro"), + ("Indian/Kerguelen", "Indian/Kerguelen"), + ("Indian/Mahe", "Indian/Mahe"), + ("Indian/Maldives", "Indian/Maldives"), + ("Indian/Mauritius", "Indian/Mauritius"), + ("Indian/Mayotte", "Indian/Mayotte"), + ("Indian/Reunion", "Indian/Reunion"), + ("Pacific/Apia", "Pacific/Apia"), + ("Pacific/Auckland", "Pacific/Auckland"), + ("Pacific/Bougainville", "Pacific/Bougainville"), + ("Pacific/Chatham", "Pacific/Chatham"), + ("Pacific/Chuuk", "Pacific/Chuuk"), + ("Pacific/Easter", "Pacific/Easter"), + ("Pacific/Efate", "Pacific/Efate"), + ("Pacific/Fakaofo", "Pacific/Fakaofo"), + ("Pacific/Fiji", "Pacific/Fiji"), + ("Pacific/Funafuti", "Pacific/Funafuti"), + ("Pacific/Galapagos", "Pacific/Galapagos"), + ("Pacific/Gambier", "Pacific/Gambier"), + ("Pacific/Guadalcanal", "Pacific/Guadalcanal"), + ("Pacific/Guam", "Pacific/Guam"), + ("Pacific/Honolulu", "Pacific/Honolulu"), + ("Pacific/Kanton", "Pacific/Kanton"), + ("Pacific/Kiritimati", "Pacific/Kiritimati"), + ("Pacific/Kosrae", "Pacific/Kosrae"), + ("Pacific/Kwajalein", "Pacific/Kwajalein"), + ("Pacific/Majuro", "Pacific/Majuro"), + ("Pacific/Marquesas", "Pacific/Marquesas"), + ("Pacific/Midway", "Pacific/Midway"), + ("Pacific/Nauru", "Pacific/Nauru"), + ("Pacific/Niue", "Pacific/Niue"), + ("Pacific/Norfolk", "Pacific/Norfolk"), + ("Pacific/Noumea", "Pacific/Noumea"), + ("Pacific/Pago_Pago", "Pacific/Pago_Pago"), + ("Pacific/Palau", "Pacific/Palau"), + ("Pacific/Pitcairn", "Pacific/Pitcairn"), + ("Pacific/Pohnpei", "Pacific/Pohnpei"), + ("Pacific/Port_Moresby", "Pacific/Port_Moresby"), + ("Pacific/Rarotonga", "Pacific/Rarotonga"), + ("Pacific/Saipan", "Pacific/Saipan"), + ("Pacific/Tahiti", "Pacific/Tahiti"), + ("Pacific/Tarawa", "Pacific/Tarawa"), + ("Pacific/Tongatapu", "Pacific/Tongatapu"), + ("Pacific/Wake", "Pacific/Wake"), + ("Pacific/Wallis", "Pacific/Wallis"), + ("UTC", "UTC"), + ], + default="UTC", + max_length=255, + ), + ), + ] diff --git a/requirements.txt b/requirements.txt index 7aadc7bf7fc..b3226d24b1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -66,8 +66,8 @@ python-json-logger>=3.1.0 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=1.6 # from TastyPie -pytz==2022.2.1 # Pinned as changes need to be vetted for their effect on Meeting fields -types-pytz==2022.2.1 # match pytz version +pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields +types-pytz==2025.2.0.20250516 # match pytz versionrequests>=2.31.0 requests>=2.31.0 types-requests>=2.27.1 requests-mock>=1.9.3 From 891049aa28c12b1afab4e16561bd31316e3b508e Mon Sep 17 00:00:00 2001 From: rjsparks <10996692+rjsparks@users.noreply.github.com> Date: Thu, 21 Aug 2025 13:39:47 +0000 Subject: [PATCH 007/313] ci: update base image target version to 20250821T1326 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 756b7021b7a..9019dbb626a 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250819T1508 +FROM ghcr.io/ietf-tools/datatracker-app-base:20250821T1326 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 2e0f1519dab..c5c7fb37b93 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250819T1508 +20250821T1326 From 86f2cfa29b8958633bf9dd23b624cfb4246f5693 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Aug 2025 08:59:40 -0500 Subject: [PATCH 008/313] chore(deps): bump types-pytz from 2022.2.1 to 2025.2.0.20250809 (#9339) Bumps [types-pytz](https://github.com/typeshed-internal/stub_uploader) from 2022.2.1 to 2025.2.0.20250809. - [Commits](https://github.com/typeshed-internal/stub_uploader/commits) --- updated-dependencies: - dependency-name: types-pytz dependency-version: 2025.2.0.20250809 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b3226d24b1b..60d3d8152e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -67,7 +67,7 @@ python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=1.6 # from TastyPie pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields -types-pytz==2025.2.0.20250516 # match pytz versionrequests>=2.31.0 +types-pytz==2025.2.0.20250809 # match pytz versionrequests>=2.31.0 requests>=2.31.0 types-requests>=2.27.1 requests-mock>=1.9.3 From f84f2e5b4ade1bff90feb845707f70f5a317253c Mon Sep 17 00:00:00 2001 From: rjsparks <10996692+rjsparks@users.noreply.github.com> Date: Thu, 21 Aug 2025 14:13:02 +0000 Subject: [PATCH 009/313] ci: update base image target version to 20250821T1359 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 9019dbb626a..d619ee99eeb 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250821T1326 +FROM ghcr.io/ietf-tools/datatracker-app-base:20250821T1359 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index c5c7fb37b93..b6fc12e1283 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250821T1326 +20250821T1359 From 64884e78ad3169a0d488ae3d046c1d65acf376b4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 21 Aug 2025 15:03:05 -0300 Subject: [PATCH 010/313] fix: actually call fromisoformat() (#9399) --- ietf/api/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index e2363479759..230f8339bdd 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -62,14 +62,14 @@ def generate_cache_key(self, *args, **kwargs): # Use a list plus a ``.join()`` because it's faster than concatenation. return "%s:%s:%s:%s" % (self._meta.api_name, self._meta.resource_name, ':'.join(args), smooshed) - def _z_aware_fromisoformat(self, value): - """datetime.datetie.fromisoformat replacement that works with python < 3.11""" + def _z_aware_fromisoformat(self, value: str) -> datetime.datetime: + """datetime.datetime.fromisoformat replacement that works with python < 3.11""" if HAVE_BROKEN_FROMISOFORMAT: if value.upper().endswith("Z"): value = value[:-1] + "+00:00" # Z -> UTC elif re.match(r"[+-][0-9][0-9]$", value[-3:]): value = value + ":00" # -04 -> -04:00 - return value + return datetime.datetime.fromisoformat(value) def filter_value_to_python( self, value, field_name, filters, filter_expr, filter_type From 3b6b4770d67a8c88aee642db7129c35be930da12 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Fri, 22 Aug 2025 03:58:37 -0400 Subject: [PATCH 011/313] ci: Fix build.yml for staging db recreate --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9c24380764c..79ef750b5d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -448,11 +448,11 @@ jobs: - name: Refresh Staging DB uses: the-actions-org/workflow-dispatch@v4 with: - workflow: update-staging-db.yml + workflow: deploy-db.yml repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "sourceDb":"datatracker" }' + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "waitClusterReady":true }' wait-for-completion: true wait-for-completion-timeout: 10m wait-for-completion-interval: 20s From 82a241084954146152196d612a13074bdc1b5de9 Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Fri, 22 Aug 2025 17:46:45 +0200 Subject: [PATCH 012/313] feat: updated text for BoF request (per IESG demand) (#9406) --- ietf/templates/doc/bofreq/bofreq_template.md | 10 +++---- ietf/templates/doc/bofreq/new_bofreq.html | 30 ++++++++++++++++++-- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/ietf/templates/doc/bofreq/bofreq_template.md b/ietf/templates/doc/bofreq/bofreq_template.md index 78949ba7bd5..49c5e511a52 100644 --- a/ietf/templates/doc/bofreq/bofreq_template.md +++ b/ietf/templates/doc/bofreq/bofreq_template.md @@ -1,15 +1,15 @@ -# Name: Exact MPLS Edges (EXAMPLE) (There's an acronym for anything if you really want one ;-) +# Name: EXAct MPLs Edges (EXAMPLE) (There's an acronym for anything if you really want one ;-) ## Description Replace this with a few paragraphs describing the BOF request. Fill in the details below. Keep items in the order they appear here. ## Required Details -- Status: (not) WG Forming -- Responsible AD: name +- Status: "not WG Forming" or "WG forming" +- Responsible AD: name (or at least area(s) if you know) - BOF proponents: name , name (1-3 people - who are requesting and coordinating discussion for proposal) - Number of people expected to attend: 100 -- Length of session (1 or 2 hours): 2 hours +- Length of session (1 or usually 2 hours): 2 hours - Conflicts (whole Areas and/or WGs) - Chair Conflicts: TBD - Technology Overlap: TBD @@ -27,7 +27,7 @@ To allow evaluation of your proposal, please include the following items: - Items, Internet-Drafts, speakers, timing - Or a URL -## Links to the mailing list, draft charter if any, relevant Internet-Drafts, etc. +## Links to the mailing list, draft charter if any (for WG-forming BoF), relevant Internet-Drafts, etc. - Mailing List: https://www.ietf.org/mailman/listinfo/example - Draft charter: {{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name='charter-ietf-EXAMPLE' %} - Relevant Internet-Drafts: diff --git a/ietf/templates/doc/bofreq/new_bofreq.html b/ietf/templates/doc/bofreq/new_bofreq.html index 9506d9c8e6e..c6aa0054f96 100644 --- a/ietf/templates/doc/bofreq/new_bofreq.html +++ b/ietf/templates/doc/bofreq/new_bofreq.html @@ -6,15 +6,39 @@ {% origin %}

Start a new BOF Request

- The IAB will also attempt to provide BoF Shepherds as described in their document on the subject only on request from the IESG. If you feel that your BoF would benefit from an IAB BoF Shepherd, please discuss this with your Area Director. + BoF proponents are strongly encouraged to review the following sources before submitting requests: +

+ + {# The following block needs to be commented out after the BoF deadline and re-opened before next BoF request opening #} +
+

+ Announcement for IETF 124: The IESG and the IAB have organized Ask Me Anything (AMA) virtual sessions + for the community to help proponents who are interested in putting up BoF proposals for IETF 124 + (see also the IETF-announce email): +

+
    +
  • 28th of August 13:00-14:00 UTC +
    • +
  • 28th of August 19:00-20:00 UTC +
    • +
+
+ {# End of the temporary block #} +

+ The IAB will also attempt to provide BoF Shepherds as described in their document on the subject only on request from the IESG. + If you feel that your BoF would benefit from an IAB BoF Shepherd, please discuss this with your Area Director.

- Choose a short descriptive title for your request. Take time to choose a good initial title - it will be used to make the filename for your request's content. The title can be changed later, but the filename will not change. + Choose a short descriptive title for your request. Take time to choose a good initial title - it will be used to make the filename for your request's content. + The title can be changed later, but the filename will not change.

For example, a request with a title of "A new important bit" will be saved as bofreq-{{ user.person.last_name|xslugify|slice:"64" }}-a-new-important-bit-00.md.

-

All the items in the template MUST be filed in.

+

All the items in the template MUST be filed in.

From b1cfa7082f60343210b8116668f182e3c67207bf Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Fri, 22 Aug 2025 22:08:59 -0400 Subject: [PATCH 013/313] ci: Increase wait-for-completion timeout to 30 minutes for staging refresh db step Increased the wait-for-completion timeout from 10 minutes to 30 minutes in the build workflow. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 79ef750b5d5..8567446cae5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -454,7 +454,7 @@ jobs: token: ${{ secrets.GH_INFRA_K8S_TOKEN }} inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "waitClusterReady":true }' wait-for-completion: true - wait-for-completion-timeout: 10m + wait-for-completion-timeout: 30m wait-for-completion-interval: 20s display-workflow-run-url: false From b3f2756f6b5d6adf853eb7779412950291169c38 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 27 Aug 2025 13:06:48 -0500 Subject: [PATCH 014/313] fix: clearly show To and From groups in liaison statement email (#9432) --- ietf/group/templatetags/group_filters.py | 7 +++++++ ietf/templates/liaisons/liaison_mail.txt | 11 +++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ietf/group/templatetags/group_filters.py b/ietf/group/templatetags/group_filters.py index c9481b767b6..bf2ad719494 100644 --- a/ietf/group/templatetags/group_filters.py +++ b/ietf/group/templatetags/group_filters.py @@ -37,3 +37,10 @@ def role_person_link(role, **kwargs): plain_name = role.person.plain_name() email = role.email.address return {'name': name, 'plain_name': plain_name, 'email': email, 'title': title, 'class': cls} + +@register.filter +def name_with_conditional_acronym(group): + if group.type_id in ("sdo", "isoc", "individ", "nomcom", "ietf", "irtf", ): + return group.name + else: + return f"{group.name} ({group.acronym})" diff --git a/ietf/templates/liaisons/liaison_mail.txt b/ietf/templates/liaisons/liaison_mail.txt index 6d6a07d7ef4..18dfe610fd7 100644 --- a/ietf/templates/liaisons/liaison_mail.txt +++ b/ietf/templates/liaisons/liaison_mail.txt @@ -1,13 +1,20 @@ -{% load ietf_filters %}{% autoescape off %}Title: {{ liaison.title|clean_whitespace }} +{% load ietf_filters group_filters %}{% autoescape off %}Title: {{ liaison.title|clean_whitespace }} Submission Date: {{ liaison.submitted|date:"Y-m-d" }} URL of the IETF Web page: {{ liaison.get_absolute_url }} + +To: {% for g in liaison.to_groups.all %}{{g|name_with_conditional_acronym}}{% if not forloop.last %}, {% endif %}{% endfor %} +From: {% for g in liaison.from_groups.all %}{{g|name_with_conditional_acronym}}{% if not forloop.last %}, {% endif %}{% endfor %} +Purpose: {{ liaison.purpose.name }} {% if liaison.deadline %}Please reply by {{ liaison.deadline }}{% endif %} + +Email Addresses +--------------- From: {% if liaison.from_contact %}{{ liaison.from_contact }}{% endif %} To: {{ liaison.to_contacts }} Cc: {{ liaison.cc_contacts }} Response Contacts: {{ liaison.response_contacts }} Technical Contacts: {{ liaison.technical_contacts }} -Purpose: {{ liaison.purpose.name }} + {% for related in liaison.source_of_set.all %} Referenced liaison: {% if related.target.title %}{{ related.target.title }}{% else %}Liaison #{{ related.target.pk }}{% endif %} ({{ related.target.get_absolute_url }}) {% endfor %} From 6e62bb32771cb564f52201a376ad6e754155343c Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 29 Aug 2025 10:44:51 -0500 Subject: [PATCH 015/313] fix: remove iab executive director specialization from the liaison app (#9435) --- ietf/liaisons/forms.py | 3 +-- ietf/liaisons/tests.py | 12 ------------ ietf/liaisons/tests_forms.py | 12 ------------ ietf/liaisons/utils.py | 1 - ietf/liaisons/views.py | 17 +++++++---------- 5 files changed, 8 insertions(+), 37 deletions(-) diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index 7483981595e..ef5b29535e8 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -105,7 +105,6 @@ def internal_groups_for_person(person: Optional[Person]): "Secretariat", "IETF Chair", "IAB Chair", - "IAB Executive Director", "Liaison Manager", "Liaison Coordinator", "Authorized Individual", @@ -115,7 +114,7 @@ def internal_groups_for_person(person: Optional[Person]): # Interesting roles, as Group queries queries = [ Q(role__person=person, role__name="chair", acronym="ietf"), - Q(role__person=person, role__name__in=("chair", "execdir"), acronym="iab"), + Q(role__person=person, role__name="chair", acronym="iab"), Q(role__person=person, role__name="ad", type="area", state="active"), Q( role__person=person, diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index 1d6cfe0c14a..8bbaa4f053d 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -123,7 +123,6 @@ def test_get_cc(self): cc = get_cc(Group.objects.get(acronym='iab')) self.assertTrue(EMAIL_ALIASES['IAB'] in cc) self.assertTrue(EMAIL_ALIASES['IABCHAIR'] in cc) - self.assertTrue(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR'] in cc) # test an Area area = Group.objects.filter(type='area').first() cc = get_cc(area) @@ -166,7 +165,6 @@ def test_get_contacts_for_group(self): # test iab contacts = get_contacts_for_group(Group.objects.get(acronym='iab')) self.assertTrue(EMAIL_ALIASES['IABCHAIR'] in contacts) - self.assertTrue(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR'] in contacts) # test iesg contacts = get_contacts_for_group(Group.objects.get(acronym='iesg')) self.assertTrue(EMAIL_ALIASES['IESG'] in contacts) @@ -534,7 +532,6 @@ def test_outgoing_access(self): RoleFactory(name_id='liaison_coordinator', group__acronym='iab', person__user__username='liaison-coordinator') mars = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group RoleFactory(name_id='secr',group=mars,person__user__username='mars-secr') - RoleFactory(name_id='execdir',group=Group.objects.get(acronym='iab'),person__user__username='iab-execdir') url = urlreverse('ietf.liaisons.views.liaison_list') addurl = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'outgoing'}) @@ -592,15 +589,6 @@ def test_outgoing_access(self): r = self.client.get(addurl) self.assertEqual(r.status_code, 200) - # IAB Executive Director - self.assertTrue(self.client.login(username="iab-execdir", password="iab-execdir+password")) - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertEqual(len(q("a.btn:contains('New outgoing liaison')")), 1) - r = self.client.get(addurl) - self.assertEqual(r.status_code, 200) - # Liaison Manager has access self.assertTrue(self.client.login(username="ulm-liaiman", password="ulm-liaiman+password")) r = self.client.get(url) diff --git a/ietf/liaisons/tests_forms.py b/ietf/liaisons/tests_forms.py index c2afddea65c..101c0c8298c 100644 --- a/ietf/liaisons/tests_forms.py +++ b/ietf/liaisons/tests_forms.py @@ -94,11 +94,6 @@ def test_all_internal_groups(self): def test_internal_groups_for_person(self): # test relies on the data created in ietf.utils.test_data.make_immutable_test_data() # todo add liaison coordinator when modeled - RoleFactory( - name_id="execdir", - group=Group.objects.get(acronym="iab"), - person__user__username="iab-execdir", - ) RoleFactory( name_id="auth", group__type_id="sdo", @@ -121,7 +116,6 @@ def test_internal_groups_for_person(self): "secretary", "ietf-chair", "iab-chair", - "iab-execdir", "sdo-authperson", ): returned_queryset = internal_groups_for_person( @@ -151,11 +145,6 @@ def test_internal_groups_for_person(self): ) def test_external_groups_for_person(self): - RoleFactory( - name_id="execdir", - group=Group.objects.get(acronym="iab"), - person__user__username="iab-execdir", - ) RoleFactory(name_id="liaison_coordinator", group__acronym="iab", person__user__username="liaison-coordinator") the_sdo = GroupFactory(type_id="sdo", acronym="the-sdo") liaison_manager = RoleFactory(name_id="liaiman", group=the_sdo).person @@ -166,7 +155,6 @@ def test_external_groups_for_person(self): "secretary", "ietf-chair", "iab-chair", - "iab-execdir", "liaison-coordinator", "ad", "sopschairman", diff --git a/ietf/liaisons/utils.py b/ietf/liaisons/utils.py index ea06c5988e2..469bbc5c87e 100644 --- a/ietf/liaisons/utils.py +++ b/ietf/liaisons/utils.py @@ -8,7 +8,6 @@ OUTGOING_LIAISON_ROLES = [ "Area Director", "IAB Chair", - "IAB Executive Director", "IETF Chair", "Liaison Manager", "Liaison Coordinator", diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py index 1b7e8d63bb4..9710149c901 100644 --- a/ietf/liaisons/views.py +++ b/ietf/liaisons/views.py @@ -30,11 +30,12 @@ from ietf.utils.response import permission_denied EMAIL_ALIASES = { - 'IETFCHAIR':'The IETF Chair ', - 'IESG':'The IESG ', - 'IAB':'The IAB ', - 'IABCHAIR':'The IAB Chair ', - 'IABEXECUTIVEDIRECTOR':'The IAB Executive Director '} + "IETFCHAIR": "The IETF Chair ", + "IESG": "The IESG ", + "IAB": "The IAB ", + "IABCHAIR": "The IAB Chair ", +} + # ------------------------------------------------- # Helper Functions @@ -84,8 +85,6 @@ def _find_person_in_emails(liaison, person): return True elif addr in ('iab@iab.org', 'iab-chair@iab.org') and has_role(person.user, "IAB Chair"): return True - elif addr in ('execd@iab.org', ) and has_role(person.user, "IAB Executive Director"): - return True return False @@ -110,7 +109,6 @@ def get_cc(group): elif group.acronym in ('iab'): emails.append(EMAIL_ALIASES['IAB']) emails.append(EMAIL_ALIASES['IABCHAIR']) - emails.append(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR']) elif group.type_id == 'area': emails.append(EMAIL_ALIASES['IETFCHAIR']) ad_roles = group.role_set.filter(name='ad') @@ -151,7 +149,6 @@ def get_contacts_for_group(group): contacts.append(EMAIL_ALIASES['IETFCHAIR']) elif group.acronym == 'iab': contacts.append(EMAIL_ALIASES['IABCHAIR']) - contacts.append(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR']) elif group.acronym == 'iesg': contacts.append(EMAIL_ALIASES['IESG']) @@ -171,7 +168,7 @@ def needs_approval(group,person): user = person.user if group.acronym in ('ietf','iesg') and has_role(user, 'IETF Chair'): return False - if group.acronym == 'iab' and (has_role(user,'IAB Chair') or has_role(user,'IAB Executive Director')): + if group.acronym == 'iab' and has_role(user,'IAB Chair'): return False if group.type_id == 'area' and group.role_set.filter(name='ad',person=person): return False From 3ca4eec5abb6927837fbc849809b587f4bde6419 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 2 Sep 2025 14:41:52 -0300 Subject: [PATCH 016/313] feat: expose State.used in admin (#9449) --- ietf/doc/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index b492aa34238..745536f9a1e 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -22,8 +22,8 @@ class StateTypeAdmin(admin.ModelAdmin): admin.site.register(StateType, StateTypeAdmin) class StateAdmin(admin.ModelAdmin): - list_display = ["slug", "type", 'name', 'order', 'desc'] - list_filter = ["type", ] + list_display = ["slug", "type", 'name', 'order', 'desc', "used"] + list_filter = ["type", "used"] search_fields = ["slug", "type__label", "type__slug", "name", "desc"] filter_horizontal = ["next_states"] admin.site.register(State, StateAdmin) From 02dbe17fe7bfe707594531bb16dffd905c5c2a53 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 2 Sep 2025 16:48:38 -0300 Subject: [PATCH 017/313] feat: history for mailtrigger models (#9452) * feat: history for mailtrigger models * chore: update copyright years * fix: use py3.9-compatible call_command syntax It seems `option=[...]` does not work with positional arguments in py3.9's argparse. * chore: update resources --- ietf/mailtrigger/admin.py | 7 +- ...storicalrecipient_historicalmailtrigger.py | 122 ++++++++++++++++++ ietf/mailtrigger/models.py | 6 +- ietf/mailtrigger/resources.py | 42 +++++- 4 files changed, 172 insertions(+), 5 deletions(-) create mode 100644 ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py diff --git a/ietf/mailtrigger/admin.py b/ietf/mailtrigger/admin.py index a60fd5b0725..8c73f2ae02e 100644 --- a/ietf/mailtrigger/admin.py +++ b/ietf/mailtrigger/admin.py @@ -1,9 +1,10 @@ -# Copyright The IETF Trust 2015-2019, All Rights Reserved +# Copyright The IETF Trust 2015-2025, All Rights Reserved from django.contrib import admin +from simple_history.admin import SimpleHistoryAdmin from ietf.mailtrigger.models import MailTrigger, Recipient -class RecipientAdmin(admin.ModelAdmin): +class RecipientAdmin(SimpleHistoryAdmin): list_display = [ 'slug', 'desc', 'template', 'has_code', ] def has_code(self, obj): return hasattr(obj,'gather_%s'%obj.slug) @@ -11,7 +12,7 @@ def has_code(self, obj): admin.site.register(Recipient, RecipientAdmin) -class MailTriggerAdmin(admin.ModelAdmin): +class MailTriggerAdmin(SimpleHistoryAdmin): list_display = [ 'slug', 'desc', ] filter_horizontal = [ 'to', 'cc', ] admin.site.register(MailTrigger, MailTriggerAdmin) diff --git a/ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py b/ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py new file mode 100644 index 00000000000..d23b72d7373 --- /dev/null +++ b/ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py @@ -0,0 +1,122 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from io import StringIO + +from django.conf import settings +from django.core import management +from django.db import migrations, models +import django.db.models.deletion +import simple_history.models + +from ietf.utils.log import log + + +def forward(apps, schema_editor): + # Fill in history for existing data using the populate_history management command + captured_stdout = StringIO() + captured_stderr = StringIO() + try: + management.call_command( + "populate_history", + "mailtrigger.MailTrigger", + "mailtrigger.Recipient", + stdout=captured_stdout, + stderr=captured_stderr, + ) + except management.CommandError as err: + log( + "Failed to populate history for mailtrigger models.\n" + "\n" + f"stdout:\n{captured_stdout.getvalue() or ''}\n" + "\n" + f"stderr:\n{captured_stderr.getvalue() or ''}\n" + ) + raise RuntimeError("Failed to populate history for mailtrigger models") from err + log( + "Populated history for mailtrigger models.\n" + "\n" + f"stdout:\n{captured_stdout.getvalue() or ''}\n" + "\n" + f"stderr:\n{captured_stderr.getvalue() or ''}\n" + ) + + +def reverse(apps, schema_editor): + pass # nothing to do + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("mailtrigger", "0006_call_for_adoption_and_last_call_issued"), + ] + + operations = [ + migrations.CreateModel( + name="HistoricalRecipient", + fields=[ + ("slug", models.CharField(db_index=True, max_length=32)), + ("desc", models.TextField(blank=True)), + ("template", models.TextField(blank=True, null=True)), + ("history_id", models.AutoField(primary_key=True, serialize=False)), + ("history_date", models.DateTimeField(db_index=True)), + ("history_change_reason", models.CharField(max_length=100, null=True)), + ( + "history_type", + models.CharField( + choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")], + max_length=1, + ), + ), + ( + "history_user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "historical recipient", + "verbose_name_plural": "historical recipients", + "ordering": ("-history_date", "-history_id"), + "get_latest_by": ("history_date", "history_id"), + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + migrations.CreateModel( + name="HistoricalMailTrigger", + fields=[ + ("slug", models.CharField(db_index=True, max_length=64)), + ("desc", models.TextField(blank=True)), + ("history_id", models.AutoField(primary_key=True, serialize=False)), + ("history_date", models.DateTimeField(db_index=True)), + ("history_change_reason", models.CharField(max_length=100, null=True)), + ( + "history_type", + models.CharField( + choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")], + max_length=1, + ), + ), + ( + "history_user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "historical mail trigger", + "verbose_name_plural": "historical mail triggers", + "ordering": ("-history_date", "-history_id"), + "get_latest_by": ("history_date", "history_id"), + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + migrations.RunPython(forward, reverse), + ] diff --git a/ietf/mailtrigger/models.py b/ietf/mailtrigger/models.py index 66b7139fa5e..435729f893a 100644 --- a/ietf/mailtrigger/models.py +++ b/ietf/mailtrigger/models.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2015-2020, All Rights Reserved +# Copyright The IETF Trust 2015-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -7,6 +7,8 @@ from email.utils import parseaddr +from simple_history.models import HistoricalRecords + from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible from ietf.utils.mail import formataddr, get_email_addresses_from_text from ietf.group.models import Group, Role @@ -38,6 +40,7 @@ class MailTrigger(models.Model): desc = models.TextField(blank=True) to = models.ManyToManyField('mailtrigger.Recipient', blank=True, related_name='used_in_to') cc = models.ManyToManyField('mailtrigger.Recipient', blank=True, related_name='used_in_cc') + history = HistoricalRecords() class Meta: ordering = ["slug"] @@ -49,6 +52,7 @@ class Recipient(models.Model): slug = models.CharField(max_length=32, primary_key=True) desc = models.TextField(blank=True) template = models.TextField(null=True, blank=True) + history = HistoricalRecords() class Meta: ordering = ["slug"] diff --git a/ietf/mailtrigger/resources.py b/ietf/mailtrigger/resources.py index eb5466618ac..daca055bf43 100644 --- a/ietf/mailtrigger/resources.py +++ b/ietf/mailtrigger/resources.py @@ -7,7 +7,7 @@ from ietf import api -from ietf.mailtrigger.models import Recipient, MailTrigger +from ietf.mailtrigger.models import MailTrigger, Recipient class RecipientResource(ModelResource): @@ -37,3 +37,43 @@ class Meta: } api.mailtrigger.register(MailTriggerResource()) +from ietf.utils.resources import UserResource +class HistoricalMailTriggerResource(ModelResource): + history_user = ToOneField(UserResource, 'history_user', null=True) + class Meta: + queryset = MailTrigger.history.model.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'historicalmailtrigger' + ordering = ['history_id', ] + filtering = { + "slug": ALL, + "desc": ALL, + "history_id": ALL, + "history_date": ALL, + "history_change_reason": ALL, + "history_type": ALL, + "history_user": ALL_WITH_RELATIONS, + } +api.mailtrigger.register(HistoricalMailTriggerResource()) + +from ietf.utils.resources import UserResource +class HistoricalRecipientResource(ModelResource): + history_user = ToOneField(UserResource, 'history_user', null=True) + class Meta: + queryset = Recipient.history.model.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'historicalrecipient' + ordering = ['history_id', ] + filtering = { + "slug": ALL, + "desc": ALL, + "template": ALL, + "history_id": ALL, + "history_date": ALL, + "history_change_reason": ALL, + "history_type": ALL, + "history_user": ALL_WITH_RELATIONS, + } +api.mailtrigger.register(HistoricalRecipientResource()) From 2960164714f0c0380d3259408b028a9150c8c27e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 3 Sep 2025 19:16:26 -0300 Subject: [PATCH 018/313] feat: Python 3.12 (#8811) * refactor: smtpd -> aiosmtpd * test: set mock return value for EmailOnFailureCommandTests The test has been working, but in a broken way, for as long as it has existed. The smtpd-based test_smtpserver was masking an exception that did not interfere with the test's effectiveness. * test: increase SMTP.line_length_limit * chore: suppress known deprecation warnings * refactor: utcfromtimestamp->fromtimestamp * refactor: it's now spelled "datetime.UTC" * feat: python 3.12 * chore: suppress deprecation warning * fix: utcnow() -> now(datetime.UTC) * chore: suppress deprecation warning * chore: more deprecation warnings * ci: update base image target version to 20250417T1507 * chore: reorg / clean up deprecation ignore list Removed a few suppressions that were OBE based on running the tests and checking versions of the dependencies that were causing them. Reordered kwargs to make it more readable (to me anyway). * chore: disable coverage test for now See the comment in settings.py for details. tl;dr coverage is unusably slow under python 3.12 as we're using it * ci: update base image target version to 20250422T1458 * ci: update base image target version to 20250604T2012 * ci: build/use py312 images (#9168) * ci: tag py312 base app * ci: datatrackerbase-app:latest -> py312 * ci: update base image target version to 20250719T0833 * refactor: update to coverage 7.9.2 + cleanup (#9256) * refactor: drop unused code_coverage_collection var * refactor: @skip_coverage -> pragma: no cover * chore(deps): bump coverage to current ver * refactor: split up set_coverage_checking() * refactor: inline IetfLiveServerTestCase (there's only one subclass) * feat: disable_coverage context mgr * chore: remove unused import * refactor: set_coverage_checking -> disable_coverage * refactor: elim more set_coverage_checking * refactor: start using coverage 7.9.2 * feat: working coverage 7.9 implementation * Extract coverage tools to ietf.utils.coverage * Revert to starting checker in settings_test Does not exactly match previous coverage reports. Need to investigate. * refactor: CustomJsonReporter->CustomDictReporter * chore: remove "migration" coverage entry Has not been populated in quite some time * test: test CoverageManager class * chore: exclude CustomDictReporter from coverage Setting up to test this will be complex and we'll notice other test failures/coverage weirdness if this does not behave. * chore: exclude coverage.py from coverage Way too meta * chore: update deps for py3.12 (#9270) * chore(deps): argon2-cffi (supports py3.14) * chore(deps): setuptools to latest (py3.9+) * chore(deps): bump beautifulsoup4 (py3.7+) * chore(deps): bump bibtexparser (py3) * chore(deps): bump bleach (py3.13) * chore(deps): bump bleach (py3.13) * chore(deps): lift pin on boto3 + adjust settings * chore(deps): bump celery (py3.13) * chore(deps): bump django-admin-rangefilter (py3.12) * chore(deps): bump django-analytical (py3.13) * chore(deps): bump django-bootstrap5 (py3.13) * chore(deps): bump django-celery-beat (py3.12) Still holding back until their #894 is conclusively resolved. The 2.8.x release adds official py3.13 support. * chore(deps): bump django-celery-results (py3.13) * chore(deps): remove django-csp (not used) * chore(deps): bump django-cors-headers (py3.13) * chore(deps): bump django-debug-toolbar (py3.13) * refactor: drop stale django-referrer-policy pkg Supported via django's SecurityMiddleware since longtime * chore(deps): bump django-simple-history (py3.13) * chore(deps): bump django-storages (py3.12) * chore(deps): bump django-tastypie+update patch * chore(deps): bump django_vite+update config * chore(deps): bump djangorestframework+remove cap * chore(deps): remove djlint * chore(deps): bump docutils (py3.14) * chore(deps): bump drf-standardized-errors (py3.13) * chore(deps): bump factory-boy (py3.13) * chore(deps): bump github3.py (py3.11??) * chore(deps): bump gunicorn (py3.12) * chore(deps): bump html2text (py3.13) * chore(deps): bump inflect * chore(deps): bump jsonfield (py3.10-3.13) * chore(deps): bump jsonschema (py3.13) * chore(deps): bump logging_tree (py3.12) * chore(deps): bump lxml (py3.13) * chore(deps): bump markdown (py3.13) * chore(deps): bump mock * chore(deps): bump oic (py3.11) * chore(deps): bump pillow (py3.13) * chore(deps): bump psycopg2 (py3.13) * chore(deps): bump pyang (py3.11) * chore(deps): bump pydyf (py3.12) * chore(deps): bump pyflakes (py3.9+) * chore(deps): bump pyopenssl (py3.13) * chore(deps): bump pyquery (py3.12) * chore(deps): bump python-dateutil (py3.12) * chore(deps): bump python-json-logger (py3.13) * chore(deps): bump python-mimeparse (py3.13) * chore(deps): bump pytz (py3.13) Brings a meeting migration to adjust tz/country choices. * chore(deps): bump requests (py3.13) * chore(deps): bump requests-mock (py3.12) * chore(deps): bump scout-apm (py3.12) * chore(deps): bump selenium (py3.13) * chore(deps): bump tblib (py3.13) * chore(deps): bump tqdm (py3.12) * chore(deps): bump unidecode (py3.11) * chore(deps): adjust requirements.txt to install correctly * chore(deps): bump urllib3, remove pin (py3.13) Situation requiring the pin to < 2.0 appears to have resolved. * chore(deps): bump weasyprint (py3.13) * chore(deps): bump xml2rfc (py3.13) * fix: lint * ci: py312 base for celery in sandbox * ci: update base image target version to 20250819T1645 * chore: finish dropping smtpd (#9384) * chore: smtpd debug server -> aiosmtpd * chore(dev): accept long SMTP lines * chore(dev): use correct aiosmtpd handler * chore: update copyright years * Revert "chore: update copyright years" This reverts commit 2814cb85dc43c9a27f9834c629474e58d1dfb0f7. --------- Co-authored-by: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com> --- .github/workflows/build-base-app.yml | 1 + .github/workflows/tests-az.yml | 2 +- .vscode/tasks.json | 5 +- README.md | 2 +- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- dev/celery/Dockerfile | 0 dev/deploy-to-container/cli.js | 6 +- dev/diff/cli.js | 6 +- dev/tests/debug.sh | 2 +- dev/tests/docker-compose.debug.yml | 2 +- docker/app.Dockerfile | 2 +- docker/base.Dockerfile | 2 +- docker/celery.Dockerfile | 2 +- docker/configs/settings_local.py | 2 + docker/configs/settings_local_vite.py | 6 +- docker/scripts/app-configure-blobstore.py | 6 +- docker/scripts/app-init.sh | 2 +- ietf/api/__init__.py | 2 +- ietf/api/tests.py | 6 +- ietf/bin/aliases-from-json.py | 2 +- ietf/doc/models.py | 2 +- ietf/doc/templatetags/ballot_icon.py | 2 +- ietf/doc/tests_draft.py | 4 +- ietf/doc/tests_utils.py | 2 +- ietf/doc/views_stats.py | 6 +- ietf/group/views.py | 2 +- ietf/idindex/index.py | 4 +- ietf/iesg/views.py | 2 +- ietf/ietfauth/views.py | 2 +- ietf/ipr/mail.py | 4 +- ietf/ipr/views.py | 14 +- ietf/liaisons/tests.py | 8 +- ...meeting_country_alter_meeting_time_zone.py | 1 + ietf/meeting/models.py | 6 +- ietf/meeting/tests_js.py | 2 +- ietf/meeting/tests_tasks.py | 2 +- ietf/meeting/tests_views.py | 29 ++-- ietf/meeting/views.py | 6 +- ietf/nomcom/tests.py | 2 +- ietf/nomcom/views.py | 4 +- ietf/settings.py | 48 +++--- ietf/settings_test.py | 5 +- ietf/submit/checkers.py | 57 ++++--- ietf/sync/iana.py | 8 +- ietf/sync/tasks.py | 2 +- ietf/sync/tests.py | 6 +- .../utils/{test_smtpserver.py => aiosmtpd.py} | 21 ++- ietf/utils/coverage.py | 90 ++++++++++ ietf/utils/decorators.py | 12 -- ietf/utils/jstest.py | 41 ++++- ietf/utils/meetecho.py | 4 +- ietf/utils/serialize.py | 2 +- ietf/utils/test_runner.py | 155 ++++++------------ ietf/utils/tests.py | 14 +- ietf/utils/tests_coverage.py | 56 +++++++ ietf/utils/tests_meetecho.py | 26 +-- ietf/utils/timezone.py | 2 +- k8s/settings_local.py | 6 +- ...astypie-django22-fielderror-response.patch | 8 +- requirements.txt | 135 ++++++++------- 61 files changed, 505 insertions(+), 359 deletions(-) create mode 100644 dev/celery/Dockerfile rename ietf/utils/{test_smtpserver.py => aiosmtpd.py} (72%) create mode 100644 ietf/utils/coverage.py create mode 100644 ietf/utils/tests_coverage.py diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index ef8a17f6b43..4a4394fca0c 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -51,6 +51,7 @@ jobs: push: true tags: | ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }} + ghcr.io/ietf-tools/datatracker-app-base:py312 ${{ github.ref == 'refs/heads/main' && 'ghcr.io/ietf-tools/datatracker-app-base:latest' || '' }} - name: Update version references diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml index 8553563a192..d1fe0cdf623 100644 --- a/.github/workflows/tests-az.yml +++ b/.github/workflows/tests-az.yml @@ -62,7 +62,7 @@ jobs: echo "Starting Containers..." sudo docker network create dtnet sudo docker run -d --name db --network=dtnet ghcr.io/ietf-tools/datatracker-db:latest & - sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:latest sleep infinity & + sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:py312 sleep infinity & wait echo "Cloning datatracker repo..." diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 4bd0b993634..8b36b0e6ac9 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -105,10 +105,11 @@ "command": "/usr/local/bin/python", "args": [ "-m", - "smtpd", + "aiosmtpd", "-n", "-c", - "DebuggingServer", + "ietf.utils.aiosmtpd.DevDebuggingHandler", + "-l", "localhost:2025" ], "presentation": { diff --git a/README.md b/README.md index abebb7ca02b..4e1b7e1a454 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ Pages will gradually be updated to Vue 3 components. These components are locate Each Vue 3 app has its own sub-directory. For example, the agenda app is located under `/client/agenda`. -The datatracker makes use of the Django-Vite plugin to point to either the Vite.js server or the precompiled production files. The `DJANGO_VITE_DEV_MODE` flag, found in the `ietf/settings_local.py` file determines whether the Vite.js server is used or not. +The datatracker makes use of the Django-Vite plugin to point to either the Vite.js server or the precompiled production files. The `DJANGO_VITE["default"]["dev_mode"]` flag, found in the `ietf/settings_local.py` file determines whether the Vite.js server is used or not. In development mode, you must start the Vite.js development server, in addition to the usual Datatracker server: diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index d619ee99eeb..658f1e56950 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250821T1359 +FROM ghcr.io/ietf-tools/datatracker-app-base:20250819T1645 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index b6fc12e1283..9e510ad8dbd 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250821T1359 +20250819T1645 diff --git a/dev/celery/Dockerfile b/dev/celery/Dockerfile new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dev/deploy-to-container/cli.js b/dev/deploy-to-container/cli.js index 1a2d993ac4b..2f0faad151d 100644 --- a/dev/deploy-to-container/cli.js +++ b/dev/deploy-to-container/cli.js @@ -85,7 +85,7 @@ async function main () { // Pull latest Datatracker Base image console.info('Pulling latest Datatracker base docker image...') - const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:latest') + const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:py312') await new Promise((resolve, reject) => { dock.modem.followProgress(appImagePullStream, (err, res) => err ? reject(err) : resolve(res)) }) @@ -214,7 +214,7 @@ async function main () { const celeryContainers = {} for (const conConf of conConfs) { celeryContainers[conConf.name] = await dock.createContainer({ - Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest', + Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312', name: `dt-${conConf.name}-${branch}`, Hostname: `dt-${conConf.name}-${branch}`, Env: [ @@ -244,7 +244,7 @@ async function main () { // Create Datatracker container console.info(`Creating Datatracker docker container... [dt-app-${branch}]`) const appContainer = await dock.createContainer({ - Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest', + Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312', name: `dt-app-${branch}`, Hostname: `dt-app-${branch}`, Env: [ diff --git a/dev/diff/cli.js b/dev/diff/cli.js index 461b0c37a0c..0cf353cc651 100644 --- a/dev/diff/cli.js +++ b/dev/diff/cli.js @@ -567,7 +567,7 @@ async function main () { { title: 'Pulling latest Datatracker base docker image...', task: async (subctx, subtask) => { - const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:latest') + const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:py312') await new Promise((resolve, reject) => { dock.modem.followProgress(appImagePullStream, (err, res) => err ? reject(err) : resolve(res)) }) @@ -648,7 +648,7 @@ async function main () { title: 'Creating source Datatracker docker container...', task: async (subctx, subtask) => { containers.appSource = await dock.createContainer({ - Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest', + Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312', name: 'dt-diff-app-source', Tty: true, Hostname: 'appsource', @@ -664,7 +664,7 @@ async function main () { title: 'Creating target Datatracker docker container...', task: async (subctx, subtask) => { containers.appTarget = await dock.createContainer({ - Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest', + Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312', name: 'dt-diff-app-target', Tty: true, Hostname: 'apptarget', diff --git a/dev/tests/debug.sh b/dev/tests/debug.sh index d87c504bb91..e92e6d9b2af 100644 --- a/dev/tests/debug.sh +++ b/dev/tests/debug.sh @@ -9,7 +9,7 @@ # Simply type "exit" + ENTER to exit and shutdown this test environment. echo "Fetching latest images..." -docker pull ghcr.io/ietf-tools/datatracker-app-base:latest +docker pull ghcr.io/ietf-tools/datatracker-app-base:py312 docker pull ghcr.io/ietf-tools/datatracker-db:latest echo "Starting containers..." docker compose -f docker-compose.debug.yml -p dtdebug --compatibility up -d diff --git a/dev/tests/docker-compose.debug.yml b/dev/tests/docker-compose.debug.yml index 8117b923751..168bbd4e92b 100644 --- a/dev/tests/docker-compose.debug.yml +++ b/dev/tests/docker-compose.debug.yml @@ -5,7 +5,7 @@ version: '3.8' services: app: - image: ghcr.io/ietf-tools/datatracker-app-base:latest + image: ghcr.io/ietf-tools/datatracker-app-base:py312 command: -f /dev/null working_dir: /__w/datatracker/datatracker entrypoint: tail diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index fee38337330..e3df9bd4b48 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:latest +FROM ghcr.io/ietf-tools/datatracker-app-base:py312 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 57aac8ee561..c1fe5b093e9 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-bookworm +FROM python:3.12-bookworm LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/docker/celery.Dockerfile b/docker/celery.Dockerfile index e7c7b9cc3fd..279d5c7550c 100644 --- a/docker/celery.Dockerfile +++ b/docker/celery.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:latest +FROM ghcr.io/ietf-tools/datatracker-app-base:py312 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index ca518714637..3ee7a4295d2 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -89,6 +89,8 @@ secret_key="minio_pass", security_token=None, client_config=botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", signature_version="s3v4", connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, read_timeout=BLOBSTORAGE_READ_TIMEOUT, diff --git a/docker/configs/settings_local_vite.py b/docker/configs/settings_local_vite.py index 7fb12a003d1..9116905b124 100644 --- a/docker/configs/settings_local_vite.py +++ b/docker/configs/settings_local_vite.py @@ -2,5 +2,9 @@ # -*- coding: utf-8 -*- from ietf.settings_local import * # pyflakes:ignore +from ietf.settings_local import DJANGO_VITE -DJANGO_VITE_DEV_MODE = True +DJANGO_VITE["default"] |= { + "dev_mode": True, + "dev_server_port": 3000, +} diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py index df4685b246f..3140e393064 100755 --- a/docker/scripts/app-configure-blobstore.py +++ b/docker/scripts/app-configure-blobstore.py @@ -17,7 +17,11 @@ def init_blobstore(): aws_access_key_id=os.environ.get("BLOB_STORE_ACCESS_KEY", "minio_root"), aws_secret_access_key=os.environ.get("BLOB_STORE_SECRET_KEY", "minio_pass"), aws_session_token=None, - config=botocore.config.Config(signature_version="s3v4"), + config=botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + signature_version="s3v4", + ), ) for bucketname in ARTIFACT_STORAGE_NAMES: try: diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index 17e0c6c7647..1d895cdf53e 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -108,7 +108,7 @@ echo "Running initial checks..." if [ -z "$EDITOR_VSCODE" ]; then CODE=0 - python -m smtpd -n -c DebuggingServer localhost:2025 & + python -m aiosmtpd -n -c ietf.utils.aiosmtpd.DevDebuggingHandler -l localhost:2025 & if [ -z "$*" ]; then echo "-----------------------------------------------------------------" echo "Ready!" diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index 230f8339bdd..d4562f97ddf 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -181,7 +181,7 @@ class Serializer(tastypie.serializers.Serializer): OPTION_ESCAPE_NULLS = "datatracker-escape-nulls" def format_datetime(self, data): - return data.astimezone(datetime.timezone.utc).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" + return data.astimezone(datetime.UTC).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" def to_simple(self, data, options): options = options or {} diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 865f877bfb1..2a44791a5cf 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -462,12 +462,12 @@ def test_api_add_session_attendees(self): self.assertTrue(session.attended_set.filter(person=recman).exists()) self.assertEqual( session.attended_set.get(person=recman).time, - datetime.datetime(2023, 9, 3, 12, 34, 56, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 9, 3, 12, 34, 56, tzinfo=datetime.UTC), ) self.assertTrue(session.attended_set.filter(person=otherperson).exists()) self.assertEqual( session.attended_set.get(person=otherperson).time, - datetime.datetime(2023, 9, 3, 3, 0, 19, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 9, 3, 3, 0, 19, tzinfo=datetime.UTC), ) def test_api_upload_polls_and_chatlog(self): @@ -871,7 +871,7 @@ def test_api_new_meeting_registration_v2_nomcom(self): self.assertEqual(volunteer.origin, 'registration') def test_api_version(self): - DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=datetime.timezone.utc), host='testapi.example.com',tz='UTC') + DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=datetime.UTC), host='testapi.example.com',tz='UTC') url = urlreverse('ietf.api.views.version') r = self.client.get(url) data = r.json() diff --git a/ietf/bin/aliases-from-json.py b/ietf/bin/aliases-from-json.py index a0c383a1acc..0da5d1f8b97 100644 --- a/ietf/bin/aliases-from-json.py +++ b/ietf/bin/aliases-from-json.py @@ -38,7 +38,7 @@ def generate_files(records, adest, vdest, postconfirm, vdomain): vpath = tmppath / "virtual" with apath.open("w") as afile, vpath.open("w") as vfile: - date = datetime.datetime.now(datetime.timezone.utc) + date = datetime.datetime.now(datetime.UTC) signature = f"# Generated by {Path(__file__).absolute()} at {date}\n" afile.write(signature) vfile.write(signature) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index b6f36cb8a76..25ee734cbe2 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1157,7 +1157,7 @@ def fake_history_obj(self, rev): elif rev_events.exists(): time = rev_events.first().time else: - time = datetime.datetime.fromtimestamp(0, datetime.timezone.utc) + time = datetime.datetime.fromtimestamp(0, datetime.UTC) dh = DocHistory(name=self.name, rev=rev, doc=self, time=time, type=self.type, title=self.title, stream=self.stream, group=self.group) diff --git a/ietf/doc/templatetags/ballot_icon.py b/ietf/doc/templatetags/ballot_icon.py index a94c1450075..07a6c7f9265 100644 --- a/ietf/doc/templatetags/ballot_icon.py +++ b/ietf/doc/templatetags/ballot_icon.py @@ -196,7 +196,7 @@ def state_age_colored(doc): .time ) except IndexError: - state_datetime = datetime.datetime(1990, 1, 1, tzinfo=datetime.timezone.utc) + state_datetime = datetime.datetime(1990, 1, 1, tzinfo=datetime.UTC) days = (timezone.now() - state_datetime).days # loosely based on the Publish Path page at the iesg wiki if iesg_state == "lc": diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index ab7eaba7684..ab33acebe6d 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -678,11 +678,11 @@ def test_in_draft_expire_freeze(self): datetime.datetime.combine( ietf_monday - datetime.timedelta(days=1), datetime.time(0, 0, 0), - tzinfo=datetime.timezone.utc, + tzinfo=datetime.UTC, ) )) self.assertFalse(in_draft_expire_freeze( - datetime.datetime.combine(ietf_monday, datetime.time(0, 0, 0), tzinfo=datetime.timezone.utc) + datetime.datetime.combine(ietf_monday, datetime.time(0, 0, 0), tzinfo=datetime.UTC) )) def test_warn_expirable_drafts(self): diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py index f610fe3d761..7db59819da8 100644 --- a/ietf/doc/tests_utils.py +++ b/ietf/doc/tests_utils.py @@ -148,7 +148,7 @@ def test_update_action_holders_resets_age(self): doc = self.doc_in_iesg_state('pub-req') doc.action_holders.set([self.ad]) dah = doc.documentactionholder_set.get(person=self.ad) - dah.time_added = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc) # arbitrary date in the past + dah.time_added = datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC) # arbitrary date in the past dah.save() right_now = timezone.now() diff --git a/ietf/doc/views_stats.py b/ietf/doc/views_stats.py index 0bbf0b91c50..028573b338d 100644 --- a/ietf/doc/views_stats.py +++ b/ietf/doc/views_stats.py @@ -18,7 +18,7 @@ from ietf.utils.timezone import date_today -epochday = datetime.datetime.utcfromtimestamp(0).date().toordinal() +epochday = datetime.datetime.fromtimestamp(0, datetime.UTC).date().toordinal() def dt(s): @@ -35,13 +35,13 @@ def model_to_timeline_data(model, field='time', **kwargs): assert field in [ f.name for f in model._meta.get_fields() ] objects = ( model.objects.filter(**kwargs) - .annotate(date=TruncDate(field, tzinfo=datetime.timezone.utc)) + .annotate(date=TruncDate(field, tzinfo=datetime.UTC)) .order_by('date') .values('date') .annotate(count=Count('id'))) if objects.exists(): obj_list = list(objects) - today = date_today(datetime.timezone.utc) + today = date_today(datetime.UTC) if not obj_list[-1]['date'] == today: obj_list += [ {'date': today, 'count': 0} ] data = [ ((e['date'].toordinal()-epochday)*1000*60*60*24, e['count']) for e in obj_list ] diff --git a/ietf/group/views.py b/ietf/group/views.py index 3529b31f683..bc785ff81e1 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -941,7 +941,7 @@ def meetings(request, acronym, group_type=None): cutoff_date = revsub_dates_by_meeting[s.meeting.pk] else: cutoff_date = s.meeting.date + datetime.timedelta(days=s.meeting.submission_correction_day_offset) - s.cached_is_cutoff = date_today(datetime.timezone.utc) > cutoff_date + s.cached_is_cutoff = date_today(datetime.UTC) > cutoff_date future, in_progress, recent, past = group_sessions(sessions) diff --git a/ietf/idindex/index.py b/ietf/idindex/index.py index 4f021c0dc7d..19eb29d4dad 100644 --- a/ietf/idindex/index.py +++ b/ietf/idindex/index.py @@ -276,7 +276,7 @@ def active_drafts_index_by_group(extra_values=()): groups = [g for g in groups_dict.values() if hasattr(g, "active_drafts")] groups.sort(key=lambda g: g.acronym) - fallback_time = datetime.datetime(1950, 1, 1, tzinfo=datetime.timezone.utc) + fallback_time = datetime.datetime(1950, 1, 1, tzinfo=datetime.UTC) for g in groups: g.active_drafts.sort(key=lambda d: d.get("initial_rev_time", fallback_time)) @@ -302,6 +302,6 @@ def id_index_txt(with_abstracts=False): return render_to_string("idindex/id_index.txt", { 'groups': groups, - 'time': timezone.now().astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z"), + 'time': timezone.now().astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S %Z"), 'with_abstracts': with_abstracts, }) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index 7b9f489b442..ffd4515c98f 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -101,7 +101,7 @@ def agenda_json(request, date=None): res = { "telechat-date": str(data["date"]), - "as-of": str(datetime.datetime.utcnow()), + "as-of": str(datetime.datetime.now(datetime.UTC)), "page-counts": telechat_page_count(date=get_agenda_date(date))._asdict(), "sections": {}, } diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py index 4219747e122..b5256b14f8b 100644 --- a/ietf/ietfauth/views.py +++ b/ietf/ietfauth/views.py @@ -517,7 +517,7 @@ def confirm_password_reset(request, auth): password = data['password'] last_login = None if data['last_login']: - last_login = datetime.datetime.fromtimestamp(data['last_login'], datetime.timezone.utc) + last_login = datetime.datetime.fromtimestamp(data['last_login'], datetime.UTC) except django.core.signing.BadSignature: raise Http404("Invalid or expired auth") diff --git a/ietf/ipr/mail.py b/ietf/ipr/mail.py index 167b11956c9..9bef751b952 100644 --- a/ietf/ipr/mail.py +++ b/ietf/ipr/mail.py @@ -66,9 +66,9 @@ def utc_from_string(s): if date is None: return None elif is_aware(date): - return date.astimezone(datetime.timezone.utc) + return date.astimezone(datetime.UTC) else: - return date.replace(tzinfo=datetime.timezone.utc) + return date.replace(tzinfo=datetime.UTC) # ---------------------------------------------------------------- # Email Functions diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index 24453df2d2d..08979a39725 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -152,13 +152,13 @@ def ipr_rfc_number(disclosureDate, thirdPartyDisclosureFlag): # RFC publication date comes from the RFC Editor announcement ipr_rfc_pub_datetime = { - 1310 : datetime.datetime(1992, 3, 13, 0, 0, tzinfo=datetime.timezone.utc), - 1802 : datetime.datetime(1994, 3, 23, 0, 0, tzinfo=datetime.timezone.utc), - 2026 : datetime.datetime(1996, 10, 29, 0, 0, tzinfo=datetime.timezone.utc), - 3668 : datetime.datetime(2004, 2, 18, 0, 0, tzinfo=datetime.timezone.utc), - 3979 : datetime.datetime(2005, 3, 2, 2, 23, tzinfo=datetime.timezone.utc), - 4879 : datetime.datetime(2007, 4, 10, 18, 21, tzinfo=datetime.timezone.utc), - 8179 : datetime.datetime(2017, 5, 31, 23, 1, tzinfo=datetime.timezone.utc), + 1310 : datetime.datetime(1992, 3, 13, 0, 0, tzinfo=datetime.UTC), + 1802 : datetime.datetime(1994, 3, 23, 0, 0, tzinfo=datetime.UTC), + 2026 : datetime.datetime(1996, 10, 29, 0, 0, tzinfo=datetime.UTC), + 3668 : datetime.datetime(2004, 2, 18, 0, 0, tzinfo=datetime.UTC), + 3979 : datetime.datetime(2005, 3, 2, 2, 23, tzinfo=datetime.UTC), + 4879 : datetime.datetime(2007, 4, 10, 18, 21, tzinfo=datetime.UTC), + 8179 : datetime.datetime(2017, 5, 31, 23, 1, tzinfo=datetime.UTC), } if disclosureDate < ipr_rfc_pub_datetime[1310]: diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index 8bbaa4f053d..a1fbf77841a 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -723,7 +723,7 @@ def test_add_incoming_liaison(self): from_groups = [ str(g.pk) for g in Group.objects.filter(type="sdo") ] to_group = Group.objects.get(acronym="mars") submitter = Person.objects.get(user__username="marschairman") - today = date_today(datetime.timezone.utc) + today = date_today(datetime.UTC) related_liaison = liaison r = self.client.post(url, dict(from_groups=from_groups, @@ -808,7 +808,7 @@ def test_add_outgoing_liaison(self): from_group = Group.objects.get(acronym="mars") to_group = Group.objects.filter(type="sdo")[0] submitter = Person.objects.get(user__username="marschairman") - today = date_today(datetime.timezone.utc) + today = date_today(datetime.UTC) related_liaison = liaison r = self.client.post(url, dict(from_groups=str(from_group.pk), @@ -878,7 +878,7 @@ def test_add_outgoing_liaison_unapproved_post_only(self): from_group = Group.objects.get(acronym="mars") to_group = Group.objects.filter(type="sdo")[0] submitter = Person.objects.get(user__username="marschairman") - today = date_today(datetime.timezone.utc) + today = date_today(datetime.UTC) r = self.client.post(url, dict(from_groups=str(from_group.pk), from_contact=submitter.email_address(), @@ -1062,7 +1062,7 @@ def test_search(self): LiaisonStatementEventFactory(type_id='posted', statement__body="Has recently in its body",statement__from_groups=[GroupFactory(type_id='sdo',acronym='ulm'),]) # Statement 2 s2 = LiaisonStatementEventFactory(type_id='posted', statement__body="That word does not occur here", statement__title="Nor does it occur here") - s2.time=datetime.datetime(2010, 1, 1, tzinfo=datetime.timezone.utc) + s2.time=datetime.datetime(2010, 1, 1, tzinfo=datetime.UTC) s2.save() # test list only, no search filters diff --git a/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py b/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py index 8f5db261127..8c467ea156c 100644 --- a/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py +++ b/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py @@ -4,6 +4,7 @@ class Migration(migrations.Migration): + dependencies = [ ("meeting", "0015_alter_meeting_time_zone"), ] diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index de0192769e6..f3df23e9169 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -149,7 +149,7 @@ def get_00_cutoff(self): cutoff_date = importantdate.date else: cutoff_date = self.date + datetime.timedelta(days=ImportantDateName.objects.get(slug='idcutoff').default_offset_days) - cutoff_time = datetime_from_date(cutoff_date, datetime.timezone.utc) + self.idsubmit_cutoff_time_utc + cutoff_time = datetime_from_date(cutoff_date, datetime.UTC) + self.idsubmit_cutoff_time_utc return cutoff_time def get_01_cutoff(self): @@ -161,7 +161,7 @@ def get_01_cutoff(self): cutoff_date = importantdate.date else: cutoff_date = self.date + datetime.timedelta(days=ImportantDateName.objects.get(slug='idcutoff').default_offset_days) - cutoff_time = datetime_from_date(cutoff_date, datetime.timezone.utc) + self.idsubmit_cutoff_time_utc + cutoff_time = datetime_from_date(cutoff_date, datetime.UTC) + self.idsubmit_cutoff_time_utc return cutoff_time def get_reopen_time(self): @@ -1172,7 +1172,7 @@ def can_manage_materials(self, user): return can_manage_materials(user,self.group) def is_material_submission_cutoff(self): - return date_today(datetime.timezone.utc) > self.meeting.get_submission_correction_date() + return date_today(datetime.UTC) > self.meeting.get_submission_correction_date() def joint_with_groups_acronyms(self): return [group.acronym for group in self.joint_with_groups.all()] diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index a184a7c6d01..262b47652cf 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -1576,7 +1576,7 @@ def test_delete_timeslot_cancel(self): def do_delete_time_interval_test(self, cancel=False): delete_time_local = datetime_from_date(self.meeting.date, self.meeting.tz()).replace(hour=10) - delete_time = delete_time_local.astimezone(datetime.timezone.utc) + delete_time = delete_time_local.astimezone(datetime.UTC) duration = datetime.timedelta(minutes=60) delete: [TimeSlot] = TimeSlotFactory.create_batch( # type: ignore[annotation-unchecked] diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py index 0c442c4bf78..a5da00ecbf5 100644 --- a/ietf/meeting/tests_tasks.py +++ b/ietf/meeting/tests_tasks.py @@ -23,7 +23,7 @@ def test_proceedings_content_refresh_task(self, mock_generate): meeting127 = MeetingFactory(type_id="ietf", number="127") # 24 * 5 + 7 # Times to be returned - now_utc = datetime.datetime.now(tz=datetime.timezone.utc) + now_utc = datetime.datetime.now(tz=datetime.UTC) hour_00_utc = now_utc.replace(hour=0) hour_01_utc = now_utc.replace(hour=1) hour_07_utc = now_utc.replace(hour=7) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index f382772485f..bd3ab772fca 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -55,9 +55,8 @@ from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose, generate_agenda_data from ietf.name.models import SessionStatusName, ImportantDateName, RoleName, ProceedingsMaterialTypeName -from ietf.utils.decorators import skip_coverage from ietf.utils.mail import outbox, empty_outbox, get_payload_text -from ietf.utils.test_runner import TestBlobstoreManager +from ietf.utils.test_runner import TestBlobstoreManager, disable_coverage from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent from ietf.utils.timezone import date_today, time_now @@ -321,11 +320,11 @@ def test_meeting_agenda(self): self.assertContains(r, session.group.parent.acronym.upper()) self.assertContains(r, slot.location.name) self.assertContains(r, "{}-{}".format( - slot.time.astimezone(datetime.timezone.utc).strftime("%H%M"), - (slot.time + slot.duration).astimezone(datetime.timezone.utc).strftime("%H%M"), + slot.time.astimezone(datetime.UTC).strftime("%H%M"), + (slot.time + slot.duration).astimezone(datetime.UTC).strftime("%H%M"), )) self.assertContains(r, "shown in UTC") - updated = meeting.updated().astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z") + updated = meeting.updated().astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S %Z") self.assertContains(r, f"Updated {updated}") # text, invalid updated (none) @@ -369,8 +368,8 @@ def test_meeting_agenda(self): self.assertContains(r, session.group.parent.acronym.upper()) self.assertContains(r, slot.location.name) self.assertContains(r, registration_text) - start_time = slot.time.astimezone(datetime.timezone.utc) - end_time = slot.end_time().astimezone(datetime.timezone.utc) + start_time = slot.time.astimezone(datetime.UTC) + end_time = slot.end_time().astimezone(datetime.UTC) self.assertContains(r, '"{}","{}","{}"'.format( start_time.strftime("%Y-%m-%d"), start_time.strftime("%H%M"), @@ -1037,7 +1036,7 @@ def test_important_dates_ical(self): updated = meeting.updated() self.assertIsNotNone(updated) - expected_updated = updated.astimezone(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + expected_updated = updated.astimezone(datetime.UTC).strftime("%Y%m%dT%H%M%SZ") self.assertContains(r, f"DTSTAMP:{expected_updated}") dtstamps_count = r.content.decode("utf-8").count(f"DTSTAMP:{expected_updated}") self.assertEqual(dtstamps_count, meeting.importantdate_set.count()) @@ -1181,8 +1180,8 @@ def test_session_draft_tarfile(self): os.unlink(filename) @skipIf(skip_pdf_tests, skip_message) - @skip_coverage - def test_session_draft_pdf(self): + @disable_coverage() + def test_session_draft_pdf(self): # pragma: no cover session, filenames = self.build_session_setup() try: url = urlreverse('ietf.meeting.views.session_draft_pdf', kwargs={'num':session.meeting.number,'acronym':session.group.acronym}) @@ -2117,8 +2116,8 @@ def test_editor_time_zone(self): # strftime() does not seem to support hours without leading 0, so do this manually time_label_string = f'{ts_start.hour:d}:{ts_start.minute:02d} - {ts_end.hour:d}:{ts_end.minute:02d}' self.assertIn(time_label_string, time_label.text()) - self.assertEqual(time_label.attr('data-start'), ts_start.astimezone(datetime.timezone.utc).isoformat()) - self.assertEqual(time_label.attr('data-end'), ts_end.astimezone(datetime.timezone.utc).isoformat()) + self.assertEqual(time_label.attr('data-start'), ts_start.astimezone(datetime.UTC).isoformat()) + self.assertEqual(time_label.attr('data-end'), ts_end.astimezone(datetime.UTC).isoformat()) ts_swap = time_label.find('.swap-timeslot-col') origin_label = ts_swap.attr('data-origin-label') @@ -2129,8 +2128,8 @@ def test_editor_time_zone(self): timeslot_elt = pq(f'#timeslot{timeslot.pk}') self.assertEqual(len(timeslot_elt), 1) - self.assertEqual(timeslot_elt.attr('data-start'), ts_start.astimezone(datetime.timezone.utc).isoformat()) - self.assertEqual(timeslot_elt.attr('data-end'), ts_end.astimezone(datetime.timezone.utc).isoformat()) + self.assertEqual(timeslot_elt.attr('data-start'), ts_start.astimezone(datetime.UTC).isoformat()) + self.assertEqual(timeslot_elt.attr('data-end'), ts_end.astimezone(datetime.UTC).isoformat()) timeslot_label = pq(f'#timeslot{timeslot.pk} .time-label') self.assertEqual(len(timeslot_label), 1) @@ -5233,7 +5232,7 @@ def test_upcoming_ical(self): updated = meeting.updated() self.assertIsNotNone(updated) - expected_updated = updated.astimezone(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + expected_updated = updated.astimezone(datetime.UTC).strftime("%Y%m%dT%H%M%SZ") self.assertContains(r, f"DTSTAMP:{expected_updated}") # With default cached_updated, 1970-01-01 diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 7fa3d212591..fcc9312609f 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -153,7 +153,7 @@ def materials(request, num=None): begin_date = meeting.get_submission_start_date() cut_off_date = meeting.get_submission_cut_off_date() cor_cut_off_date = meeting.get_submission_correction_date() - today_utc = date_today(datetime.timezone.utc) + today_utc = date_today(datetime.UTC) old = timezone.now() - datetime.timedelta(days=1) if settings.SERVER_MODE != 'production' and '_testoverride' in request.GET: pass @@ -1921,7 +1921,7 @@ def slides_field(item): write_row(headings) - tz = datetime.timezone.utc if utc else schedule.meeting.tz() + tz = datetime.UTC if utc else schedule.meeting.tz() for item in filtered_assignments: row = [] row.append(item.timeslot.time.astimezone(tz).strftime("%Y-%m-%d")) @@ -2814,7 +2814,7 @@ def session_attendance(request, session_id, num): raise Http404("Bluesheets not found") cor_cut_off_date = session.meeting.get_submission_correction_date() - today_utc = date_today(datetime.timezone.utc) + today_utc = date_today(datetime.UTC) was_there = False can_add = False if request.user.is_authenticated: diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index cc2e0826d3d..dcdb9ef836f 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -2930,7 +2930,7 @@ def test_decorate_volunteers_with_qualifications(self): elig_date.year - 3, elig_date.month, 28 if elig_date.month == 2 and elig_date.day == 29 else elig_date.day, - tzinfo=datetime.timezone.utc, + tzinfo=datetime.UTC, ) ) nomcom.volunteer_set.create(person=author_person) diff --git a/ietf/nomcom/views.py b/ietf/nomcom/views.py index c04e13f92b7..3f90be5253d 100644 --- a/ietf/nomcom/views.py +++ b/ietf/nomcom/views.py @@ -981,7 +981,7 @@ def view_feedback_topic(request, year, topic_id): reviewer = request.user.person last_seen = TopicFeedbackLastSeen.objects.filter(reviewer=reviewer,topic=topic).first() - last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.timezone.utc) + last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.UTC) if last_seen: last_seen.save() else: @@ -1044,7 +1044,7 @@ def view_feedback_nominee(request, year, nominee_id): }) last_seen = FeedbackLastSeen.objects.filter(reviewer=reviewer,nominee=nominee).first() - last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.timezone.utc) + last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.UTC) if last_seen: last_seen.save() else: diff --git a/ietf/settings.py b/ietf/settings.py index 3af01d76e6e..753508dc993 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -14,19 +14,27 @@ from hashlib import sha384 from typing import Any, Dict, List, Tuple # pyflakes:ignore +# DeprecationWarnings are suppressed by default, enable them warnings.simplefilter("always", DeprecationWarning) -warnings.filterwarnings("ignore", message="pkg_resources is deprecated as an API") -warnings.filterwarnings("ignore", "Log out via GET requests is deprecated") # happens in oidc_provider -warnings.filterwarnings("ignore", module="tastypie", message="The django.utils.datetime_safe module is deprecated.") -warnings.filterwarnings("ignore", module="oidc_provider", message="The django.utils.timezone.utc alias is deprecated.") + +# Warnings that must be resolved for Django 5.x +warnings.filterwarnings("ignore", "Log out via GET requests is deprecated") # caused by oidc_provider +warnings.filterwarnings("ignore", message="The django.utils.timezone.utc alias is deprecated.", module="oidc_provider") +warnings.filterwarnings("ignore", message="The django.utils.datetime_safe module is deprecated.", module="tastypie") warnings.filterwarnings("ignore", message="The USE_DEPRECATED_PYTZ setting,") # https://github.com/ietf-tools/datatracker/issues/5635 warnings.filterwarnings("ignore", message="The USE_L10N setting is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5648 warnings.filterwarnings("ignore", message="django.contrib.auth.hashers.CryptPasswordHasher is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5663 -warnings.filterwarnings("ignore", message="'urllib3\\[secure\\]' extra is deprecated") -warnings.filterwarnings("ignore", message="The logout\\(\\) view is superseded by") + +# Other DeprecationWarnings +warnings.filterwarnings("ignore", message="pkg_resources is deprecated as an API", module="pyang.plugin") warnings.filterwarnings("ignore", message="Report.file_reporters will no longer be available in Coverage.py 4.2", module="coverage.report") -warnings.filterwarnings("ignore", message="Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated", module="bleach") -warnings.filterwarnings("ignore", message="HTTPResponse.getheader\\(\\) is deprecated", module='selenium.webdriver') +warnings.filterwarnings("ignore", message="currentThread\\(\\) is deprecated", module="coverage.pytracer") +warnings.filterwarnings("ignore", message="co_lnotab is deprecated", module="coverage.parser") +warnings.filterwarnings("ignore", message="datetime.datetime.utcnow\\(\\) is deprecated", module="botocore.auth") +warnings.filterwarnings("ignore", message="datetime.datetime.utcnow\\(\\) is deprecated", module="oic.utils.time_util") +warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="oic.utils.time_util") +warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="pytz.tzinfo") + base_path = pathlib.Path(__file__).resolve().parent BASE_DIR = str(base_path) @@ -447,23 +455,24 @@ def skip_unreadable_post(record): "ietf.middleware.SMTPExceptionMiddleware", "ietf.middleware.Utf8ExceptionMiddleware", "ietf.middleware.redirect_trailing_period_middleware", - "django_referrer_policy.middleware.ReferrerPolicyMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", "django.middleware.security.SecurityMiddleware", - #"csp.middleware.CSPMiddleware", "ietf.middleware.unicode_nfkc_normalization_middleware", "ietf.middleware.is_authenticated_header_middleware", ] ROOT_URLCONF = 'ietf.urls' -DJANGO_VITE_ASSETS_PATH = os.path.join(BASE_DIR, 'static/dist-neue') +# Configure django_vite +DJANGO_VITE: dict = {"default": {}} if DEBUG: - DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') + DJANGO_VITE["default"]["manifest_path"] = os.path.join( + BASE_DIR, 'static/dist-neue/manifest.json' + ) # Additional locations of static files (in addition to each app's static/ dir) STATICFILES_DIRS = ( - DJANGO_VITE_ASSETS_PATH, + os.path.join(BASE_DIR, "static/dist-neue"), # for django_vite os.path.join(BASE_DIR, 'static/dist'), os.path.join(BASE_DIR, 'secr/static/dist'), ) @@ -567,8 +576,6 @@ def skip_unreadable_post(record): CORS_ALLOW_METHODS = ( 'GET', 'OPTIONS', ) CORS_URLS_REGEX = r'^(/api/.*|.*\.json|.*/json/?)$' -# Setting for django_referrer_policy.middleware.ReferrerPolicyMiddleware -REFERRER_POLICY = 'strict-origin-when-cross-origin' # django.middleware.security.SecurityMiddleware SECURE_BROWSER_XSS_FILTER = True @@ -581,6 +588,7 @@ def skip_unreadable_post(record): #SECURE_SSL_REDIRECT = True # Relax the COOP policy to allow Meetecho authentication pop-up SECURE_CROSS_ORIGIN_OPENER_POLICY = "unsafe-none" +SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin" # Override this in your settings_local with the IP addresses relevant for you: INTERNAL_IPS = ( @@ -666,11 +674,6 @@ def skip_unreadable_post(record): IDNITS3_BASE_URL = "https://author-tools.ietf.org/idnits3/results" IDNITS_SERVICE_URL = "https://author-tools.ietf.org/idnits" -# Content security policy configuration (django-csp) -# (In current production, the Content-Security-Policy header is completely set by nginx configuration, but -# we try to keep this in sync to avoid confusion) -CSP_DEFAULT_SRC = ("'self'", "'unsafe-inline'", f"data: {IDTRACKER_BASE_URL} http://ietf.org/ https://www.ietf.org/ https://analytics.ietf.org/ https://static.ietf.org") - # The name of the method to use to invoke the test suite TEST_RUNNER = 'ietf.utils.test_runner.IetfTestRunner' @@ -709,6 +712,7 @@ def skip_unreadable_post(record): "ietf/utils/patch.py", "ietf/utils/test_data.py", "ietf/utils/jstest.py", + "ietf/utils/coverage.py", ] # These are code line regex patterns @@ -738,8 +742,8 @@ def skip_unreadable_post(record): TEST_CODE_COVERAGE_CHECKER = None if SERVER_MODE != 'production': - import coverage - TEST_CODE_COVERAGE_CHECKER = coverage.Coverage(source=[ BASE_DIR ], cover_pylib=False, omit=TEST_CODE_COVERAGE_EXCLUDE_FILES) + from ietf.utils.coverage import CoverageManager + TEST_CODE_COVERAGE_CHECKER = CoverageManager() TEST_CODE_COVERAGE_REPORT_PATH = "coverage/" TEST_CODE_COVERAGE_REPORT_URL = os.path.join(STATIC_URL, TEST_CODE_COVERAGE_REPORT_PATH, "index.html") diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 9a42e8b99d8..6479069db02 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -14,7 +14,7 @@ import shutil import tempfile from ietf.settings import * # pyflakes:ignore -from ietf.settings import TEST_CODE_COVERAGE_CHECKER, ORIG_AUTH_PASSWORD_VALIDATORS +from ietf.settings import ORIG_AUTH_PASSWORD_VALIDATORS import debug # pyflakes:ignore debug.debug = True @@ -52,10 +52,9 @@ def __getitem__(self, item): BLOBDB_DATABASE = "default" DATABASE_ROUTERS = [] # type: ignore -if TEST_CODE_COVERAGE_CHECKER and not TEST_CODE_COVERAGE_CHECKER._started: # pyflakes:ignore +if TEST_CODE_COVERAGE_CHECKER: # pyflakes:ignore TEST_CODE_COVERAGE_CHECKER.start() # pyflakes:ignore - def tempdir_with_cleanup(**kwargs): """Utility to create a temporary dir and arrange cleanup""" _dir = tempfile.mkdtemp(**kwargs) diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index 89908748a73..e02b6865767 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -18,7 +18,7 @@ from ietf.utils import tool_version from ietf.utils.log import log, assertion from ietf.utils.pipe import pipe -from ietf.utils.test_runner import set_coverage_checking +from ietf.utils.test_runner import disable_coverage class DraftSubmissionChecker(object): name = "" @@ -247,34 +247,33 @@ def check_file_txt(self, path): ) # yanglint - set_coverage_checking(False) # we can't count the following as it may or may not be run, depending on setup - if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY): - cmd_template = settings.SUBMIT_YANGLINT_COMMAND - command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir, - draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, - cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, ) - code, out, err = pipe(cmd) - out = out.decode('utf-8') - err = err.decode('utf-8') - if code > 0 or len(err.strip()) > 0: - err_lines = err.splitlines() - for line in err_lines: - if line.strip(): - try: - if 'err : ' in line: - errors += 1 - if 'warn: ' in line: - warnings += 1 - except ValueError: - pass - #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "{version}: {template}:\n{output}\n".format( - version=tool_version[command], - template=cmd_template, - output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, - ) - set_coverage_checking(True) + with disable_coverage(): # pragma: no cover + if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY): + cmd_template = settings.SUBMIT_YANGLINT_COMMAND + command = [ w for w in cmd_template.split() if not '=' in w ][0] + cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir, + draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, + cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, ) + code, out, err = pipe(cmd) + out = out.decode('utf-8') + err = err.decode('utf-8') + if code > 0 or len(err.strip()) > 0: + err_lines = err.splitlines() + for line in err_lines: + if line.strip(): + try: + if 'err : ' in line: + errors += 1 + if 'warn: ' in line: + warnings += 1 + except ValueError: + pass + #passed = passed and code == 0 # For the submission tool. Yang checks always pass + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) else: errors += 1 message += "No such file: %s\nPossible mismatch between extracted xym file name and returned module name?\n" % (path) diff --git a/ietf/sync/iana.py b/ietf/sync/iana.py index f46fe407d42..0d40c5337e2 100644 --- a/ietf/sync/iana.py +++ b/ietf/sync/iana.py @@ -66,8 +66,8 @@ def update_rfc_log_from_protocol_page(rfc_names, rfc_must_published_later_than): def fetch_changes_json(url, start, end): - url += "?start=%s&end=%s" % (urlquote(start.astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S")), - urlquote(end.astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S"))) + url += "?start=%s&end=%s" % (urlquote(start.astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S")), + urlquote(end.astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"))) # HTTP basic auth username = "ietfsync" password = settings.IANA_SYNC_PASSWORD @@ -161,7 +161,7 @@ def update_history_with_changes(changes, send_email=True): for c in changes: docname = c['doc'] - timestamp = datetime.datetime.strptime(c["time"], "%Y-%m-%d %H:%M:%S",).replace(tzinfo=datetime.timezone.utc) + timestamp = datetime.datetime.strptime(c["time"], "%Y-%m-%d %H:%M:%S",).replace(tzinfo=datetime.UTC) if c['type'] in ("iana_state", "iana_review"): if c['type'] == "iana_state": @@ -247,7 +247,7 @@ def parse_review_email(text): review_time = parsedate_to_datetime(msg["Date"]) # parsedate_to_datetime() may return a naive timezone - treat as UTC if review_time.tzinfo is None or review_time.tzinfo.utcoffset(review_time) is None: - review_time = review_time.replace(tzinfo=datetime.timezone.utc) + review_time = review_time.replace(tzinfo=datetime.UTC) # by by = None diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index 18ab4fe66e8..e4174d37296 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -152,7 +152,7 @@ def iana_protocols_update_task(): 2012, 11, 26, - tzinfo=datetime.timezone.utc, + tzinfo=datetime.UTC, ) try: diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 182b6e24c40..3432f6214ad 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -206,7 +206,7 @@ def test_iana_review_mail(self): doc_name, review_time, by, comment = iana.parse_review_email(msg.encode('utf-8')) self.assertEqual(doc_name, draft.name) - self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 12, 0, rtime, tzinfo=datetime.timezone.utc)) + self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 12, 0, rtime, tzinfo=datetime.UTC)) self.assertEqual(by, Person.objects.get(user__username="iana")) self.assertIn("there are no IANA Actions", comment.replace("\n", "")) @@ -240,7 +240,7 @@ def test_ingest_review_email(self, mock_parse_review_email, mock_add_review_comm args = ( "doc-name", - datetime.datetime.now(tz=datetime.timezone.utc), + datetime.datetime.now(tz=datetime.UTC), PersonFactory(), "yadda yadda yadda", ) @@ -1121,7 +1121,7 @@ def test_iana_protocols_update_task( ) self.assertEqual( published_later_than, - {datetime.datetime(2012,11,26,tzinfo=datetime.timezone.utc)} + {datetime.datetime(2012,11,26,tzinfo=datetime.UTC)} ) # try with an exception diff --git a/ietf/utils/test_smtpserver.py b/ietf/utils/aiosmtpd.py similarity index 72% rename from ietf/utils/test_smtpserver.py rename to ietf/utils/aiosmtpd.py index 40da758d668..3e4cd65dd99 100644 --- a/ietf/utils/test_smtpserver.py +++ b/ietf/utils/aiosmtpd.py @@ -1,10 +1,14 @@ # Copyright The IETF Trust 2014-2025, All Rights Reserved -# -*- coding: utf-8 -*- +"""aiosmtpd-related utilities +These are for testing / dev use. If you're using this for production code, think very +hard about the choices you're making... +""" +from aiosmtpd import handlers from aiosmtpd.controller import Controller from aiosmtpd.smtp import SMTP from email.utils import parseaddr -from typing import Optional +from typing import Optional, TextIO class SMTPTestHandler: @@ -54,3 +58,16 @@ def start(self): def stop(self): self.controller.stop() + + +class DevDebuggingHandler(handlers.Debugging): + """Debugging handler for use in dev ONLY""" + def __init__(self, stream: Optional[TextIO] = None): + # Allow longer lines than the 1001 that RFC 5321 requires. As of 2025-04-16 the + # datatracker emits some non-compliant messages. + # See https://aiosmtpd.aio-libs.org/en/latest/smtp.html + # Doing this in a handler class is a huge hack. Tests all pass with this set + # to 4000, but make the limit longer for dev just in case. + SMTP.line_length_limit = 10000 + super().__init__(stream) + diff --git a/ietf/utils/coverage.py b/ietf/utils/coverage.py new file mode 100644 index 00000000000..bd205ce5867 --- /dev/null +++ b/ietf/utils/coverage.py @@ -0,0 +1,90 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from coverage import Coverage, CoverageData, FileReporter +from coverage.control import override_config as override_coverage_config +from coverage.results import Numbers +from coverage.report_core import get_analysis_to_report +from coverage.results import Analysis +from django.conf import settings + + +class CoverageManager: + checker: Coverage | None = None + started = False + + def start(self): + if settings.SERVER_MODE != "production" and not self.started: + self.checker = Coverage( + source=[settings.BASE_DIR], + cover_pylib=False, + omit=settings.TEST_CODE_COVERAGE_EXCLUDE_FILES, + ) + for exclude_regex in getattr( + settings, + "TEST_CODE_COVERAGE_EXCLUDE_LINES", + [], + ): + self.checker.exclude(exclude_regex) + self.checker.start() + self.started = True + + def stop(self): + if self.checker is not None: + self.checker.stop() + + def save(self): + if self.checker is not None: + self.checker.save() + + def report(self, include: list[str] | None = None): + if self.checker is None: + return None + reporter = CustomDictReporter() + with override_coverage_config( + self.checker, + report_include=include, + ): + return reporter.report(self.checker) + + +class CustomDictReporter: # pragma: no cover + total = Numbers() + + def report(self, coverage): + coverage_data = coverage.get_data() + coverage_data.set_query_contexts(None) + measured_files = {} + for file_reporter, analysis in get_analysis_to_report(coverage, None): + measured_files[file_reporter.relative_filename()] = self.report_one_file( + coverage_data, + analysis, + file_reporter, + ) + tot_numer, tot_denom = self.total.ratio_covered + return { + "coverage": 1 if tot_denom == 0 else tot_numer / tot_denom, + "covered": measured_files, + "format": 5, + } + + def report_one_file( + self, + coverage_data: CoverageData, + analysis: Analysis, + file_reporter: FileReporter, + ): + """Extract the relevant report data for a single file.""" + nums = analysis.numbers + self.total += nums + n_statements = nums.n_statements + numer, denom = nums.ratio_covered + fraction_covered = 1 if denom == 0 else numer / denom + missing_line_nums = sorted(analysis.missing) + # Extract missing lines from source files + source_lines = file_reporter.source().splitlines() + missing_lines = [source_lines[num - 1] for num in missing_line_nums] + return ( + n_statements, + fraction_covered, + missing_line_nums, + missing_lines, + ) diff --git a/ietf/utils/decorators.py b/ietf/utils/decorators.py index 5e94dda91d6..b50e0e7f96d 100644 --- a/ietf/utils/decorators.py +++ b/ietf/utils/decorators.py @@ -15,21 +15,9 @@ import debug # pyflakes:ignore -from ietf.utils.test_runner import set_coverage_checking from ietf.person.models import Person, PersonalApiKey, PersonApiKeyEvent from ietf.utils import log -def skip_coverage(f): - @wraps(f) - def _wrapper(*args, **kwargs): - if settings.TEST_CODE_COVERAGE_CHECKER: - set_coverage_checking(False) - result = f(*args, **kwargs) - set_coverage_checking(True) - return result - else: - return f(*args, **kwargs) - return _wrapper def person_required(f): @wraps(f) diff --git a/ietf/utils/jstest.py b/ietf/utils/jstest.py index 215d78d65fd..cf242fc4eb9 100644 --- a/ietf/utils/jstest.py +++ b/ietf/utils/jstest.py @@ -3,6 +3,8 @@ import os +from django.conf import settings +from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.urls import reverse as urlreverse from unittest import skipIf @@ -21,7 +23,11 @@ from ietf.utils.pipe import pipe -from ietf.utils.test_runner import IetfLiveServerTestCase +from ietf.utils.test_runner import ( + set_template_coverage, + set_url_coverage, + load_and_run_fixtures, +) executable_name = 'geckodriver' code, out, err = pipe('{} --version'.format(executable_name)) @@ -49,17 +55,44 @@ def ifSeleniumEnabled(func): return skipIf(skip_selenium, skip_message)(func) -class IetfSeleniumTestCase(IetfLiveServerTestCase): +class IetfSeleniumTestCase(StaticLiveServerTestCase): # pragma: no cover login_view = 'ietf.ietfauth.views.login' + @classmethod + def setUpClass(cls): + set_template_coverage(False) + set_url_coverage(False) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + set_template_coverage(True) + set_url_coverage(True) + def setUp(self): - super(IetfSeleniumTestCase, self).setUp() + super().setUp() + # LiveServerTestCase uses TransactionTestCase which seems to + # somehow interfere with the fixture loading process in + # IetfTestRunner when running multiple tests (the first test + # is fine, in the next ones the fixtures have been wiped) - + # this is no doubt solvable somehow, but until then we simply + # recreate them here + from ietf.person.models import Person + if not Person.objects.exists(): + load_and_run_fixtures(verbosity=0) + self.replaced_settings = dict() + if hasattr(settings, 'IDTRACKER_BASE_URL'): + self.replaced_settings['IDTRACKER_BASE_URL'] = settings.IDTRACKER_BASE_URL + settings.IDTRACKER_BASE_URL = self.live_server_url self.driver = start_web_driver() self.driver.set_window_size(1024,768) def tearDown(self): - super(IetfSeleniumTestCase, self).tearDown() self.driver.close() + for k, v in self.replaced_settings.items(): + setattr(settings, k, v) + super().tearDown() def absreverse(self,*args,**kwargs): return '%s%s'%(self.live_server_url, urlreverse(*args, **kwargs)) diff --git a/ietf/utils/meetecho.py b/ietf/utils/meetecho.py index 0dbf75736ad..7654f67cd15 100644 --- a/ietf/utils/meetecho.py +++ b/ietf/utils/meetecho.py @@ -27,7 +27,7 @@ class MeetechoAPI: - timezone = datetime.timezone.utc + timezone = datetime.UTC def __init__( self, api_base: str, client_id: str, client_secret: str, request_timeout=3.01 @@ -504,7 +504,7 @@ def _should_send_update(self, session): if self.slides_notify_time < datetime.timedelta(0): return True # < 0 means "always" for a scheduled session else: - now = datetime.datetime.now(tz=datetime.timezone.utc) + now = datetime.datetime.now(tz=datetime.UTC) return (timeslot.time - self.slides_notify_time) < now < (timeslot.end_time() + self.slides_notify_time) def add(self, session: "Session", slides: "Document", order: int): diff --git a/ietf/utils/serialize.py b/ietf/utils/serialize.py index 342d211cf5c..77f97942cbe 100644 --- a/ietf/utils/serialize.py +++ b/ietf/utils/serialize.py @@ -16,7 +16,7 @@ def object_as_shallow_dict(obj): if isinstance(f, models.ManyToManyField): v = list(v.values_list("pk", flat=True)) elif isinstance(f, models.DateTimeField): - v = v.astimezone(datetime.timezone.utc).isoformat() + v = v.astimezone(datetime.UTC).isoformat() elif isinstance(f, models.DateField): v = v.strftime('%Y-%m-%d') diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index a9b2e5d572e..1a3d4e5c3d1 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -48,6 +48,8 @@ import subprocess import tempfile import copy +from contextlib import contextmanager + import boto3 import botocore.config import factory.random @@ -57,10 +59,6 @@ from typing import Callable, Optional from urllib.parse import urlencode -from coverage.report import Reporter -from coverage.results import Numbers -from coverage.misc import NotPython - import django from django.conf import settings from django.contrib.staticfiles.testing import StaticLiveServerTestCase @@ -84,7 +82,7 @@ import ietf import ietf.utils.mail from ietf.utils.management.commands import pyflakes -from ietf.utils.test_smtpserver import SMTPTestServerDriver +from ietf.utils.aiosmtpd import SMTPTestServerDriver from ietf.utils.test_utils import TestCase from mypy_boto3_s3.service_resource import Bucket @@ -96,11 +94,11 @@ old_destroy: Optional[Callable] = None old_create: Optional[Callable] = None -template_coverage_collection = None -code_coverage_collection = None -url_coverage_collection = None +template_coverage_collection = False +url_coverage_collection = False validation_settings = {"validate_html": None, "validate_html_harder": None, "show_logging": False} + def start_vnu_server(port=8888): "Start a vnu validation server on the indicated port" vnu = subprocess.Popen( @@ -463,50 +461,29 @@ def save_test_results(failures, test_labels): tfile.write("%s OK\n" % (timestr, )) tfile.close() -def set_coverage_checking(flag=True): + +def set_template_coverage(flag): global template_coverage_collection - global code_coverage_collection + orig = template_coverage_collection + template_coverage_collection = flag + return orig + + +def set_url_coverage(flag): global url_coverage_collection - if settings.SERVER_MODE == 'test': - if flag: - settings.TEST_CODE_COVERAGE_CHECKER.collector.resume() - template_coverage_collection = True - code_coverage_collection = True - url_coverage_collection = True - else: - settings.TEST_CODE_COVERAGE_CHECKER.collector.pause() - template_coverage_collection = False - code_coverage_collection = False - url_coverage_collection = False - -class CoverageReporter(Reporter): - def report(self): - self.find_file_reporters(None) - - total = Numbers() - result = {"coverage": 0.0, "covered": {}, "format": 5, } - for fr in self.file_reporters: - try: - analysis = self.coverage._analyze(fr) - nums = analysis.numbers - missing_nums = sorted(analysis.missing) - with io.open(analysis.filename, encoding='utf-8') as file: - lines = file.read().splitlines() - missing_lines = [ lines[l-1] for l in missing_nums ] - result["covered"][fr.relative_filename()] = (nums.n_statements, nums.pc_covered/100.0, missing_nums, missing_lines) - total += nums - except KeyboardInterrupt: # pragma: not covered - raise - except Exception: - report_it = not self.config.ignore_errors - if report_it: - typ, msg = sys.exc_info()[:2] - if typ is NotPython and not fr.should_be_python(): - report_it = False - if report_it: - raise - result["coverage"] = total.pc_covered/100.0 - return result + orig = url_coverage_collection + url_coverage_collection = flag + return orig + + +@contextmanager +def disable_coverage(): + """Context manager/decorator that disables template/url coverage""" + orig_template = set_template_coverage(False) + orig_url = set_url_coverage(False) + yield + set_template_coverage(orig_template) + set_url_coverage(orig_url) class CoverageTest(unittest.TestCase): @@ -594,23 +571,24 @@ def ignore_pattern(regex, pattern): self.skipTest("Coverage switched off with --skip-coverage") def code_coverage_test(self): - if self.runner.check_coverage: - include = [ os.path.join(path, '*') for path in self.runner.test_paths ] - checker = self.runner.code_coverage_checker - checker.stop() + if ( + self.runner.check_coverage + and settings.TEST_CODE_COVERAGE_CHECKER is not None + ): + coverage_manager = settings.TEST_CODE_COVERAGE_CHECKER + coverage_manager.stop() # Save to the .coverage file - checker.save() + coverage_manager.save() # Apply the configured and requested omit and include data - checker.config.from_args(ignore_errors=None, omit=settings.TEST_CODE_COVERAGE_EXCLUDE_FILES, - include=include, file=None) - for pattern in settings.TEST_CODE_COVERAGE_EXCLUDE_LINES: - checker.exclude(pattern) # Maybe output an HTML report if self.runner.run_full_test_suite and self.runner.html_report: - checker.html_report(directory=settings.TEST_CODE_COVERAGE_REPORT_DIR) - # In any case, build a dictionary with per-file data for this run - reporter = CoverageReporter(checker, checker.config) - self.runner.coverage_data["code"] = reporter.report() + coverage_manager.checker.html_report( + directory=settings.TEST_CODE_COVERAGE_REPORT_DIR + ) + # Generate the output report data + self.runner.coverage_data["code"] = coverage_manager.report( + include=[str(pathlib.Path(p) / "*") for p in self.runner.test_paths] + ) self.report_test_result("code") else: self.skipTest("Coverage switched off with --skip-coverage") @@ -824,23 +802,12 @@ def setup_test_environment(self, **kwargs): "covered": {}, "format": 1, }, - "migration": { - "present": {}, - "format": 3, - } } settings.TEMPLATES[0]['OPTIONS']['loaders'] = ('ietf.utils.test_runner.TemplateCoverageLoader',) + settings.TEMPLATES[0]['OPTIONS']['loaders'] settings.MIDDLEWARE = ('ietf.utils.test_runner.record_urls_middleware',) + tuple(settings.MIDDLEWARE) - self.code_coverage_checker = settings.TEST_CODE_COVERAGE_CHECKER - if not self.code_coverage_checker._started: - sys.stderr.write(" ** Warning: In %s: Expected the coverage checker to have\n" - " been started already, but it wasn't. Doing so now. Coverage numbers\n" - " will be off, though.\n" % __name__) - self.code_coverage_checker.start() - if settings.SITE_ID != 1: print(" Changing SITE_ID to '1' during testing.") settings.SITE_ID = 1 @@ -1140,9 +1107,8 @@ def _extra_tests(self): ), ] if self.check_coverage: - global template_coverage_collection, code_coverage_collection, url_coverage_collection + global template_coverage_collection, url_coverage_collection template_coverage_collection = True - code_coverage_collection = True url_coverage_collection = True tests += [ PyFlakesTestCase(test_runner=self, methodName='pyflakes_test'), @@ -1226,37 +1192,6 @@ def run_tests(self, test_labels, extra_tests=None, **kwargs): return failures -class IetfLiveServerTestCase(StaticLiveServerTestCase): - @classmethod - def setUpClass(cls): - set_coverage_checking(False) - super(IetfLiveServerTestCase, cls).setUpClass() - - def setUp(self): - super(IetfLiveServerTestCase, self).setUp() - # LiveServerTestCase uses TransactionTestCase which seems to - # somehow interfere with the fixture loading process in - # IetfTestRunner when running multiple tests (the first test - # is fine, in the next ones the fixtures have been wiped) - - # this is no doubt solvable somehow, but until then we simply - # recreate them here - from ietf.person.models import Person - if not Person.objects.exists(): - load_and_run_fixtures(verbosity=0) - self.replaced_settings = dict() - if hasattr(settings, 'IDTRACKER_BASE_URL'): - self.replaced_settings['IDTRACKER_BASE_URL'] = settings.IDTRACKER_BASE_URL - settings.IDTRACKER_BASE_URL = self.live_server_url - - @classmethod - def tearDownClass(cls): - super(IetfLiveServerTestCase, cls).tearDownClass() - set_coverage_checking(True) - - def tearDown(self): - for k, v in self.replaced_settings.items(): - setattr(settings, k, v) - super().tearDown() class TestBlobstoreManager(): # N.B. buckets and blobstore are intentional Class-level attributes @@ -1267,7 +1202,11 @@ class TestBlobstoreManager(): aws_access_key_id="minio_root", aws_secret_access_key="minio_pass", aws_session_token=None, - config = botocore.config.Config(signature_version="s3v4"), + config = botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + signature_version="s3v4", + ), #config=botocore.config.Config(signature_version=botocore.UNSIGNED), verify=False ) diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 01433888fe8..3288309095c 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -54,7 +54,11 @@ decode_header_value, show_that_mail_was_sent, ) -from ietf.utils.test_runner import get_template_paths, set_coverage_checking +from ietf.utils.test_runner import ( + get_template_paths, + set_template_coverage, + set_url_coverage, +) from ietf.utils.test_utils import TestCase, unicontent from ietf.utils.text import parse_unicode from ietf.utils.timezone import timezone_not_near_midnight @@ -311,14 +315,15 @@ def qualified(name): return list(callbacks) -class TemplateChecksTestCase(TestCase): +class TemplateChecksTestCase(TestCase): # pragma: no cover paths = [] # type: List[str] templates = {} # type: Dict[str, Template] def setUp(self): super().setUp() - set_coverage_checking(False) + set_template_coverage(False) + set_url_coverage(False) self.paths = get_template_paths() # already filtered ignores self.paths.sort() for path in self.paths: @@ -328,7 +333,8 @@ def setUp(self): pass def tearDown(self): - set_coverage_checking(True) + set_template_coverage(True) + set_url_coverage(True) super().tearDown() def test_parse_templates(self): diff --git a/ietf/utils/tests_coverage.py b/ietf/utils/tests_coverage.py new file mode 100644 index 00000000000..68795994a7e --- /dev/null +++ b/ietf/utils/tests_coverage.py @@ -0,0 +1,56 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +"""Tests of the coverage.py module""" + +from unittest import mock + +from django.test import override_settings + +from .coverage import CoverageManager +from .test_utils import TestCase + + +class CoverageManagerTests(TestCase): + @override_settings( + BASE_DIR="/path/to/project/ietf", + TEST_CODE_COVERAGE_EXCLUDE_FILES=["a.py"], + TEST_CODE_COVERAGE_EXCLUDE_LINES=["some-regex"], + ) + @mock.patch("ietf.utils.coverage.Coverage") + def test_coverage_manager(self, mock_coverage): + """CoverageManager managed coverage correctly in non-production mode + + Presumes we're not running tests in production mode. + """ + cm = CoverageManager() + self.assertFalse(cm.started) + + cm.start() + self.assertTrue(cm.started) + self.assertEqual(cm.checker, mock_coverage.return_value) + self.assertTrue(mock_coverage.called) + coverage_kwargs = mock_coverage.call_args.kwargs + self.assertEqual(coverage_kwargs["source"], ["/path/to/project/ietf"]) + self.assertEqual(coverage_kwargs["omit"], ["a.py"]) + self.assertTrue(isinstance(cm.checker.exclude, mock.Mock)) + assert isinstance(cm.checker.exclude, mock.Mock) # for type checker + self.assertEqual(cm.checker.exclude.call_count, 1) + cm.checker.exclude.assert_called_with("some-regex") + + @mock.patch("ietf.utils.coverage.Coverage") + def test_coverage_manager_is_defanged_in_production(self, mock_coverage): + """CoverageManager is a no-op in production mode""" + # Be careful faking settings.SERVER_MODE, but there's really no other way to + # test this. + with override_settings(SERVER_MODE="production"): + cm = CoverageManager() + cm.start() + + # Check that nothing actually happened + self.assertFalse(mock_coverage.called) + self.assertIsNone(cm.checker) + self.assertFalse(cm.started) + + # Check that other methods are guarded appropriately + cm.stop() + cm.save() + self.assertIsNone(cm.report()) diff --git a/ietf/utils/tests_meetecho.py b/ietf/utils/tests_meetecho.py index a10ac68c27d..502e936483a 100644 --- a/ietf/utils/tests_meetecho.py +++ b/ietf/utils/tests_meetecho.py @@ -98,7 +98,7 @@ def test_schedule_meeting(self): api_response = api.schedule_meeting( wg_token='my-token', room_id=18, - start_time=datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.timezone.utc), + start_time=datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.UTC), duration=datetime.timedelta(minutes=130), description='interim-2021-wgname-01', extrainfo='message for staff', @@ -127,7 +127,7 @@ def test_schedule_meeting(self): ) # same time in different time zones for start_time in [ - datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.timezone.utc), + datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.UTC), datetime.datetime(2021, 9, 14, 7, 0, 0, tzinfo=ZoneInfo('America/Halifax')), datetime.datetime(2021, 9, 14, 13, 0, 0, tzinfo=ZoneInfo('Europe/Kiev')), datetime.datetime(2021, 9, 14, 5, 0, 0, tzinfo=ZoneInfo('Pacific/Easter')), @@ -198,7 +198,7 @@ def test_fetch_meetings(self): '3d55bce0-535e-4ba8-bb8e-734911cf3c32': { 'room': { 'id': 18, - 'start_time': datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.timezone.utc), + 'start_time': datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.UTC), 'duration': datetime.timedelta(minutes=130), 'description': 'interim-2021-wgname-01', }, @@ -208,7 +208,7 @@ def test_fetch_meetings(self): 'e68e96d4-d38f-475b-9073-ecab46ca96a5': { 'room': { 'id': 23, - 'start_time': datetime.datetime(2021, 9, 15, 14, 30, 0, tzinfo=datetime.timezone.utc), + 'start_time': datetime.datetime(2021, 9, 15, 14, 30, 0, tzinfo=datetime.UTC), 'duration': datetime.timedelta(minutes=30), 'description': 'interim-2021-wgname-02', }, @@ -386,7 +386,7 @@ def test_request_helper_exception(self): def test_time_serialization(self): """Time de/serialization should be consistent""" - time = timezone.now().astimezone(datetime.timezone.utc).replace(microsecond=0) # cut off to 0 microseconds + time = timezone.now().astimezone(datetime.UTC).replace(microsecond=0) # cut off to 0 microseconds api = MeetechoAPI(API_BASE, CLIENT_ID, CLIENT_SECRET) self.assertEqual(api._deserialize_time(api._serialize_time(time)), time) @@ -400,7 +400,7 @@ def test_conference_from_api_dict(self): 'session-1-uuid': { 'room': { 'id': 1, - 'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc), + 'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC), 'duration': datetime.timedelta(minutes=45), 'description': 'some-description', }, @@ -410,7 +410,7 @@ def test_conference_from_api_dict(self): 'session-2-uuid': { 'room': { 'id': 2, - 'start_time': datetime.datetime(2022,2,5,4,5,6, tzinfo=datetime.timezone.utc), + 'start_time': datetime.datetime(2022,2,5,4,5,6, tzinfo=datetime.UTC), 'duration': datetime.timedelta(minutes=90), 'description': 'another-description', }, @@ -427,7 +427,7 @@ def test_conference_from_api_dict(self): id=1, public_id='session-1-uuid', description='some-description', - start_time=datetime.datetime(2022, 2, 4, 1, 2, 3, tzinfo=datetime.timezone.utc), + start_time=datetime.datetime(2022, 2, 4, 1, 2, 3, tzinfo=datetime.UTC), duration=datetime.timedelta(minutes=45), url='https://example.com/some/url', deletion_token='delete-me', @@ -437,7 +437,7 @@ def test_conference_from_api_dict(self): id=2, public_id='session-2-uuid', description='another-description', - start_time=datetime.datetime(2022, 2, 5, 4, 5, 6, tzinfo=datetime.timezone.utc), + start_time=datetime.datetime(2022, 2, 5, 4, 5, 6, tzinfo=datetime.UTC), duration=datetime.timedelta(minutes=90), url='https://example.com/another/url', deletion_token='delete-me-too', @@ -453,7 +453,7 @@ def test_fetch(self, mock_fetch, _): 'session-1-uuid': { 'room': { 'id': 1, - 'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc), + 'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC), 'duration': datetime.timedelta(minutes=45), 'description': 'some-description', }, @@ -472,7 +472,7 @@ def test_fetch(self, mock_fetch, _): id=1, public_id='session-1-uuid', description='some-description', - start_time=datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc), + start_time=datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC), duration=datetime.timedelta(minutes=45), url='https://example.com/some/url', deletion_token='delete-me', @@ -488,7 +488,7 @@ def test_create(self, mock_schedule, _): 'session-1-uuid': { 'room': { 'id': 1, # value should match session_id param to cm.create() below - 'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc), + 'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC), 'duration': datetime.timedelta(minutes=45), 'description': 'some-description', }, @@ -506,7 +506,7 @@ def test_create(self, mock_schedule, _): id=1, public_id='session-1-uuid', description='some-description', - start_time=datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc), + start_time=datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC), duration=datetime.timedelta(minutes=45), url='https://example.com/some/url', deletion_token='delete-me', diff --git a/ietf/utils/timezone.py b/ietf/utils/timezone.py index a396b5e82d9..e08dfa02f2c 100644 --- a/ietf/utils/timezone.py +++ b/ietf/utils/timezone.py @@ -26,7 +26,7 @@ def _tzinfo(tz: Union[str, datetime.tzinfo, None]): Accepts a tzinfo or string containing a timezone name. Defaults to UTC if tz is None. """ if tz is None: - return datetime.timezone.utc + return datetime.UTC elif isinstance(tz, datetime.tzinfo): return tz else: diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 482a4b110a3..c1436e158b6 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -280,7 +280,9 @@ def _multiline_to_list(s): PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME # Normally only set for debug, but needed until we have a real FS -DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, "static/dist-neue/manifest.json") +DJANGO_VITE["default"]["manifest_path"] = os.path.join( + BASE_DIR, "static/dist-neue/manifest.json" +) # Binaries that are different in the docker image DE_GFM_BINARY = "/usr/local/bin/de-gfm" @@ -379,6 +381,8 @@ def _multiline_to_list(s): secret_key=_blob_store_secret_key, security_token=None, client_config=botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", signature_version="s3v4", connect_timeout=_blob_store_connect_timeout, read_timeout=_blob_store_read_timeout, diff --git a/patch/tastypie-django22-fielderror-response.patch b/patch/tastypie-django22-fielderror-response.patch index ffb152d3195..3b4418fc66b 100644 --- a/patch/tastypie-django22-fielderror-response.patch +++ b/patch/tastypie-django22-fielderror-response.patch @@ -1,5 +1,5 @@ ---- tastypie/resources.py.orig 2020-08-24 13:14:25.463166100 +0200 -+++ tastypie/resources.py 2020-08-24 13:15:55.133759224 +0200 +--- tastypie/resources.py.orig 2025-07-29 19:00:01.526948002 +0000 ++++ tastypie/resources.py 2025-07-29 19:07:15.324127008 +0000 @@ -12,7 +12,7 @@ ObjectDoesNotExist, MultipleObjectsReturned, ValidationError, FieldDoesNotExist ) @@ -9,13 +9,13 @@ from django.db.models.fields.related import ForeignKey from django.urls.conf import re_path from tastypie.utils.timezone import make_naive_utc -@@ -2198,6 +2198,8 @@ +@@ -2216,6 +2216,8 @@ return self.authorized_read_list(objects, bundle) except ValueError: raise BadRequest("Invalid resource lookup data provided (mismatched type).") + except FieldError as e: + raise BadRequest("Invalid resource lookup: %s." % e) - + def obj_get(self, bundle, **kwargs): """ --- tastypie/paginator.py.orig 2020-08-25 15:24:46.391588425 +0200 diff --git a/requirements.txt b/requirements.txt index 60d3d8152e6..cf7c920fa3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,86 +1,85 @@ # -*- conf-mode -*- -setuptools>=51.1.0 # Require this first, to prevent later errors +setuptools>=80.9.0 # Require this first, to prevent later errors # aiosmtpd>=1.4.6 -argon2-cffi>=21.3.0 # For the Argon2 password hasher option -beautifulsoup4>=4.11.1 # Only used in tests -bibtexparser>=1.2.0 # Only used in tests -bleach>=6 -types-bleach>=6 -boto3>=1.35,<1.36 -boto3-stubs[s3]>=1.35,<1.36 -botocore>=1.35,<1.36 -celery>=5.2.6 -coverage>=4.5.4,<5.0 # Coverage 5.x moves from a json database to SQLite. Moving to 5.x will require substantial rewrites in ietf.utils.test_runner and ietf.release.views +argon2-cffi>=25.1.0 # For the Argon2 password hasher option +beautifulsoup4>=4.13.4 # Only used in tests +bibtexparser>=1.4.3 # Only used in tests +bleach>=6.2.0 # project is deprecated but supported +types-bleach>=6.2.0 +boto3>=1.39.15 +boto3-stubs[s3]>=1.39.15 +botocore>=1.39.15 +celery>=5.5.3 +coverage>=7.9.2 defusedxml>=0.7.1 # for TastyPie when using xml; not a declared dependency Django>4.2,<5 -django-admin-rangefilter>=0.13.2 -django-analytical>=3.1.0 -django-bootstrap5>=21.3 -django-celery-beat>=2.3.0,<2.8.0 # pin until https://github.com/celery/django-celery-beat/issues/875 is resolved, then revisit -django-celery-results>=2.5.1 -django-csp>=3.7 -django-cors-headers>=3.11.0 -django-debug-toolbar>=3.2.4 -django-markup>=1.5 # Limited use - need to reconcile against direct use of markdown +django-admin-rangefilter>=0.13.3 +django-analytical>=3.2.0 +django-bootstrap5>=25.1 +django-celery-beat>=2.7.0,<2.8.0 # pin until https://github.com/celery/django-celery-beat/issues/875 is resolved, then revisit +django-celery-results>=2.6.0 +django-cors-headers>=4.7.0 +django-debug-toolbar>=6.0.0 +django-markup>=1.10 # Limited use - need to reconcile against direct use of markdown django-oidc-provider==0.8.2 # 0.8.3 changes logout flow and claim return -django-referrer-policy>=1.0 -django-simple-history>=3.0.0 -django-storages>=1.14.4 +django-simple-history>=3.10.1 +django-storages>=1.14.6 django-stubs>=4.2.7,<5 # The django-stubs version used determines the the mypy version indicated below -django-tastypie>=0.14.7,<0.15.0 # Version must be locked in sync with version of Django -django-vite>=2.0.2,<3 +django-tastypie>=0.15.1 # Version must be kept in sync with Django +django-vite>=3.1.0 django-widget-tweaks>=1.4.12 -djangorestframework>=3.15,<4 -djlint>=1.0.0 # To auto-indent templates via "djlint --profile django --reformat" -docutils>=0.18.1 # Used only by dbtemplates for RestructuredText +djangorestframework>=3.16.0 +docutils>=0.22.0 # Used only by dbtemplates for RestructuredText +types-docutils>=0.21.0 # should match docutils (0.22.0 not out yet) drf-spectacular>=0.27 -drf-standardized-errors[openapi] >= 0.14 -types-docutils>=0.18.1 -factory-boy>=3.3 -gunicorn>=20.1.0 +drf-standardized-errors[openapi] >= 0.15.0 +factory-boy>=3.3.3 +gunicorn>=23.0.0 hashids>=1.3.1 -html2text>=2020.1.16 # Used only to clean comment field of secr/sreq +html2text>=2025.4.15 # Used only to clean comment field of secr/sreq html5lib>=1.1 # Only used in tests icalendar>=5.0.0 -inflect>= 6.0.2 -jsonfield>=3.1.0,<3.2.0 # 3.2.0 needs py3.10; deprecated-replace with Django JSONField -jsonschema[format]>=4.2.1 -jwcrypto>=1.2 # for signed notifications - this is aspirational, and is not really used. -logging_tree>=1.9 # Used only by the showloggers management command -lxml>=5.3.0 -markdown>=3.3.6 -types-markdown>=3.3.6 -mypy~=1.7.0 # Version requirements determined by django-stubs. -oic>=1.3 # Used only by tests -Pillow>=9.1.0 -psycopg2>=2.9.6 -pyang>=2.5.3 -pydyf>0.8.0 -pyflakes>=2.4.0 -pyopenssl>=22.0.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency -pyquery>=1.4.3 -python-dateutil>=2.8.2 -types-python-dateutil>=2.8.2 -python-json-logger>=3.1.0 +inflect>= 7.5.0 +jsonfield>=3.2.0 # deprecated - need to replace with Django's JSONField +jsonschema[format]>=4.25.0 +jwcrypto>=1.5.6 # for signed notifications - this is aspirational, and is not really used. +logging_tree>=1.10 # Used only by the showloggers management command +lxml>=6.0.0 +markdown>=3.8.0 +types-markdown>=3.8.0 +mock>=5.2.0 # should replace with unittest.mock and remove dependency +types-mock>=5.2.0 +mypy~=1.7.0 # Version requirements determined by django-stubs. +oic>=1.7.0 # Used only by tests +pillow>=11.3.0 +psycopg2>=2.9.10 +pyang>=2.6.1 +pydyf>=0.11.0 +pyflakes>=3.4.0 +pyopenssl>=25.1.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency +pyquery>=2.0.1 +python-dateutil>=2.9.0 +types-python-dateutil>=2.9.0 +python-json-logger>=3.3.0 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache -python-mimeparse>=1.6 # from TastyPie +python-mimeparse>=2.0.0 # from TastyPie pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields -types-pytz==2025.2.0.20250809 # match pytz versionrequests>=2.31.0 -requests>=2.31.0 -types-requests>=2.27.1 -requests-mock>=1.9.3 +types-pytz==2025.2.0.20250809 # match pytz version +requests>=2.32.4 +types-requests>=2.32.4 +requests-mock>=1.12.1 rfc2html>=2.0.3 -scout-apm>=2.24.2 -selenium>=4.0 -tblib>=1.7.0 # So that the django test runner provides tracebacks -tlds>=2022042700 # Used to teach bleach about which TLDs currently exist -tqdm>=4.64.0 -types-zxcvbn~=4.5.0.20250223 # match zxcvbn version -Unidecode>=1.3.4 -urllib3>=1.26,<2 -weasyprint>=64.1 -xml2rfc>=3.23.0 +scout-apm>=3.4.0 +selenium>=4.34.2 +tblib>=3.1.0 # So that the django test runner provides tracebacks +tlds>=2022042700 # Used to teach bleach about which TLDs currently exist +tqdm>=4.67.1 +unidecode>=1.4.0 +urllib3>=2.5.0 +weasyprint>=66.0 +xml2rfc>=3.30.0 xym>=0.6,<1.0 zxcvbn>=4.5.0 +types-zxcvbn~=4.5.0.20250223 # match zxcvbn version From b14512e840d8dfccf4e418ac184c77321595278b Mon Sep 17 00:00:00 2001 From: rjsparks <10996692+rjsparks@users.noreply.github.com> Date: Wed, 3 Sep 2025 22:29:19 +0000 Subject: [PATCH 019/313] ci: update base image target version to 20250903T2216 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 658f1e56950..d3b186e1f58 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250819T1645 +FROM ghcr.io/ietf-tools/datatracker-app-base:20250903T2216 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 9e510ad8dbd..9d8427efdb3 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250819T1645 +20250903T2216 From e444d9e73c78a1100ad5b909f2b15012be287889 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 3 Sep 2025 20:55:17 -0300 Subject: [PATCH 020/313] chore: use :latest instead of :py312 (#9460) --- .github/workflows/tests-az.yml | 2 +- dev/deploy-to-container/cli.js | 6 +++--- dev/diff/cli.js | 6 +++--- dev/tests/debug.sh | 2 +- dev/tests/docker-compose.debug.yml | 2 +- docker/app.Dockerfile | 2 +- docker/celery.Dockerfile | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml index d1fe0cdf623..8553563a192 100644 --- a/.github/workflows/tests-az.yml +++ b/.github/workflows/tests-az.yml @@ -62,7 +62,7 @@ jobs: echo "Starting Containers..." sudo docker network create dtnet sudo docker run -d --name db --network=dtnet ghcr.io/ietf-tools/datatracker-db:latest & - sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:py312 sleep infinity & + sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:latest sleep infinity & wait echo "Cloning datatracker repo..." diff --git a/dev/deploy-to-container/cli.js b/dev/deploy-to-container/cli.js index 2f0faad151d..1a2d993ac4b 100644 --- a/dev/deploy-to-container/cli.js +++ b/dev/deploy-to-container/cli.js @@ -85,7 +85,7 @@ async function main () { // Pull latest Datatracker Base image console.info('Pulling latest Datatracker base docker image...') - const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:py312') + const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:latest') await new Promise((resolve, reject) => { dock.modem.followProgress(appImagePullStream, (err, res) => err ? reject(err) : resolve(res)) }) @@ -214,7 +214,7 @@ async function main () { const celeryContainers = {} for (const conConf of conConfs) { celeryContainers[conConf.name] = await dock.createContainer({ - Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312', + Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest', name: `dt-${conConf.name}-${branch}`, Hostname: `dt-${conConf.name}-${branch}`, Env: [ @@ -244,7 +244,7 @@ async function main () { // Create Datatracker container console.info(`Creating Datatracker docker container... [dt-app-${branch}]`) const appContainer = await dock.createContainer({ - Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312', + Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest', name: `dt-app-${branch}`, Hostname: `dt-app-${branch}`, Env: [ diff --git a/dev/diff/cli.js b/dev/diff/cli.js index 0cf353cc651..461b0c37a0c 100644 --- a/dev/diff/cli.js +++ b/dev/diff/cli.js @@ -567,7 +567,7 @@ async function main () { { title: 'Pulling latest Datatracker base docker image...', task: async (subctx, subtask) => { - const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:py312') + const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:latest') await new Promise((resolve, reject) => { dock.modem.followProgress(appImagePullStream, (err, res) => err ? reject(err) : resolve(res)) }) @@ -648,7 +648,7 @@ async function main () { title: 'Creating source Datatracker docker container...', task: async (subctx, subtask) => { containers.appSource = await dock.createContainer({ - Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312', + Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest', name: 'dt-diff-app-source', Tty: true, Hostname: 'appsource', @@ -664,7 +664,7 @@ async function main () { title: 'Creating target Datatracker docker container...', task: async (subctx, subtask) => { containers.appTarget = await dock.createContainer({ - Image: 'ghcr.io/ietf-tools/datatracker-app-base:py312', + Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest', name: 'dt-diff-app-target', Tty: true, Hostname: 'apptarget', diff --git a/dev/tests/debug.sh b/dev/tests/debug.sh index e92e6d9b2af..d87c504bb91 100644 --- a/dev/tests/debug.sh +++ b/dev/tests/debug.sh @@ -9,7 +9,7 @@ # Simply type "exit" + ENTER to exit and shutdown this test environment. echo "Fetching latest images..." -docker pull ghcr.io/ietf-tools/datatracker-app-base:py312 +docker pull ghcr.io/ietf-tools/datatracker-app-base:latest docker pull ghcr.io/ietf-tools/datatracker-db:latest echo "Starting containers..." docker compose -f docker-compose.debug.yml -p dtdebug --compatibility up -d diff --git a/dev/tests/docker-compose.debug.yml b/dev/tests/docker-compose.debug.yml index 168bbd4e92b..8117b923751 100644 --- a/dev/tests/docker-compose.debug.yml +++ b/dev/tests/docker-compose.debug.yml @@ -5,7 +5,7 @@ version: '3.8' services: app: - image: ghcr.io/ietf-tools/datatracker-app-base:py312 + image: ghcr.io/ietf-tools/datatracker-app-base:latest command: -f /dev/null working_dir: /__w/datatracker/datatracker entrypoint: tail diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index e3df9bd4b48..fee38337330 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:py312 +FROM ghcr.io/ietf-tools/datatracker-app-base:latest LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/docker/celery.Dockerfile b/docker/celery.Dockerfile index 279d5c7550c..e7c7b9cc3fd 100644 --- a/docker/celery.Dockerfile +++ b/docker/celery.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:py312 +FROM ghcr.io/ietf-tools/datatracker-app-base:latest LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive From c4d69d0118a068c873dc066fe9adde829e86f14e Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 5 Sep 2025 17:22:52 -0500 Subject: [PATCH 021/313] feat: links to postorious (#9470) * feat: links to postorious * fix: remove redundant divider * chore: better use of whitespace * chore: remove what the cat typed in * chore: more stray removal --- ietf/templates/base/menu_user.html | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/ietf/templates/base/menu_user.html b/ietf/templates/base/menu_user.html index 9a0bf568384..fd921638a45 100644 --- a/ietf/templates/base/menu_user.html +++ b/ietf/templates/base/menu_user.html @@ -115,6 +115,37 @@ {% endif %} +
  • + + List subscriptions + + +
    • {% if user|has_role:"Reviewer" %}
  • Date: Tue, 16 Sep 2025 17:55:07 -0300 Subject: [PATCH 022/313] chore: hide weasyprint internal deprecation warning (#9544) --- ietf/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ietf/settings.py b/ietf/settings.py index 753508dc993..d6be1d1e0f5 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -34,6 +34,7 @@ warnings.filterwarnings("ignore", message="datetime.datetime.utcnow\\(\\) is deprecated", module="oic.utils.time_util") warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="oic.utils.time_util") warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="pytz.tzinfo") +warnings.filterwarnings("ignore", message="'instantiateVariableFont' is deprecated", module="weasyprint") base_path = pathlib.Path(__file__).resolve().parent From c71871855769d9c2980cad853cf92a9ec25cb50a Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 16 Sep 2025 15:55:45 -0500 Subject: [PATCH 023/313] fix: normalize 3gpp groups and resolve duplication (#9505) * fix: don't bother the rfc-editor with group type sdo name changes * fix: normalize 3gpp groups and resolve duplication * fix: improve guard, update t2 * fix: exclude the task from test coverage * fix: exclude harder * fix: tweak the pragma --- ietf/group/models.py | 2 + ietf/group/tasks.py | 121 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/ietf/group/models.py b/ietf/group/models.py index 608dcc86b99..2d5e7c4e6f9 100644 --- a/ietf/group/models.py +++ b/ietf/group/models.py @@ -491,6 +491,8 @@ def notify_rfceditor_of_group_name_change(sender, instance=None, **kwargs): current = Group.objects.get(pk=instance.pk) except Group.DoesNotExist: return + if current.type_id == "sdo": + return addr = settings.RFC_EDITOR_GROUP_NOTIFICATION_EMAIL if addr and instance.name != current.name: msg = """ diff --git a/ietf/group/tasks.py b/ietf/group/tasks.py index 693aafb3853..ada83e80e25 100644 --- a/ietf/group/tasks.py +++ b/ietf/group/tasks.py @@ -9,12 +9,15 @@ from django.conf import settings from django.template.loader import render_to_string +from django.utils import timezone from ietf.doc.storage_utils import store_file +from ietf.liaisons.models import LiaisonStatement from ietf.utils import log +from ietf.utils.test_runner import disable_coverage -from .models import Group -from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles +from .models import Group, GroupHistory +from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles, save_group_in_history from .views import extract_last_name, roles @@ -113,3 +116,117 @@ def generate_wg_summary_files_task(): store_file("indexes", "1wg-summary.txt", f, allow_overwrite=True) with summary_by_acronym_file.open("rb") as f: store_file("indexes", "1wg-summary-by-acronym.txt", f, allow_overwrite=True) + +@shared_task +@disable_coverage() +def run_once_adjust_liaison_groups(): # pragma: no cover + log.log("Starting run_once_adjust_liaison_groups") + if all( + [ + Group.objects.filter( + acronym__in=[ + "3gpp-tsg-ct", + "3gpp-tsg-ran-wg1", + "3gpp-tsg-ran-wg4", + "3gpp-tsg-sa", + "3gpp-tsg-sa-wg5", + "3gpp-tsgct", # duplicates 3gpp-tsg-ct above already + "3gpp-tsgct-ct1", # will normalize all acronyms to hyphenated form + "3gpp-tsgct-ct3", # and consistently match the name + "3gpp-tsgct-ct4", # (particularly use of WG) + "3gpp-tsgran", + "3gpp-tsgran-ran2", + "3gpp-tsgsa", # duplicates 3gpp-tsg-sa above + "3gpp-tsgsa-sa2", # will normalize + "3gpp-tsgsa-sa3", + "3gpp-tsgsa-sa4", + "3gpp-tsgt-wg2", + ] + ).count() + == 16, + not Group.objects.filter( + acronym__in=[ + "3gpp-tsg-ran-wg3", + "3gpp-tsg-ct-wg1", + "3gpp-tsg-ct-wg3", + "3gpp-tsg-ct-wg4", + "3gpp-tsg-ran", + "3gpp-tsg-ran-wg2", + "3gpp-tsg-sa-wg2", + "3gpp-tsg-sa-wg3", + "3gpp-tsg-sa-wg4", + "3gpp-tsg-t-wg2", + ] + ).exists(), + Group.objects.filter(acronym="o3gpptsgran3").exists(), + not LiaisonStatement.objects.filter( + to_groups__acronym__in=["3gpp-tsgct", "3gpp-tsgsa"] + ).exists(), + not LiaisonStatement.objects.filter( + from_groups__acronym="3gpp-tsgct" + ).exists(), + LiaisonStatement.objects.filter(from_groups__acronym="3gpp-tsgsa").count() + == 1, + LiaisonStatement.objects.get(from_groups__acronym="3gpp-tsgsa").pk == 1448, + ] + ): + for old_acronym, new_acronym, new_name in ( + ("o3gpptsgran3", "3gpp-tsg-ran-wg3", "3GPP TSG RAN WG3"), + ("3gpp-tsgct-ct1", "3gpp-tsg-ct-wg1", "3GPP TSG CT WG1"), + ("3gpp-tsgct-ct3", "3gpp-tsg-ct-wg3", "3GPP TSG CT WG3"), + ("3gpp-tsgct-ct4", "3gpp-tsg-ct-wg4", "3GPP TSG CT WG4"), + ("3gpp-tsgran", "3gpp-tsg-ran", "3GPP TSG RAN"), + ("3gpp-tsgran-ran2", "3gpp-tsg-ran-wg2", "3GPP TSG RAN WG2"), + ("3gpp-tsgsa-sa2", "3gpp-tsg-sa-wg2", "3GPP TSG SA WG2"), + ("3gpp-tsgsa-sa3", "3gpp-tsg-sa-wg3", "3GPP TSG SA WG3"), + ("3gpp-tsgsa-sa4", "3gpp-tsg-sa-wg4", "3GPP TSG SA WG4"), + ("3gpp-tsgt-wg2", "3gpp-tsg-t-wg2", "3GPP TSG T WG2"), + ): + group = Group.objects.get(acronym=old_acronym) + save_group_in_history(group) + group.time = timezone.now() + group.acronym = new_acronym + group.name = new_name + if old_acronym.startswith("3gpp-tsgct-"): + group.parent = Group.objects.get(acronym="3gpp-tsg-ct") + elif old_acronym.startswith("3gpp-tsgsa-"): + group.parent = Group.objects.get(acronym="3gpp-tsg-sa") + group.save() + group.groupevent_set.create( + time=group.time, + by_id=1, # (System) + type="info_changed", + desc=f"acronym changed from {old_acronym} to {new_acronym}, name set to {new_name}", + ) + + for acronym, new_name in (("3gpp-tsg-ct", "3GPP TSG CT"),): + group = Group.objects.get(acronym=acronym) + save_group_in_history(group) + group.time = timezone.now() + group.name = new_name + group.save() + group.groupevent_set.create( + time=group.time, + by_id=1, # (System) + type="info_changed", + desc=f"name set to {new_name}", + ) + + ls = LiaisonStatement.objects.get(pk=1448) + ls.from_groups.remove(Group.objects.get(acronym="3gpp-tsgsa")) + ls.from_groups.add(Group.objects.get(acronym="3gpp-tsg-sa")) + + # Rewriting history to effectively merge the histories of the duplicate groups + GroupHistory.objects.filter(parent__acronym="3gpp-tsgsa").update( + parent=Group.objects.get(acronym="3gpp-tsg-sa") + ) + GroupHistory.objects.filter(parent__acronym="3gpp-tsgct").update( + parent=Group.objects.get(acronym="3gpp-tsg-ct") + ) + + deleted = Group.objects.filter( + acronym__in=["3gpp-tsgsa", "3gpp-tsgct"] + ).delete() + log.log(f"Deleted Groups: {deleted}") + else: + log.log("* Refusing to continue as preconditions have changed") From 0a1705193dfde6695921191540049b88d91d9ec9 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 17 Sep 2025 12:45:32 -0500 Subject: [PATCH 024/313] fix: update draft-stream-ietf state descriptions (#9543) --- .../0026_change_wg_state_descriptions.py | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 ietf/doc/migrations/0026_change_wg_state_descriptions.py diff --git a/ietf/doc/migrations/0026_change_wg_state_descriptions.py b/ietf/doc/migrations/0026_change_wg_state_descriptions.py new file mode 100644 index 00000000000..b02b12c97e4 --- /dev/null +++ b/ietf/doc/migrations/0026_change_wg_state_descriptions.py @@ -0,0 +1,117 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations + +def forward(apps, schema_editor): + State = apps.get_model("doc","State") + for name, desc in [ + ("WG Document","The document has been adopted by the Working Group (WG) and is under development. A document can only be adopted by one WG at a time. However, a document may be transferred between WGs."), + ("Parked WG Document","The Working Group (WG) document is in a temporary state where it will not be actively developed. The reason for the pause is explained via a datatracker comments section."), + ("Dead WG Document","The Working Group (WG) document has been abandoned by the WG. No further development is planned in this WG. A decision to resume work on this document and move it out of this state is possible."), + ("In WG Last Call","The Working Group (WG) document is currently subject to an active WG Last Call (WGLC) review per Section 7.4 of RFC2418."), + ("Waiting for Implementation","The progression of this Working Group (WG) document towards publication is paused as it awaits implementation. The process governing the approach to implementations is WG-specific."), + ("Held by WG","Held by Working Group (WG) chairs for administrative reasons. See document history for details."), + ("Waiting for WG Chair Go-Ahead","The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed"), + ("WG Consensus: Waiting for Write-Up","The Working Group (WG) document has consensus to proceed to publication. However, the document is waiting for a document shepherd write-up per RFC4858."), + ("Submitted to IESG for Publication","The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication. See the “IESG State” or “RFC Editor State” for further details on the state of the document."), + ("Candidate for WG Adoption","The individual submission document has been marked by the Working Group (WG) chairs as a candidate for adoption by the WG, but no adoption call has been started."), + ("Call For Adoption By WG Issued","A call for adoption of the individual submission document has been issued by the Working Group (WG) chairs. This call is still running but the WG has not yet reached consensus for adoption."), + ("Adopted by a WG","The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted."), + ("Adopted for WG Info Only","The document is adopted by the Working Group (WG) for its internal use. The WG has decided that it will not pursue publication of it as an RFC."), + ]: + State.objects.filter(name=name).update(desc=desc) + +def reverse(apps, schema_editor): + State = apps.get_model("doc","State") + for name, desc in [ + ("WG Document","""4.2.4. WG Document + + The "WG Document" state describes an I-D that has been adopted by an IETF WG and is being actively developed. + + A WG Chair may transition an I-D into the "WG Document" state at any time as long as the I-D is not being considered or developed in any other WG. + + Alternatively, WG Chairs may rely upon new functionality to be added to the Datatracker to automatically move version-00 drafts into the "WG Document" state as described in Section 4.1. + + Under normal conditions, it should not be possible for an I-D to be in the "WG Document" state in more than one WG at a time. This said, I-Ds may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs."""), + ("Parked WG Document","""4.2.5. Parked WG Document + + A "Parked WG Document" is an I-D that has lost its author or editor, is waiting for another document to be written or for a review to be completed, or cannot be progressed by the working group for some other reason. + + Some of the annotation tags described in Section 4.3 may be used in conjunction with this state to indicate why an I-D has been parked, and/or what may need to happen for the I-D to be un-parked. + + Parking a WG draft will not prevent it from expiring; however, this state can be used to indicate why the I-D has stopped progressing in the WG. + + A "Parked WG Document" that is not expired may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs."""), + ("Dead WG Document","""4.2.6. Dead WG Document + + A "Dead WG Document" is an I-D that has been abandoned. Note that 'Dead' is not always a final state for a WG I-D. If consensus is subsequently achieved, a "Dead WG Document" may be resurrected. A "Dead WG Document" that is not resurrected will eventually expire. + + Note that an I-D that is declared to be "Dead" in one WG and that is not expired may be transferred to a non-dead state in another WG with the consent of the WG Chairs and the responsible ADs."""), + ("In WG Last Call","""4.2.7. In WG Last Call + + A document "In WG Last Call" is an I-D for which a WG Last Call (WGLC) has been issued and is in progress. + + Note that conducting a WGLC is an optional part of the IETF WG process, per Section 7.4 of RFC 2418 [RFC2418]. + + If a WG Chair decides to conduct a WGLC on an I-D, the "In WG Last Call" state can be used to track the progress of the WGLC. The Chair may configure the Datatracker to send a WGLC message to one or more mailing lists when the Chair moves the I-D into this state. The WG Chair may also be able to select a different set of mailing lists for a different document undergoing a WGLC; some documents may deserve coordination with other WGs. + + A WG I-D in this state should remain "In WG Last Call" until the WG Chair moves it to another state. The WG Chair may configure the Datatracker to send an e-mail after a specified period of time to remind or 'nudge' the Chair to conclude the WGLC and to determine the next state for the document. + + It is possible for one WGLC to lead into another WGLC for the same document. For example, an I-D that completed a WGLC as an "Informational" document may need another WGLC if a decision is taken to convert the I-D into a Standards Track document."""), + ("Waiting for Implementation","""In some areas, it can be desirable to wait for multiple interoperable implementations before progressing a draft to be an RFC, and in some WGs this is required. This state should be entered after WG Last Call has completed."""), + ("Held by WG","""Held by WG, see document history for details."""), + ("Waiting for WG Chair Go-Ahead","""4.2.8. Waiting for WG Chair Go-Ahead + + A WG Chair may wish to place an I-D that receives a lot of comments during a WGLC into the "Waiting for WG Chair Go-Ahead" state. This state describes an I-D that has undergone a WGLC; however, the Chair is not yet ready to call consensus on the document. + + If comments from the WGLC need to be responded to, or a revision to the I-D is needed, the Chair may place an I-D into this state until all of the WGLC comments are adequately addressed and the (possibly revised) document is in the I-D repository."""), + ("WG Consensus: Waiting for Write-Up","""4.2.9. WG Consensus: Waiting for Writeup + + A document in the "WG Consensus: Waiting for Writeup" state has essentially completed its development within the working group, and is nearly ready to be sent to the IESG for publication. The last thing to be done is the preparation of a protocol writeup by a Document Shepherd. The IESG requires that a document shepherd writeup be completed before publication of the I-D is requested. The IETF document shepherding process and the role of a WG Document Shepherd is described in RFC 4858 [RFC4858] + + A WG Chair may call consensus on an I-D without a formal WGLC and transition an I-D that was in the "WG Document" state directly into this state. + + The name of this state includes the words "Waiting for Writeup" because a good document shepherd writeup takes time to prepare."""), + ("Submitted to IESG for Publication","""4.2.10. Submitted to IESG for Publication + + This state describes a WG document that has been submitted to the IESG for publication and that has not been sent back to the working group for revision. + + An I-D in this state may be under review by the IESG, it may have been approved and be in the RFC Editor's queue, or it may have been published as an RFC. Other possibilities exist too. The document may be "Dead" (in the IESG state machine) or in a "Do Not Publish" state."""), + ("Candidate for WG Adoption","""The document has been marked as a candidate for WG adoption by the WG Chair. This state can be used before a call for adoption is issued (and the document is put in the "Call For Adoption By WG Issued" state), to indicate that the document is in the queue for a call for adoption, even if none has been issued yet."""), + ("Call For Adoption By WG Issued","""4.2.1. Call for Adoption by WG Issued + + The "Call for Adoption by WG Issued" state should be used to indicate when an I-D is being considered for adoption by an IETF WG. An I-D that is in this state is actively being considered for adoption and has not yet achieved consensus, preference, or selection in the WG. + + This state may be used to describe an I-D that someone has asked a WG to consider for adoption, if the WG Chair has agreed with the request. This state may also be used to identify an I-D that a WG Chair asked an author to write specifically for consideration as a candidate WG item [WGDTSPEC], and/or an I-D that is listed as a 'candidate draft' in the WG's charter. + + Under normal conditions, it should not be possible for an I-D to be in the "Call for Adoption by WG Issued" state in more than one working group at the same time. This said, it is not uncommon for authors to "shop" their I-Ds to more than one WG at a time, with the hope of getting their documents adopted somewhere. + + After this state is implemented in the Datatracker, an I-D that is in the "Call for Adoption by WG Issued" state will not be able to be "shopped" to any other WG without the consent of the WG Chairs and the responsible ADs impacted by the shopping. + + Note that Figure 1 includes an arc leading from this state to outside of the WG state machine. This illustrates that some I-Ds that are considered do not get adopted as WG drafts. An I-D that is not adopted as a WG draft will transition out of the WG state machine and revert back to having no stream-specific state; however, the status change history log of the I-D will record that the I-D was previously in the "Call for Adoption by WG Issued" state."""), + ("Adopted by a WG","""4.2.2. Adopted by a WG + + The "Adopted by a WG" state describes an individual submission I-D that an IETF WG has agreed to adopt as one of its WG drafts. + + WG Chairs who use this state will be able to clearly indicate when their WGs adopt individual submission I-Ds. This will facilitate the Datatracker's ability to correctly capture "Replaces" information for WG drafts and correct "Replaced by" information for individual submission I-Ds that have been replaced by WG drafts. + + This state is needed because the Datatracker uses the filename of an I-D as a key to search its database for status information about the I-D, and because the filename of a WG I-D is supposed to be different from the filename of an individual submission I-D. The filename of an individual submission I-D will typically be formatted as 'draft-author-wgname-topic-nn'. + + The filename of a WG document is supposed to be formatted as 'draft- ietf-wgname-topic-nn'. + + An individual I-D that is adopted by a WG may take weeks or months to be resubmitted by the author as a new (version-00) WG draft. If the "Adopted by a WG" state is not used, the Datatracker has no way to determine that an I-D has been adopted until a new version of the I-D is submitted to the WG by the author and until the I-D is approved for posting by a WG Chair."""), + ("Adopted for WG Info Only","""4.2.3. Adopted for WG Info Only + + The "Adopted for WG Info Only" state describes a document that contains useful information for the WG that adopted it, but the document is not intended to be published as an RFC. The WG will not actively develop the contents of the I-D or progress it for publication as an RFC. The only purpose of the I-D is to provide information for internal use by the WG."""), + ]: + State.objects.filter(name=name).update(desc=desc) + +class Migration(migrations.Migration): + + dependencies = [ + ("doc", "0025_storedobject_storedobject_unique_name_per_store"), + ] + + operations = [ + migrations.RunPython(forward, reverse) + ] From 327447f91fa21ef7620d958b5f8fc1f00d4f85a5 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 17 Sep 2025 13:42:09 -0500 Subject: [PATCH 025/313] feat: iesg dashboard of wg documents (#9363) * feat: iesg dashboard of wg documents (#8999) * fix: removed template html cruft * fix: avoid triggering a Ghostery false positive * fix: remove related-id, milestone, and last meeting columns * fix: make wgs with no docs show in last table * fix: remove wg w/wo docs columns from first three thables * fix: Make table names closer to original request * chore: ruff format ietf.iesg.utils * feat: refactor, test, cleanup * chore: added comment about the test wg acronyms --------- Co-authored-by: Jennifer Richards --- ietf/iesg/tests.py | 1583 +++++++++++++++++++++++ ietf/iesg/urls.py | 1 + ietf/iesg/utils.py | 296 ++++- ietf/iesg/views.py | 12 +- ietf/templates/iesg/working_groups.html | 159 +++ 5 files changed, 2028 insertions(+), 23 deletions(-) create mode 100644 ietf/templates/iesg/working_groups.html diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py index 746ea3f56f9..f3778d1ded1 100644 --- a/ietf/iesg/tests.py +++ b/ietf/iesg/tests.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- +from collections import Counter import datetime import io import tarfile @@ -24,7 +25,9 @@ from ietf.group.models import Group, GroupMilestone, Role from ietf.iesg.agenda import get_agenda_date, agenda_data, fill_in_agenda_administrivia, agenda_sections from ietf.iesg.models import TelechatDate, TelechatAgendaContent +from ietf.iesg.utils import get_wg_dashboard_info from ietf.name.models import StreamName, TelechatAgendaSectionName +from ietf.person.factories import PersonFactory from ietf.person.models import Person from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent from ietf.iesg.factories import IESGMgmtItemFactory, TelechatAgendaContentFactory @@ -182,6 +185,1586 @@ def test_ietf_activity(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) + def test_working_groups(self): + # Clean away the wasted built-for-every-test noise + Group.objects.filter(type__in=["wg", "area"]).delete() + + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + self.assertEqual(area_summary, []) + self.assertEqual( + area_totals, {"group_count": 0, "doc_count": 0, "page_count": 0} + ) + self.assertEqual(ad_summary, []) + self.assertEqual(noad_summary, []) + self.assertEqual( + ad_totals, + { + "ad_group_count": 0, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 0, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + totals, + { + "group_count": 0, + "doc_count": 0, + "page_count": 0, + "groups_with_docs_count": 0, + }, + ) + self.assertEqual(wg_summary, []) + + # Construct Areas with WGs similar in shape to a real moment of the IETF + + # Note that this test construciton uses the first letter of the wg acronyms + # for convenience to switch on whether groups have documents with assigned ADs. + # (Search for ` if wg_acronym[0] > "g"`) + # There's no other significance to the names of the area directors or the + # acronyms of the areas and groups other than being distinct. Taking the + # values from sets of similar things hopefully helps with debugging the tests. + + areas = {} + for area_acronym in ["red", "orange", "yellow", "green", "blue", "violet"]: + areas[area_acronym] = GroupFactory(type_id="area", acronym=area_acronym) + for ad, area, wgs in [ + ("Alpha", "red", ["bassoon"]), + ("Bravo", "orange", ["celesta"]), + ("Charlie", "orange", ["clarinet", "cymbals"]), + ("Delta", "yellow", ["flute"]), + ("Echo", "yellow", ["glockenspiel"]), + ("Foxtrot", "green", ["gong", "guitar"]), + ("Golf", "green", ["harp"]), + ("Hotel", "blue", ["harpsichord"]), + ("Indigo", "blue", ["oboe", "organ"]), + ("Juliet", "violet", ["piano"]), + ("Kilo", "violet", ["piccolo"]), + ("Lima", "violet", ["saxophone", "tambourine"]), + ]: + p = Person.objects.filter(name=ad).first() or PersonFactory(name=ad) + RoleFactory(group=areas[area], person=p, name_id="ad") + for wg in wgs: + g = GroupFactory(acronym=wg, type_id="wg", parent=areas[area]) + RoleFactory(group=g, person=p, name_id="ad") + + # Some ADs have out of area groups + g = GroupFactory(acronym="timpani", parent=areas["orange"]) + RoleFactory(group=g, person=Person.objects.get(name="Juliet"), name_id="ad") + + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + + self.assertEqual( + area_summary, + [ + { + "area": "red", + "groups_in_area": 1, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "orange", + "groups_in_area": 4, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "yellow", + "groups_in_area": 2, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "green", + "groups_in_area": 3, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "blue", + "groups_in_area": 3, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "violet", + "groups_in_area": 4, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + ], + ) + self.assertEqual( + area_totals, {"group_count": 0, "doc_count": 0, "page_count": 0} + ) + self.assertEqual( + ad_summary, + [ + { + "ad": "Alpha", + "area": "red", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Bravo", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Charlie", + "area": "orange", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Delta", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Echo", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Foxtrot", + "area": "green", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Golf", + "area": "green", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Hotel", + "area": "blue", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Indigo", + "area": "blue", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Juliet", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Juliet", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Kilo", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Lima", + "area": "violet", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + ], + ) + self.assertEqual( + noad_summary, + [ + { + "ad": "Alpha", + "area": "red", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Bravo", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Charlie", + "area": "orange", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Delta", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Echo", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Foxtrot", + "area": "green", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Golf", + "area": "green", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Hotel", + "area": "blue", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Indigo", + "area": "blue", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Juliet", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Juliet", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Kilo", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Lima", + "area": "violet", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + ], + ) + self.assertEqual( + ad_totals, + { + "ad_group_count": 17, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 17, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + totals, + { + "group_count": 17, + "doc_count": 0, + "page_count": 0, + "groups_with_docs_count": 0, + }, + ) + self.assertEqual( + wg_summary, + [ + { + "wg": "bassoon", + "area": "red", + "ad": "Alpha", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "celesta", + "area": "orange", + "ad": "Bravo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "clarinet", + "area": "orange", + "ad": "Charlie", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "cymbals", + "area": "orange", + "ad": "Charlie", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "flute", + "area": "yellow", + "ad": "Delta", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "glockenspiel", + "area": "yellow", + "ad": "Echo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "gong", + "area": "green", + "ad": "Foxtrot", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "guitar", + "area": "green", + "ad": "Foxtrot", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "harp", + "area": "green", + "ad": "Golf", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "harpsichord", + "area": "blue", + "ad": "Hotel", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "oboe", + "area": "blue", + "ad": "Indigo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "organ", + "area": "blue", + "ad": "Indigo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "piano", + "area": "violet", + "ad": "Juliet", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "piccolo", + "area": "violet", + "ad": "Kilo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "saxophone", + "area": "violet", + "ad": "Lima", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "tambourine", + "area": "violet", + "ad": "Lima", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "timpani", + "area": "orange", + "ad": "Juliet", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + ], + ) + + # As seen above, all doc and page counts are currently 0 + + # We'll give a group a document but not assign it to its AD + WgDraftFactory( + group=Group.objects.get(acronym="saxophone"), pages=len("saxophone") + ) + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + count_violet_dicts = 0 + for d in area_summary: + if d["area"] == "violet": + count_violet_dicts += 1 + self.assertEqual(d["groups_with_docs"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + self.assertEqual(d["group_percent"], 100.0) + self.assertEqual(d["doc_percent"], 100.0) + self.assertEqual(d["page_percent"], 100.0) + else: + self.assertEqual(d["groups_with_docs"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_violet_dicts, 1) + + self.assertEqual( + area_totals, {"group_count": 1, "doc_count": 1, "page_count": 9} + ) + + # No AD has this document, even though it's in Lima's group + count_lima_dicts = 0 + for d in ad_summary: + if d["ad"] == "Lima": + count_lima_dicts += 1 + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_lima_dicts, 1) + + # It's in Lima's group, so normally it will eventually land on Lima + count_lima_dicts = 0 + for d in noad_summary: + if d["ad"] == "Lima": + count_lima_dicts += 1 + self.assertEqual(d["doc_group_count"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + self.assertEqual(d["group_percent"], 100.0) + self.assertEqual(d["doc_percent"], 100.0) + self.assertEqual(d["page_percent"], 100.0) + else: + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_lima_dicts, 1) + + self.assertEqual( + ad_totals, + { + "ad_group_count": 17, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 17, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 9, + }, + ) + self.assertEqual( + totals, + { + "group_count": 17, + "doc_count": 1, + "page_count": 9, + "groups_with_docs_count": 1, + }, + ) + + count_sax_dicts = 0 + for d in wg_summary: + if d["wg"] == "saxophone": + count_sax_dicts += 1 + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + else: + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(count_sax_dicts, 1) + + # Assign that doc to Lima + self.assertEqual(Document.objects.count(), 1) + Document.objects.all().update(ad=Person.objects.get(name="Lima")) + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + count_violet_dicts = 0 + for d in area_summary: + if d["area"] == "violet": + count_violet_dicts += 1 + self.assertEqual(d["groups_with_docs"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + self.assertEqual(d["group_percent"], 100.0) + self.assertEqual(d["doc_percent"], 100.0) + self.assertEqual(d["page_percent"], 100.0) + else: + self.assertEqual(d["groups_with_docs"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_violet_dicts, 1) + + self.assertEqual( + area_totals, {"group_count": 1, "doc_count": 1, "page_count": 9} + ) + + # This time it will show up as a doc assigned to Lima + count_lima_dicts = 0 + for d in ad_summary: + if d["ad"] == "Lima": + count_lima_dicts += 1 + self.assertEqual(d["doc_group_count"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + self.assertEqual(d["group_percent"], 100.0) + self.assertEqual(d["doc_percent"], 100.0) + self.assertEqual(d["page_percent"], 100.0) + else: + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_lima_dicts, 1) + + # and there will be no noad documents + count_lima_dicts = 0 + for d in noad_summary: + if d["ad"] == "Lima": + count_lima_dicts += 1 + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_lima_dicts, 1) + + self.assertEqual( + ad_totals, + { + "ad_group_count": 17, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 9, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 17, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + totals, + { + "group_count": 17, + "doc_count": 1, + "page_count": 9, + "groups_with_docs_count": 1, + }, + ) + + count_sax_dicts = 0 + for d in wg_summary: + if d["wg"] == "saxophone": + count_sax_dicts += 1 + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + else: + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(count_sax_dicts, 1) + + # Now give Lima a document in a group that's not in their area: + WgDraftFactory( + group=Group.objects.get(acronym="gong"), + pages=len("gong"), + ad=Person.objects.get(name="Lima"), + ) + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + seen_dicts = Counter([d["area"] for d in area_summary]) + for d in areas: + self.assertEqual(seen_dicts[area], 1 if area in ["violet", "green"] else 0) + for d in area_summary: + if d["area"] in ["violet", "green"]: + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9 if d["area"] == "violet" else 4) + self.assertEqual(d["group_percent"], 50) + self.assertEqual(d["doc_percent"], 50) + self.assertEqual( + d["page_percent"], + 100 * 9 / 13 if d["area"] == "violet" else 100 * 4 / 13, + ) + else: + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + + self.assertEqual( + area_totals, {"group_count": 2, "doc_count": 2, "page_count": 13} + ) + + for d in ad_summary: + if d["ad"] == "Lima": + self.assertEqual(d["doc_group_count"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9 if d["area"] == "violet" else 4) + self.assertEqual(d["group_percent"], 50) + self.assertEqual(d["doc_percent"], 50) + self.assertEqual( + d["page_percent"], + 100 * 9 / 13 if d["area"] == "violet" else 100 * 4 / 13, + ) + else: + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual( + d["doc_count"], 0 + ) # Note in particular this is 0 for Foxtrot + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + + for d in wg_summary: + if d["wg"] == "gong": + # Lima's doc in gong above counts at the dict for gong even though the ad reported there is Foxtrot. + self.assertEqual( + d, + { + "wg": "gong", + "area": "green", + "ad": "Foxtrot", + "doc_count": 1, + "page_count": 4, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + ) + elif d["ad"] == "Lima": + self.assertEqual( + d["area"], "violet" + ) # The out of area assignment is not reflected in the wg_summary at all. + + # Now pile on a lot of documents + for wg_acronym in [ + "bassoon", + "celesta", + "clarinet", + "cymbals", + "flute", + "glockenspiel", + "gong", + "guitar", + "harp", + "harpsichord", + "oboe", + "organ", + "piano", + "piccolo", + "saxophone", + "tambourine", + "timpani", + ]: + if wg_acronym in ["bassoon", "celesta"]: + continue # Those WGs have no docs + # The rest have a doc that's not assigned to any ad + WgDraftFactory( + group=Group.objects.get(acronym=wg_acronym), pages=len(wg_acronym) + ) + if wg_acronym[0] > "g": + # Some have a doc assigned to the responsible ad + WgDraftFactory( + group=Group.objects.get(acronym=wg_acronym), + pages=len(wg_acronym), + ad=Role.objects.get(name_id="ad", group__acronym=wg_acronym).person, + ) + # The other AD for an area might be covering a doc + WgDraftFactory( + group=Group.objects.get(acronym="saxophone"), + pages=len("saxophone"), + ad=Person.objects.get(name="Juliet"), + ) + # An Ad not associated with the group or the area is responsible for a doc + WgDraftFactory( + group=Group.objects.get(acronym="bassoon"), + pages=len("bassoon"), + ad=Person.objects.get(name="Juliet"), + ) + + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + + self.assertEqual( + area_summary, + [ + { + "area": "red", + "groups_in_area": 1, + "groups_with_docs": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 6.25, + "doc_percent": 3.571428571428571, + "page_percent": 3.5897435897435894, + }, + { + "area": "orange", + "groups_in_area": 4, + "groups_with_docs": 3, + "doc_count": 4, + "page_count": 29, + "group_percent": 18.75, + "doc_percent": 14.285714285714285, + "page_percent": 14.871794871794872, + }, + { + "area": "yellow", + "groups_in_area": 2, + "groups_with_docs": 2, + "doc_count": 2, + "page_count": 17, + "group_percent": 12.5, + "doc_percent": 7.142857142857142, + "page_percent": 8.717948717948717, + }, + { + "area": "green", + "groups_in_area": 3, + "groups_with_docs": 3, + "doc_count": 5, + "page_count": 22, + "group_percent": 18.75, + "doc_percent": 17.857142857142858, + "page_percent": 11.282051282051283, + }, + { + "area": "blue", + "groups_in_area": 3, + "groups_with_docs": 3, + "doc_count": 6, + "page_count": 40, + "group_percent": 18.75, + "doc_percent": 21.428571428571427, + "page_percent": 20.51282051282051, + }, + { + "area": "violet", + "groups_in_area": 4, + "groups_with_docs": 4, + "doc_count": 10, + "page_count": 80, + "group_percent": 25.0, + "doc_percent": 35.714285714285715, + "page_percent": 41.02564102564102, + }, + ], + ) + self.assertEqual( + area_totals, {"group_count": 16, "doc_count": 28, "page_count": 195} + ) + self.assertEqual( + ad_summary, + [ + { + "ad": "Alpha", + "area": "red", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Bravo", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Charlie", + "area": "orange", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Delta", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Echo", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Foxtrot", + "area": "green", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Golf", + "area": "green", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 4, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 4.395604395604396, + }, + { + "ad": "Hotel", + "area": "blue", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 11, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 12.087912087912088, + }, + { + "ad": "Indigo", + "area": "blue", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 9, + "group_percent": 16.666666666666664, + "doc_percent": 15.384615384615385, + "page_percent": 9.89010989010989, + }, + { + "ad": "Juliet", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 7.6923076923076925, + }, + { + "ad": "Juliet", + "area": "red", + "ad_group_count": 0, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 7.6923076923076925, + }, + { + "ad": "Juliet", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 14, + "group_percent": 16.666666666666664, + "doc_percent": 15.384615384615385, + "page_percent": 15.384615384615385, + }, + { + "ad": "Kilo", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 7.6923076923076925, + }, + { + "ad": "Lima", + "area": "green", + "ad_group_count": 0, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 4, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 4.395604395604396, + }, + { + "ad": "Lima", + "area": "violet", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 3, + "page_count": 28, + "group_percent": 16.666666666666664, + "doc_percent": 23.076923076923077, + "page_percent": 30.76923076923077, + }, + ], + ) + self.assertEqual( + noad_summary, + [ + { + "ad": "Alpha", + "area": "red", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Bravo", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Charlie", + "area": "orange", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 15, + "group_percent": 13.333333333333334, + "doc_percent": 13.333333333333334, + "page_percent": 14.423076923076922, + }, + { + "ad": "Delta", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 5, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 4.807692307692308, + }, + { + "ad": "Echo", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 12, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 11.538461538461538, + }, + { + "ad": "Foxtrot", + "area": "green", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 10, + "group_percent": 13.333333333333334, + "doc_percent": 13.333333333333334, + "page_percent": 9.615384615384617, + }, + { + "ad": "Golf", + "area": "green", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 4, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 3.8461538461538463, + }, + { + "ad": "Hotel", + "area": "blue", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 11, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 10.576923076923077, + }, + { + "ad": "Indigo", + "area": "blue", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 9, + "group_percent": 13.333333333333334, + "doc_percent": 13.333333333333334, + "page_percent": 8.653846153846153, + }, + { + "ad": "Juliet", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 6.730769230769231, + }, + { + "ad": "Juliet", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 5, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 4.807692307692308, + }, + { + "ad": "Kilo", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 6.730769230769231, + }, + { + "ad": "Lima", + "area": "violet", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 19, + "group_percent": 13.333333333333334, + "doc_percent": 13.333333333333334, + "page_percent": 18.269230769230766, + }, + ], + ) + self.assertEqual( + ad_totals, + { + "ad_group_count": 17, + "doc_group_count": 12, + "doc_count": 13, + "page_count": 91, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 17, + "doc_group_count": 15, + "doc_count": 15, + "page_count": 104, + }, + ) + self.assertEqual( + totals, + { + "group_count": 17, + "doc_count": 28, + "page_count": 195, + "groups_with_docs_count": 16, + }, + ) + self.assertEqual( + wg_summary, + [ + { + "wg": "bassoon", + "area": "red", + "ad": "Alpha", + "doc_count": 1, + "page_count": 7, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "celesta", + "area": "orange", + "ad": "Bravo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "clarinet", + "area": "orange", + "ad": "Charlie", + "doc_count": 1, + "page_count": 8, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "cymbals", + "area": "orange", + "ad": "Charlie", + "doc_count": 1, + "page_count": 7, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "flute", + "area": "yellow", + "ad": "Delta", + "doc_count": 1, + "page_count": 5, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "glockenspiel", + "area": "yellow", + "ad": "Echo", + "doc_count": 1, + "page_count": 12, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "gong", + "area": "green", + "ad": "Foxtrot", + "doc_count": 2, + "page_count": 8, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "guitar", + "area": "green", + "ad": "Foxtrot", + "doc_count": 1, + "page_count": 6, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "harp", + "area": "green", + "ad": "Golf", + "doc_count": 2, + "page_count": 8, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "harpsichord", + "area": "blue", + "ad": "Hotel", + "doc_count": 2, + "page_count": 22, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "oboe", + "area": "blue", + "ad": "Indigo", + "doc_count": 2, + "page_count": 8, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "organ", + "area": "blue", + "ad": "Indigo", + "doc_count": 2, + "page_count": 10, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "piano", + "area": "violet", + "ad": "Juliet", + "doc_count": 2, + "page_count": 10, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "piccolo", + "area": "violet", + "ad": "Kilo", + "doc_count": 2, + "page_count": 14, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "saxophone", + "area": "violet", + "ad": "Lima", + "doc_count": 4, + "page_count": 36, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "tambourine", + "area": "violet", + "ad": "Lima", + "doc_count": 2, + "page_count": 20, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "timpani", + "area": "orange", + "ad": "Juliet", + "doc_count": 2, + "page_count": 14, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + ], + ) + + # Make sure the view doesn't _crash_ - the template is a dead-simple rendering of the dicts, but this test doesn't prove that + url = urlreverse("ietf.iesg.views.working_groups") + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + class IESGAgendaTests(TestCase): def setUp(self): diff --git a/ietf/iesg/urls.py b/ietf/iesg/urls.py index d8cfec9f90a..5fd9dea0ccb 100644 --- a/ietf/iesg/urls.py +++ b/ietf/iesg/urls.py @@ -59,6 +59,7 @@ url(r'^agenda/telechat-(?:%(date)s-)?docs.tgz' % settings.URL_REGEXPS, views.telechat_docs_tarfile), url(r'^discusses/$', views.discusses), url(r'^ietf-activity/$', views.ietf_activity), + url(r'^working-groups/$', views.working_groups), url(r'^milestones/$', views.milestones_needing_review), url(r'^photos/$', views.photos), ] diff --git a/ietf/iesg/utils.py b/ietf/iesg/utils.py index 56571dc7538..9051cf92b28 100644 --- a/ietf/iesg/utils.py +++ b/ietf/iesg/utils.py @@ -1,32 +1,45 @@ -from collections import namedtuple +from collections import Counter, defaultdict, namedtuple -import debug # pyflakes:ignore +import datetime + +import debug # pyflakes:ignore + +from django.db import models +from django.utils import timezone from ietf.doc.models import Document, STATUSCHANGE_RELATIONS from ietf.doc.utils_search import fill_in_telechat_date +from ietf.group.models import Group from ietf.iesg.agenda import get_doc_section +from ietf.person.utils import get_active_ads + +TelechatPageCount = namedtuple( + "TelechatPageCount", + ["for_approval", "for_action", "related", "ad_pages_left_to_ballot_on"], +) -TelechatPageCount = namedtuple('TelechatPageCount',['for_approval','for_action','related','ad_pages_left_to_ballot_on']) def telechat_page_count(date=None, docs=None, ad=None): if not date and not docs: return TelechatPageCount(0, 0, 0, 0) if not docs: - candidates = Document.objects.filter(docevent__telechatdocevent__telechat_date=date).distinct() + candidates = Document.objects.filter( + docevent__telechatdocevent__telechat_date=date + ).distinct() fill_in_telechat_date(candidates) - docs = [ doc for doc in candidates if doc.telechat_date()==date ] + docs = [doc for doc in candidates if doc.telechat_date() == date] - for_action =[d for d in docs if get_doc_section(d).endswith('.3')] + for_action = [d for d in docs if get_doc_section(d).endswith(".3")] - for_approval = set(docs)-set(for_action) + for_approval = set(docs) - set(for_action) - drafts = [d for d in for_approval if d.type_id == 'draft'] + drafts = [d for d in for_approval if d.type_id == "draft"] ad_pages_left_to_ballot_on = 0 pages_for_approval = 0 - + for draft in drafts: pages_for_approval += draft.pages or 0 if ad: @@ -39,30 +52,269 @@ def telechat_page_count(date=None, docs=None, ad=None): pages_for_action = 0 for d in for_action: - if d.type_id == 'draft': + if d.type_id == "draft": pages_for_action += d.pages or 0 - elif d.type_id == 'statchg': + elif d.type_id == "statchg": for rel in d.related_that_doc(STATUSCHANGE_RELATIONS): pages_for_action += rel.pages or 0 - elif d.type_id == 'conflrev': - for rel in d.related_that_doc('conflrev'): + elif d.type_id == "conflrev": + for rel in d.related_that_doc("conflrev"): pages_for_action += rel.pages or 0 else: pass related_pages = 0 - for d in for_approval-set(drafts): - if d.type_id == 'statchg': + for d in for_approval - set(drafts): + if d.type_id == "statchg": for rel in d.related_that_doc(STATUSCHANGE_RELATIONS): related_pages += rel.pages or 0 - elif d.type_id == 'conflrev': - for rel in d.related_that_doc('conflrev'): + elif d.type_id == "conflrev": + for rel in d.related_that_doc("conflrev"): related_pages += rel.pages or 0 else: # There's really nothing to rely on to give a reading load estimate for charters pass - - return TelechatPageCount(for_approval=pages_for_approval, - for_action=pages_for_action, - related=related_pages, - ad_pages_left_to_ballot_on=ad_pages_left_to_ballot_on) + + return TelechatPageCount( + for_approval=pages_for_approval, + for_action=pages_for_action, + related=related_pages, + ad_pages_left_to_ballot_on=ad_pages_left_to_ballot_on, + ) + + +def get_wg_dashboard_info(): + docs = ( + Document.objects.filter( + group__type="wg", + group__state="active", + states__type="draft", + states__slug="active", + ) + .filter(models.Q(ad__isnull=True) | models.Q(ad__in=get_active_ads())) + .distinct() + .prefetch_related("group", "group__parent") + .exclude( + states__type="draft-stream-ietf", + states__slug__in=["c-adopt", "wg-cand", "dead", "parked", "info"], + ) + ) + groups = Group.objects.filter(state="active", type="wg") + areas = Group.objects.filter(state="active", type="area") + + total_group_count = groups.count() + total_doc_count = docs.count() + total_page_count = docs.aggregate(models.Sum("pages"))["pages__sum"] or 0 + totals = { + "group_count": total_group_count, + "doc_count": total_doc_count, + "page_count": total_page_count, + } + + # Since this view is primarily about counting subsets of the above docs query and the + # expected number of returned documents is just under 1000 typically - do the totaling + # work in python rather than asking the db to do it. + + groups_for_area = defaultdict(set) + pages_for_area = defaultdict(lambda: 0) + docs_for_area = defaultdict(lambda: 0) + groups_for_ad = defaultdict(lambda: defaultdict(set)) + pages_for_ad = defaultdict(lambda: defaultdict(lambda: 0)) + docs_for_ad = defaultdict(lambda: defaultdict(lambda: 0)) + groups_for_noad = defaultdict(lambda: defaultdict(set)) + pages_for_noad = defaultdict(lambda: defaultdict(lambda: 0)) + docs_for_noad = defaultdict(lambda: defaultdict(lambda: 0)) + docs_for_wg = defaultdict(lambda: 0) + pages_for_wg = defaultdict(lambda: 0) + groups_total = set() + pages_total = 0 + docs_total = 0 + + responsible_for_group = defaultdict(lambda: defaultdict(lambda: "None")) + responsible_count = defaultdict(lambda: defaultdict(lambda: 0)) + for group in groups: + responsible = f"{', '.join([r.person.plain_name() for r in group.role_set.filter(name_id='ad')])}" + docs_for_noad[responsible][group.parent.acronym] = ( + 0 # Ensure these keys are present later + ) + docs_for_ad[responsible][group.parent.acronym] = 0 + responsible_for_group[group.acronym][group.parent.acronym] = responsible + responsible_count[responsible][group.parent.acronym] += 1 + + for doc in docs: + docs_for_wg[doc.group] += 1 + pages_for_wg[doc.group] += doc.pages + groups_for_area[doc.group.area.acronym].add(doc.group.acronym) + pages_for_area[doc.group.area.acronym] += doc.pages + docs_for_area[doc.group.area.acronym] += 1 + + if doc.ad is None: + responsible = responsible_for_group[doc.group.acronym][ + doc.group.parent.acronym + ] + groups_for_noad[responsible][doc.group.parent.acronym].add( + doc.group.acronym + ) + pages_for_noad[responsible][doc.group.parent.acronym] += doc.pages + docs_for_noad[responsible][doc.group.parent.acronym] += 1 + else: + responsible = f"{doc.ad.plain_name()}" + groups_for_ad[responsible][doc.group.parent.acronym].add(doc.group.acronym) + pages_for_ad[responsible][doc.group.parent.acronym] += doc.pages + docs_for_ad[responsible][doc.group.parent.acronym] += 1 + + docs_total += 1 + groups_total.add(doc.group.acronym) + pages_total += doc.pages + + groups_total = len(groups_total) + totals["groups_with_docs_count"] = groups_total + + area_summary = [] + + for area in areas: + group_count = len(groups_for_area[area.acronym]) + doc_count = docs_for_area[area.acronym] + page_count = pages_for_area[area.acronym] + area_summary.append( + { + "area": area.acronym, + "groups_in_area": groups.filter(parent=area).count(), + "groups_with_docs": group_count, + "doc_count": doc_count, + "page_count": page_count, + "group_percent": group_count / groups_total * 100 + if groups_total != 0 + else 0, + "doc_percent": doc_count / docs_total * 100 if docs_total != 0 else 0, + "page_percent": page_count / pages_total * 100 + if pages_total != 0 + else 0, + } + ) + area_totals = { + "group_count": groups_total, + "doc_count": docs_total, + "page_count": pages_total, + } + + noad_summary = [] + noad_totals = { + "ad_group_count": 0, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + } + for ad in docs_for_noad: + for area in docs_for_noad[ad]: + noad_totals["ad_group_count"] += responsible_count[ad][area] + noad_totals["doc_group_count"] += len(groups_for_noad[ad][area]) + noad_totals["doc_count"] += docs_for_noad[ad][area] + noad_totals["page_count"] += pages_for_noad[ad][area] + for ad in docs_for_noad: + for area in docs_for_noad[ad]: + noad_summary.append( + { + "ad": ad, + "area": area, + "ad_group_count": responsible_count[ad][area], + "doc_group_count": len(groups_for_noad[ad][area]), + "doc_count": docs_for_noad[ad][area], + "page_count": pages_for_noad[ad][area], + "group_percent": len(groups_for_noad[ad][area]) + / noad_totals["doc_group_count"] + * 100 + if noad_totals["doc_group_count"] != 0 + else 0, + "doc_percent": docs_for_noad[ad][area] + / noad_totals["doc_count"] + * 100 + if noad_totals["doc_count"] != 0 + else 0, + "page_percent": pages_for_noad[ad][area] + / noad_totals["page_count"] + * 100 + if noad_totals["page_count"] != 0 + else 0, + } + ) + noad_summary.sort(key=lambda r: (r["ad"], r["area"])) + + ad_summary = [] + ad_totals = { + "ad_group_count": 0, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + } + for ad in docs_for_ad: + for area in docs_for_ad[ad]: + ad_totals["ad_group_count"] += responsible_count[ad][area] + ad_totals["doc_group_count"] += len(groups_for_ad[ad][area]) + ad_totals["doc_count"] += docs_for_ad[ad][area] + ad_totals["page_count"] += pages_for_ad[ad][area] + for ad in docs_for_ad: + for area in docs_for_ad[ad]: + ad_summary.append( + { + "ad": ad, + "area": area, + "ad_group_count": responsible_count[ad][area], + "doc_group_count": len(groups_for_ad[ad][area]), + "doc_count": docs_for_ad[ad][area], + "page_count": pages_for_ad[ad][area], + "group_percent": len(groups_for_ad[ad][area]) + / ad_totals["doc_group_count"] + * 100 + if ad_totals["doc_group_count"] != 0 + else 0, + "doc_percent": docs_for_ad[ad][area] / ad_totals["doc_count"] * 100 + if ad_totals["doc_count"] != 0 + else 0, + "page_percent": pages_for_ad[ad][area] + / ad_totals["page_count"] + * 100 + if ad_totals["page_count"] != 0 + else 0, + } + ) + ad_summary.sort(key=lambda r: (r["ad"], r["area"])) + + rfc_counter = Counter( + Document.objects.filter(type="rfc").values_list("group__acronym", flat=True) + ) + recent_rfc_counter = Counter( + Document.objects.filter( + type="rfc", + docevent__type="published_rfc", + docevent__time__gte=timezone.now() - datetime.timedelta(weeks=104), + ).values_list("group__acronym", flat=True) + ) + for wg in set(groups) - set(docs_for_wg.keys()): + docs_for_wg[wg] += 0 + pages_for_wg[wg] += 0 + wg_summary = [] + for wg in docs_for_wg: + wg_summary.append( + { + "wg": wg.acronym, + "area": wg.parent.acronym, + "ad": responsible_for_group[wg.acronym][wg.parent.acronym], + "doc_count": docs_for_wg[wg], + "page_count": pages_for_wg[wg], + "rfc_count": rfc_counter[wg.acronym], + "recent_rfc_count": recent_rfc_counter[wg.acronym], + } + ) + wg_summary.sort(key=lambda r: (r["wg"], r["area"])) + + return ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index ffd4515c98f..014b2904259 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -61,7 +61,7 @@ from ietf.group.models import GroupMilestone, Role from ietf.iesg.agenda import agenda_data, agenda_sections, fill_in_agenda_docs, get_agenda_date from ietf.iesg.models import TelechatDate, TelechatAgendaContent -from ietf.iesg.utils import telechat_page_count +from ietf.iesg.utils import get_wg_dashboard_info, telechat_page_count from ietf.ietfauth.utils import has_role, role_required, user_is_person from ietf.name.models import TelechatAgendaSectionName from ietf.person.models import Person @@ -626,3 +626,13 @@ def telechat_agenda_content_view(request, section): content=content.text, content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", ) + +def working_groups(request): + + area_summary, area_totals, ad_summary, noad_summary, ad_totals, noad_totals, totals, wg_summary = get_wg_dashboard_info() + + return render( + request, + "iesg/working_groups.html", + dict(area_summary=area_summary, area_totals=area_totals, ad_summary=ad_summary, noad_summary=noad_summary, ad_totals=ad_totals, noad_totals=noad_totals, totals=totals, wg_summary=wg_summary), + ) diff --git a/ietf/templates/iesg/working_groups.html b/ietf/templates/iesg/working_groups.html new file mode 100644 index 00000000000..b799636857f --- /dev/null +++ b/ietf/templates/iesg/working_groups.html @@ -0,0 +1,159 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load origin static %} +{% block pagehead %} + +{% endblock %} +{% block title %}IESG view of working groups{% endblock %} +{% block content %} + {% origin %} +

    IESG view of working groups

    +

    Area Size and Load

    + + + + + + + + {# (divider) #} + + + + + + {% for area in area_summary %} + + + + + + + + + {% endfor %} + + + + + + + + + + + +
    Area NameWGsI-DsPages% I-Ds% Pages
    {{area.area}}{{area.groups_in_area}}{{area.doc_count}}{{area.page_count}}{{area.doc_percent|floatformat:1}}{{area.page_percent|floatformat:1}}
    Totals{{totals.group_count}}{{totals.doc_count}}{{totals.page_count}}
    + +

    Area Director Load: Documents not yet directly assigned to AD

    +
    Typically these are pre-pubreq documents
    + + + + + + + + + {# (divider) #} + + + + + + {% for ad in noad_summary %} + + + + + + + + + + {% endfor %} + + + + + + + + + + + + +
    ADArea NameWGs for ADI-DsPages% I-Ds% Pages
    {{ad.ad}}{{ad.area}}{{ad.ad_group_count}}{{ad.doc_count}}{{ad.page_count}}{{ad.doc_percent|floatformat:1}}{{ad.page_percent|floatformat:1}}
    Totals{{noad_totals.ad_group_count}}{{noad_totals.doc_count}}{{noad_totals.page_count}}
    + +

    Area Director Load: Documents directly assigned to AD

    + + + + + + + + + {# (divider) #} + + + + + + {% for ad in ad_summary %} + + + + + + + + + + {% endfor %} + + + + + + + + + + + + +
    ADArea NameWGs for ADI-DsPages% I-Ds% Pages
    {{ad.ad}}{{ad.area}}{{ad.ad_group_count}}{{ad.doc_count}}{{ad.page_count}}{{ad.doc_percent|floatformat:1}}{{ad.page_percent|floatformat:1}}
    Totals{{ad_totals.ad_group_count}}{{ad_totals.doc_count}}{{ad_totals.page_count}}
    + +

    Working Group Summary

    + + + + + + + + + + + + + + {% for wg in wg_summary %} + + + + + + + + + + {% endfor %} + +
    WGAreaADI-DsPagesRFCsRFCs in last 2 years
    {{wg.wg}}{{wg.area}}{{wg.ad}}{{wg.doc_count}}{{wg.page_count}}{{wg.rfc_count}}{{wg.recent_rfc_count}}
    +{% endblock %} +{% block js %} + +{% endblock %} \ No newline at end of file From acffceba0b8f61d6a5c972080df41f9e86743919 Mon Sep 17 00:00:00 2001 From: Phil Whipps Date: Thu, 18 Sep 2025 04:46:20 +1000 Subject: [PATCH 026/313] fix: Rev Fix Option 2 - Htmlized url regex (#9538) * Update Rev Regex in settings.py Removing single value revision numbers as that is against the naming standard (https://authors.ietf.org/naming-your-internet-draft#version) and causes issues with htmlized documents with -1 in the name (eg draft-ietf-oauth-v2-1) * Reverse REGEX Change * Update URLS REgex for REV Directly insert Regex for REV rather than reference settings.URL_REGEXPS. This is to resolve issue https://github.com/ietf-tools/datatracker/issues/9533 --- ietf/doc/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py index 60255af856c..6f1b698a9fc 100644 --- a/ietf/doc/urls.py +++ b/ietf/doc/urls.py @@ -75,7 +75,7 @@ # This block should really all be at the idealized docs.ietf.org service url(r'^html/(?Pbcp[0-9]+?)(\.txt|\.html)?/?$', RedirectView.as_view(url=settings.RFC_EDITOR_INFO_BASE_URL+"%(name)s", permanent=False)), url(r'^html/(?Pstd[0-9]+?)(\.txt|\.html)?/?$', RedirectView.as_view(url=settings.RFC_EDITOR_INFO_BASE_URL+"%(name)s", permanent=False)), - url(r'^html/%(name)s(?:-%(rev)s)?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, views_doc.document_html), + url(r'^html/%(name)s(?:-(?P[0-9]{2}(-[0-9]{2})?))?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, views_doc.document_html), url(r'^id/%(name)s(?:-%(rev)s)?(?:\.(?P(txt|html|xml)))?/?$' % settings.URL_REGEXPS, views_doc.document_raw_id), url(r'^pdf/%(name)s(?:-%(rev)s)?(?:\.(?P[a-z]+))?/?$' % settings.URL_REGEXPS, views_doc.document_pdfized), From 6b58aa4bd47fd5fe84750a0cc66dd38b8e801c72 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 18 Sep 2025 10:20:03 -0500 Subject: [PATCH 027/313] fix: edit only attachments actually attached to this liaison statement (#9548) * fix: edit only attachments actually attached to this liaison statement * chore: remove unused import --------- Co-authored-by: Jennifer Richards --- ietf/liaisons/tests.py | 29 +++++++++++++++++++++++------ ietf/liaisons/views.py | 11 +++++++---- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index a1fbf77841a..2f86f387890 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -939,17 +939,34 @@ def test_liaison_add_attachment(self): ) def test_liaison_edit_attachment(self): - - attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1') - url = urlreverse('ietf.liaisons.views.liaison_edit_attachment', kwargs=dict(object_id=attachment.statement_id,doc_id=attachment.document_id)) + attachment = LiaisonStatementAttachmentFactory(document__name="liaiatt-1") + url = urlreverse( + "ietf.liaisons.views.liaison_edit_attachment", + kwargs=dict( + object_id=attachment.statement_id, doc_id=attachment.document_id + ), + ) login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertEqual(r.status_code, 200) - post_data = dict(title='New Title') - r = self.client.post(url,post_data) + post_data = dict(title="New Title") + r = self.client.post(url, post_data) attachment = LiaisonStatementAttachment.objects.get(pk=attachment.pk) self.assertEqual(r.status_code, 302) - self.assertEqual(attachment.document.title,'New Title') + self.assertEqual(attachment.document.title, "New Title") + + # ensure attempts to edit attachments not attached to this liaison statement fail + other_attachment = LiaisonStatementAttachmentFactory(document__name="liaiatt-2") + url = urlreverse( + "ietf.liaisons.views.liaison_edit_attachment", + kwargs=dict( + object_id=attachment.statement_id, doc_id=other_attachment.document_id + ), + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + r = self.client.post(url, dict(title="New Title")) + self.assertEqual(r.status_code, 404) def test_liaison_delete_attachment(self): attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1') diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py index 9710149c901..f9136a8d14a 100644 --- a/ietf/liaisons/views.py +++ b/ietf/liaisons/views.py @@ -7,15 +7,14 @@ from django.contrib import messages from django.urls import reverse as urlreverse -from django.core.exceptions import ValidationError +from django.core.exceptions import ValidationError, ObjectDoesNotExist from django.core.validators import validate_email from django.db.models import Q, Prefetch -from django.http import HttpResponse +from django.http import Http404, HttpResponse from django.shortcuts import render, get_object_or_404, redirect import debug # pyflakes:ignore -from ietf.doc.models import Document from ietf.ietfauth.utils import role_required, has_role from ietf.group.models import Group, Role from ietf.liaisons.models import (LiaisonStatement,LiaisonStatementEvent, @@ -444,7 +443,11 @@ def liaison_edit(request, object_id): def liaison_edit_attachment(request, object_id, doc_id): '''Edit the Liaison Statement attachment title''' liaison = get_object_or_404(LiaisonStatement, pk=object_id) - doc = get_object_or_404(Document, pk=doc_id) + try: + doc = liaison.attachments.get(pk=doc_id) + except ObjectDoesNotExist: + raise Http404 + if not can_edit_liaison(request.user, liaison): permission_denied(request, "You are not authorized for this action.") From 76f56ceabf4a101c7a8f72946778b7bb5b63f570 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 18 Sep 2025 10:20:30 -0500 Subject: [PATCH 028/313] fix: adjust anachronystic urls - doc_ids became numeric years ago. (#9549) --- ietf/liaisons/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/liaisons/urls.py b/ietf/liaisons/urls.py index a4afbfef5d8..0fbd29425e4 100644 --- a/ietf/liaisons/urls.py +++ b/ietf/liaisons/urls.py @@ -26,8 +26,8 @@ url(r'^(?P\d+)/$', views.liaison_detail), url(r'^(?P\d+)/addcomment/$', views.add_comment), url(r'^(?P\d+)/edit/$', views.liaison_edit), - url(r'^(?P\d+)/edit-attachment/(?P[A-Za-z0-9._+-]+)$', views.liaison_edit_attachment), - url(r'^(?P\d+)/delete-attachment/(?P[A-Za-z0-9._+-]+)$', views.liaison_delete_attachment), + url(r'^(?P\d+)/edit-attachment/(?P[0-9]+)$', views.liaison_edit_attachment), + url(r'^(?P\d+)/delete-attachment/(?P[0-9]+)$', views.liaison_delete_attachment), url(r'^(?P\d+)/history/$', views.liaison_history), url(r'^(?P\d+)/reply/$', views.liaison_reply), url(r'^(?P\d+)/resend/$', views.liaison_resend), From ad5823e0c6ebaa88ae6c949e1bdefeab951cb280 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 18 Sep 2025 10:22:31 -0500 Subject: [PATCH 029/313] fix: properly guard state transitions (#9554) Co-authored-by: Jennifer Richards --- ietf/liaisons/tests.py | 3 +++ ietf/liaisons/views.py | 32 +++++++++++++++++++------------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index 2f86f387890..c3ff9dbe940 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -363,6 +363,9 @@ def test_approval_process(self): self.assertEqual(len(q('form button[name=approved]')), 0) # check the detail page / authorized + r = self.client.post(url, dict(dead="1")) + self.assertEqual(r.status_code, 403) + mailbox_before = len(outbox) self.client.login(username="ulm-liaiman", password="ulm-liaiman+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py index f9136a8d14a..6a1e6e3def3 100644 --- a/ietf/liaisons/views.py +++ b/ietf/liaisons/views.py @@ -7,7 +7,7 @@ from django.contrib import messages from django.urls import reverse as urlreverse -from django.core.exceptions import ValidationError, ObjectDoesNotExist +from django.core.exceptions import ValidationError, ObjectDoesNotExist, PermissionDenied from django.core.validators import validate_email from django.db.models import Q, Prefetch from django.http import Http404, HttpResponse @@ -404,22 +404,28 @@ def liaison_detail(request, object_id): if request.method == 'POST': - if request.POST.get('approved'): - liaison.change_state(state_id='approved',person=person) - liaison.change_state(state_id='posted',person=person) - send_liaison_by_email(request, liaison) - messages.success(request,'Liaison Statement Approved and Posted') - elif request.POST.get('dead'): - liaison.change_state(state_id='dead',person=person) - messages.success(request,'Liaison Statement Killed') - elif request.POST.get('resurrect'): - liaison.change_state(state_id='pending',person=person) - messages.success(request,'Liaison Statement Resurrected') - elif request.POST.get('do_action_taken') and can_take_care: + if request.POST.get('do_action_taken') and can_take_care: liaison.tags.remove('required') liaison.tags.add('taken') can_take_care = False messages.success(request,'Action handled') + else: + if can_edit: + if request.POST.get('approved'): + liaison.change_state(state_id='approved',person=person) + liaison.change_state(state_id='posted',person=person) + send_liaison_by_email(request, liaison) + messages.success(request,'Liaison Statement Approved and Posted') + elif request.POST.get('dead'): + liaison.change_state(state_id='dead',person=person) + messages.success(request,'Liaison Statement Killed') + elif request.POST.get('resurrect'): + liaison.change_state(state_id='pending',person=person) + messages.success(request,'Liaison Statement Resurrected') + else: + pass + else: + raise PermissionDenied() relations_by = [i.target for i in liaison.source_of_set.filter(target__state__slug='posted')] relations_to = [i.source for i in liaison.target_of_set.filter(source__state__slug='posted')] From e1c75d46161939acaf093bb50cf91af9a2cbb7ea Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 18 Sep 2025 10:32:26 -0500 Subject: [PATCH 030/313] fix: disable removing liaison attachments pending reimplementation (#9555) --- ietf/liaisons/tests.py | 16 ++++++++-------- ietf/liaisons/views.py | 39 ++++++++++++++++++++++----------------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index c3ff9dbe940..fd1c22be775 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -971,14 +971,14 @@ def test_liaison_edit_attachment(self): r = self.client.post(url, dict(title="New Title")) self.assertEqual(r.status_code, 404) - def test_liaison_delete_attachment(self): - attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1') - liaison = attachment.statement - url = urlreverse('ietf.liaisons.views.liaison_delete_attachment', kwargs=dict(object_id=liaison.pk,attach_id=attachment.pk)) - login_testing_unauthorized(self, "secretary", url) - r = self.client.get(url) - self.assertEqual(r.status_code, 302) - self.assertEqual(liaison.liaisonstatementattachment_set.filter(removed=False).count(),0) + # def test_liaison_delete_attachment(self): + # attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1') + # liaison = attachment.statement + # url = urlreverse('ietf.liaisons.views.liaison_delete_attachment', kwargs=dict(object_id=liaison.pk,attach_id=attachment.pk)) + # login_testing_unauthorized(self, "secretary", url) + # r = self.client.get(url) + # self.assertEqual(r.status_code, 302) + # self.assertEqual(liaison.liaisonstatementattachment_set.filter(removed=False).count(),0) def test_in_response(self): '''A statement with purpose=in_response must have related statement specified''' diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py index 6a1e6e3def3..6a6f5797143 100644 --- a/ietf/liaisons/views.py +++ b/ietf/liaisons/views.py @@ -17,8 +17,7 @@ from ietf.ietfauth.utils import role_required, has_role from ietf.group.models import Group, Role -from ietf.liaisons.models import (LiaisonStatement,LiaisonStatementEvent, - LiaisonStatementAttachment) +from ietf.liaisons.models import LiaisonStatement,LiaisonStatementEvent from ietf.liaisons.utils import (get_person_for_user, can_add_outgoing_liaison, can_add_incoming_liaison, can_edit_liaison,can_submit_liaison_required, can_add_liaison) @@ -377,23 +376,29 @@ def liaison_history(request, object_id): def liaison_delete_attachment(request, object_id, attach_id): liaison = get_object_or_404(LiaisonStatement, pk=object_id) - attach = get_object_or_404(LiaisonStatementAttachment, pk=attach_id) + if not can_edit_liaison(request.user, liaison): permission_denied(request, "You are not authorized for this action.") - - # FIXME: this view should use POST instead of GET when deleting - attach.removed = True - attach.save() - - # create event - LiaisonStatementEvent.objects.create( - type_id='modified', - by=get_person_for_user(request.user), - statement=liaison, - desc='Attachment Removed: {}'.format(attach.document.title) - ) - messages.success(request, 'Attachment Deleted') - return redirect('ietf.liaisons.views.liaison_detail', object_id=liaison.pk) + else: + permission_denied(request, "This operation is temporarily unavailable. Ask the secretariat to mark the attachment as removed using the admin.") + + # The following will be replaced with a different approach in the next generation of the liaison tool + # attach = get_object_or_404(LiaisonStatementAttachment, pk=attach_id) + + # # FIXME: this view should use POST instead of GET when deleting + # attach.removed = True + # debug.say("Got here") + # attach.save() + + # # create event + # LiaisonStatementEvent.objects.create( + # type_id='modified', + # by=get_person_for_user(request.user), + # statement=liaison, + # desc='Attachment Removed: {}'.format(attach.document.title) + # ) + # messages.success(request, 'Attachment Deleted') + # return redirect('ietf.liaisons.views.liaison_detail', object_id=liaison.pk) def liaison_detail(request, object_id): liaison = get_object_or_404(LiaisonStatement, pk=object_id) From 87e550c74ffef0f5b64b78a6a487321ebe923f11 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 19 Sep 2025 13:55:14 -0300 Subject: [PATCH 031/313] refactor: compare tokens using compare_digest (#9562) * refactor: compare tokens using compare_digest * test: test new helper * refactor: const-time for auth_token check also --- ietf/submit/tests.py | 31 +++++++++++++++++++++++++++- ietf/submit/views.py | 49 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 6b9002502bd..ede63d2752c 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -51,8 +51,9 @@ process_submission_xml, process_uploaded_submission, process_and_validate_submission, apply_yang_checker_to_draft, run_all_yang_model_checks) +from ietf.submit.views import access_token_is_valid, auth_token_is_valid from ietf.utils import tool_version -from ietf.utils.accesstoken import generate_access_token +from ietf.utils.accesstoken import generate_access_token, generate_random_key from ietf.utils.mail import outbox, get_payload_text from ietf.utils.test_runner import TestBlobstoreManager from ietf.utils.test_utils import login_testing_unauthorized, TestCase @@ -3500,3 +3501,31 @@ def test_submissionerror(self, mock_sanitize_message): mock_sanitize_message.call_args_list, [mock.call("hi"), mock.call("there")], ) + + +class HelperTests(TestCase): + def test_access_token_is_valid(self): + submission: Submission = SubmissionFactory() # type: ignore + valid_token = submission.access_token() + access_key = submission.access_key # accept this for backwards compat + invalid_token = "not the valid token" + self.assertTrue(access_token_is_valid(submission, valid_token)) + self.assertTrue(access_token_is_valid(submission, access_key)) + self.assertFalse(access_token_is_valid(submission, invalid_token)) + + def test_auth_token_is_valid(self): + auth_key = generate_random_key() + submission: Submission = SubmissionFactory(auth_key = auth_key) # type: ignore + valid_token = generate_access_token(submission.auth_key) + auth_key = submission.auth_key # accept this for backwards compat + invalid_token = "not the valid token" + self.assertTrue(auth_token_is_valid(submission, valid_token)) + self.assertTrue(auth_token_is_valid(submission, auth_key)) + self.assertFalse(auth_token_is_valid(submission, invalid_token)) + + submission.auth_key = "" + submission.save() + self.assertFalse(auth_token_is_valid(submission, valid_token)) + self.assertFalse(auth_token_is_valid(submission, auth_key)) + self.assertFalse(auth_token_is_valid(submission, invalid_token)) + self.assertFalse(auth_token_is_valid(submission, "")) diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 043b6130168..8329a312bb7 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import re import datetime +from secrets import compare_digest from typing import Optional, cast # pyflakes:ignore from urllib.parse import urljoin @@ -255,19 +256,48 @@ def search_submission(request): ) -def can_edit_submission(user, submission, access_token): - key_matched = access_token and submission.access_token() == access_token - if not key_matched: key_matched = submission.access_key == access_token # backwards-compat - return key_matched or has_role(user, "Secretariat") +def access_token_is_valid(submission: Submission, access_token: str): + """Check whether access_token is valid for submission, in constant time""" + token_matched = compare_digest(submission.access_token(), access_token) + # also compare key directly for backwards compatibility + key_matched = compare_digest(submission.access_key, access_token) + return token_matched or key_matched + + +def auth_token_is_valid(submission: Submission, auth_token: str): + """Check whether auth_token is valid for submission, in constant time""" + auth_key = submission.auth_key + if not auth_key: + # Make the same calls as the other branch to keep constant time, then + # return False because there is no auth key + compare_digest(generate_access_token("fake"), auth_token) + compare_digest("fake", auth_token) + return False + else: + token_matched = compare_digest(generate_access_token(auth_key), auth_token) + # also compare key directly for backwards compatibility + key_matched = compare_digest(auth_key, auth_token) + return token_matched or key_matched + + +def can_edit_submission(user, submission: Submission, access_token: str | None): + if has_role(user, "Secretariat"): + return True + elif not access_token: + return False + return access_token_is_valid(submission, access_token) + def submission_status(request, submission_id, access_token=None): # type: (HttpRequest, str, Optional[str]) -> HttpResponse submission = get_object_or_404(Submission, pk=submission_id) - key_matched = access_token and submission.access_token() == access_token - if not key_matched: key_matched = submission.access_key == access_token # backwards-compat - if access_token and not key_matched: - raise Http404 + if access_token: + key_matched = access_token_is_valid(submission, access_token) + if not key_matched: + raise Http404 + else: + key_matched = False if submission.state.slug == "cancel": errors = {} @@ -621,8 +651,7 @@ def edit_submission(request, submission_id, access_token=None): def confirm_submission(request, submission_id, auth_token): submission = get_object_or_404(Submission, pk=submission_id) - key_matched = submission.auth_key and auth_token == generate_access_token(submission.auth_key) - if not key_matched: key_matched = auth_token == submission.auth_key # backwards-compat + key_matched = submission.auth_key and auth_token_is_valid(submission, auth_token) if request.method == 'POST' and submission.state_id in ("auth", "aut-appr") and key_matched: # Set a temporary state 'confirmed' to avoid entering this code From 4be83ce312dde9b434f86cff928daf5882809239 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 19 Sep 2025 11:58:49 -0500 Subject: [PATCH 032/313] fix: remove deprecated iesg docs view, link from ad dashboard to ad view of working groups (#9563) * fix: remove deprecated iesg docs view * fix: link from ad dashboard to ad view of working groups --- ietf/doc/tests.py | 11 --- ietf/doc/urls.py | 4 +- ietf/doc/views_search.py | 27 +----- ietf/templates/doc/ad_list.html | 5 +- .../templates/doc/drafts_in_iesg_process.html | 83 ------------------- 5 files changed, 7 insertions(+), 123 deletions(-) delete mode 100644 ietf/templates/doc/drafts_in_iesg_process.html diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index fa8c7fa4fcf..16dcfb7754d 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -449,17 +449,6 @@ def test_drafts_in_last_call(self): self.assertContains(r, draft.title) self.assertContains(r, escape(draft.action_holders.first().name)) - def test_in_iesg_process(self): - doc_in_process = IndividualDraftFactory() - doc_in_process.action_holders.set([PersonFactory()]) - doc_in_process.set_state(State.objects.get(type='draft-iesg', slug='lc')) - doc_not_in_process = IndividualDraftFactory() - r = self.client.get(urlreverse('ietf.doc.views_search.drafts_in_iesg_process')) - self.assertEqual(r.status_code, 200) - self.assertContains(r, doc_in_process.title) - self.assertContains(r, escape(doc_in_process.action_holders.first().name)) - self.assertNotContains(r, doc_not_in_process.title) - def test_indexes(self): draft = IndividualDraftFactory() rfc = WgRfcFactory() diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py index 6f1b698a9fc..7b444782d77 100644 --- a/ietf/doc/urls.py +++ b/ietf/doc/urls.py @@ -53,13 +53,13 @@ url(r'^ad/?$', views_search.ad_workload), url(r'^ad/(?P[^/]+)/?$', views_search.docs_for_ad), url(r'^ad2/(?P[\w.-]+)/$', RedirectView.as_view(url='/doc/ad/%(name)s/', permanent=True)), - url(r'^for_iesg/?$', views_search.docs_for_iesg), + url(r'^for_iesg/?$', RedirectView.as_view(pattern_name='ietf.doc.views_search.docs_for_iesg', permanent=False)), url(r'^rfc-status-changes/?$', views_status_change.rfc_status_changes), url(r'^start-rfc-status-change/(?:%(name)s/)?$' % settings.URL_REGEXPS, views_status_change.start_rfc_status_change), url(r'^bof-requests/?$', views_bofreq.bof_requests), url(r'^bof-requests/new/$', views_bofreq.new_bof_request), url(r'^statement/new/$', views_statement.new_statement), - url(r'^iesg/?$', views_search.drafts_in_iesg_process), + url(r'^iesg/?$', views_search.docs_for_iesg), url(r'^email-aliases/?$', views_doc.email_aliases), url(r'^downref/?$', views_downref.downref_registry), url(r'^downref/add/?$', views_downref.downref_registry_add), diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 67ff0c2f211..2144c23e06d 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -59,7 +59,7 @@ import debug # pyflakes:ignore from ietf.doc.models import ( Document, DocHistory, State, - LastCallDocEvent, NewRevisionDocEvent, IESG_SUBSTATE_TAGS, + NewRevisionDocEvent, IESG_SUBSTATE_TAGS, IESG_BALLOT_ACTIVE_STATES, IESG_STATCHG_CONFLREV_ACTIVE_STATES, IESG_CHARTER_ACTIVE_STATES ) from ietf.doc.fields import select2_id_doc_name_json @@ -849,31 +849,6 @@ def drafts_in_last_call(request): 'form':form, 'docs':results, 'meta':meta, 'pages':pages }) -def drafts_in_iesg_process(request): - states = State.objects.filter(type="draft-iesg").exclude(slug__in=('idexists', 'pub', 'dead', 'rfcqueue')) - title = "Documents in IESG process" - - grouped_docs = [] - - for s in states.order_by("order"): - docs = Document.objects.filter(type="draft", states=s).distinct().order_by("time").select_related("ad", "group", "group__parent") - if docs: - if s.slug == "lc": - for d in docs: - e = d.latest_event(LastCallDocEvent, type="sent_last_call") - # If we don't have an event, use an arbitrary date in the past (but not datetime.datetime.min, - # which causes problems with timezone conversions) - d.lc_expires = e.expires if e else datetime.datetime(1950, 1, 1) - docs = list(docs) - docs.sort(key=lambda d: d.lc_expires) - - grouped_docs.append((s, docs)) - - return render(request, 'doc/drafts_in_iesg_process.html', { - "grouped_docs": grouped_docs, - "title": title, - }) - def recent_drafts(request, days=7): slowcache = caches['slowpages'] cache_key = f'recentdraftsview{days}' diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html index 7f7e95a8734..cac709021ea 100644 --- a/ietf/templates/doc/ad_list.html +++ b/ietf/templates/doc/ad_list.html @@ -33,7 +33,10 @@

    IESG Dashboard

    are only shown to logged-in Area Directors. {% endif %} -

    Documents in IESG Processing

    +

    + Documents in IESG Processing + IESG view of Working Groups +

    {% for dt in metadata %}

    {{ dt.type.1 }} State Counts

    diff --git a/ietf/templates/doc/drafts_in_iesg_process.html b/ietf/templates/doc/drafts_in_iesg_process.html deleted file mode 100644 index d9b09e984e4..00000000000 --- a/ietf/templates/doc/drafts_in_iesg_process.html +++ /dev/null @@ -1,83 +0,0 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin %} -{% load ietf_filters static %} -{% load textfilters person_filters %} -{% block pagehead %} - -{% endblock %} -{% block title %}{{ title }}{% endblock %} -{% block content %} - {% origin %} -

    {{ title }}

    -

    This view is deprecated, and will soon redirect to a different representation

    -
    - - - - - - - - - - {% for state, docs in grouped_docs %} - - - - - - - {% for doc in docs %} - - - - - - - - {% endfor %} - - {% endfor %} -
    Area - {% if state.slug == "lc" %} - Expires at - {% else %} - Date - {% endif %} - DocumentIntended levelAD
    {{ state.name }}
    - {% if doc.area_acronym %} - {{ doc.area_acronym }} - {% endif %} - - {% if state.slug == "lc" %} - {% if doc.lc_expires %}{{ doc.lc_expires|date:"Y-m-d" }}{% endif %} - {% else %} - {{ doc.time|date:"Y-m-d" }} - {% endif %} - - {{ doc.name }} -
    - {{ doc.title }} - {% if doc.action_holders_enabled and doc.action_holders.exists %} -
    - Action holder{{ doc.documentactionholder_set.all|pluralize }}: - {% for action_holder in doc.documentactionholder_set.all %} - {% person_link action_holder.person title=action_holder.role_for_doc %}{% if action_holder|action_holder_badge %} {{ action_holder|action_holder_badge }}{% endif %}{% if not forloop.last %},{% endif %} - {% endfor %} - {% endif %} - {% if doc.note %} -
    - Note: {{ doc.note|urlize_ietf_docs|linkify|linebreaksbr }} - {% endif %} -     		- {% if doc.intended_std_level %} - {{ doc.intended_std_level.name }} - {% else %} - (None) - {% endif %} - {% person_link doc.ad %}
    -{% endblock %} -{% block js %} - -{% endblock %} \ No newline at end of file From 5e1f46d05cc23faa95b741f9133f40fe58c1cd46 Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Mon, 29 Sep 2025 15:47:23 +0200 Subject: [PATCH 033/313] feat: Distinguish I-Ds on WG plate from I-Ds on IESG plate (#9214) * Add "Outside of the WG Internet-Draft" when IESG state != idexists * No plural forms in the dividers * Use different search_heading * Use the right stream_id * Adding tests_info coverage for prepare_group_documents * fix: move identifying and sorting doxs with IESG into search utility. * fix: improve ordering conditional --------- Co-authored-by: Robert Sparks --- ietf/doc/utils_search.py | 9 ++++++++- ietf/group/tests_info.py | 22 +++++++++++++++++++++- ietf/group/views.py | 1 - 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py index cfc8a872f81..a5f461f9bb7 100644 --- a/ietf/doc/utils_search.py +++ b/ietf/doc/utils_search.py @@ -108,7 +108,10 @@ def fill_in_document_table_attributes(docs, have_telechat_date=False): d.search_heading = "Withdrawn Internet-Draft" d.expirable = False else: - d.search_heading = "%s Internet-Draft" % d.get_state() + if d.type_id == "draft" and d.stream_id == 'ietf' and d.get_state_slug('draft-iesg') != 'idexists': # values can be: ad-eval idexists approved rfcqueue dead iesg-eva + d.search_heading = "%s with the IESG Internet-Draft" % d.get_state() + else: + d.search_heading = "%s Internet-Draft" % d.get_state() if state_slug == "active": d.expirable = d.pk in expirable_pks else: @@ -221,6 +224,10 @@ def num(i): if d.type_id == "draft": res.append(num(["Active", "Expired", "Replaced", "Withdrawn", "RFC"].index(d.search_heading.split()[0]))) + if "with the IESG" in d.search_heading: + res.append("1") + else: + res.append("0") else: res.append(d.type_id); res.append("-"); diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index eb85860ece9..34f8500854b 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -27,7 +27,7 @@ from ietf.community.models import CommunityList from ietf.community.utils import reset_name_contains_index_for_rule -from ietf.doc.factories import WgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory +from ietf.doc.factories import WgDraftFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory from ietf.doc.models import Document, DocEvent, State from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils_charter import charter_name_for_group @@ -413,6 +413,7 @@ def test_group_documents(self): self.assertContains(r, draft3.name) for ah in draft3.action_holders.all(): self.assertContains(r, escape(ah.name)) + self.assertContains(r, "Active with the IESG Internet-Draft") # draft3 is pub-req hence should have such a divider self.assertContains(r, 'for 173 days', count=1) # the old_dah should be tagged self.assertContains(r, draft4.name) self.assertNotContains(r, draft5.name) @@ -425,6 +426,25 @@ def test_group_documents(self): q = PyQuery(r.content) self.assertTrue(any([draft2.name in x.attrib['href'] for x in q('table td a.track-untrack-doc')])) + # Let's also check the IRTF stream + rg = GroupFactory(type_id='rg') + setup_default_community_list_for_group(rg) + rgDraft = RgDraftFactory(group=rg) + rgDraft4 = RgDraftFactory(group=rg) + rgDraft4.set_state(State.objects.get(slug='irsg-w')) + rgDraft7 = RgDraftFactory(group=rg) + rgDraft7.set_state(State.objects.get(type='draft-stream-%s' % rgDraft7.stream_id, slug='dead')) + for url in group_urlreverse_list(rg, 'ietf.group.views.group_documents'): + with self.settings(DOC_ACTION_HOLDER_MAX_AGE_DAYS=20): + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertContains(r, rgDraft.name) + self.assertContains(r, rg.name) + self.assertContains(r, rg.acronym) + self.assertNotContains(r, draft3.name) # As draft3 is a WG draft, it should not be listed here + self.assertContains(r, rgDraft4.name) + self.assertNotContains(r, rgDraft7.name) + # test the txt version too while we're at it for url in group_urlreverse_list(group, 'ietf.group.views.group_documents_txt'): r = self.client.get(url) diff --git a/ietf/group/views.py b/ietf/group/views.py index bc785ff81e1..efe3eca15d5 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -443,7 +443,6 @@ def prepare_group_documents(request, group, clist): return docs, meta, docs_related, meta_related - def get_leadership(group_type): people = Person.objects.filter( role__name__slug="chair", From ba8b73190df413c39deaa6b546ad2bc5405fd86c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 30 Sep 2025 13:40:33 -0300 Subject: [PATCH 034/313] ci: DB persistence for blobdb, too --- k8s/settings_local.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index c1436e158b6..c09bd70c869 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -114,15 +114,17 @@ def _multiline_to_list(s): # Configure persistent connections. A setting of 0 is Django's default. _conn_max_age = os.environ.get("DATATRACKER_DB_CONN_MAX_AGE", "0") -# A string "none" means unlimited age. -DATABASES["default"]["CONN_MAX_AGE"] = ( - None if _conn_max_age.lower() == "none" else int(_conn_max_age) -) +for dbname in ["default", "blobdb"]: + # A string "none" means unlimited age. + DATABASES[dbname]["CONN_MAX_AGE"] = ( + None if _conn_max_age.lower() == "none" else int(_conn_max_age) + ) # Enable connection health checks if DATATRACKER_DB_CONN_HEALTH_CHECK is the string "true" _conn_health_checks = bool( os.environ.get("DATATRACKER_DB_CONN_HEALTH_CHECKS", "false").lower() == "true" ) -DATABASES["default"]["CONN_HEALTH_CHECKS"] = _conn_health_checks +for dbname in ["default", "blobdb"]: + DATABASES[dbname]["CONN_HEALTH_CHECKS"] = _conn_health_checks # DATATRACKER_ADMINS is a newline-delimited list of addresses parseable by email.utils.parseaddr _admins_str = os.environ.get("DATATRACKER_ADMINS", None) From d1cbdcb2afca5987706165a1928fece3da25a5ee Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 7 Oct 2025 14:54:08 -0300 Subject: [PATCH 035/313] chore: fix docker-compose comment (#9679) Allows the commented-out options to work if uncommented. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 100119c464a..8c6e0ea4866 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ services: db: image: ghcr.io/ietf-tools/datatracker-db:latest # build: - # context: .. + # context: . # dockerfile: docker/db.Dockerfile restart: unless-stopped volumes: From a8e8b9e95bedececcda0f54bedbe4de8f69d90a2 Mon Sep 17 00:00:00 2001 From: Absit Iniuria Date: Tue, 7 Oct 2025 19:02:37 +0100 Subject: [PATCH 036/313] feat: split liaison_statement_posted mailtrigger into outgoing and incoming (#9553) * fix: add new fixtures and mt slugs * fix: edit mt reverse func * chore: edit multiline and hash comments * fix: adjust migration * chore: remove stray whitespace --------- Co-authored-by: Robert Sparks --- ietf/liaisons/mails.py | 5 +- ietf/liaisons/tests.py | 39 +++++----- ietf/liaisons/views.py | 76 ++----------------- ..._statement_incoming_and_outgoing_posted.py | 72 ++++++++++++++++++ ietf/mailtrigger/utils.py | 71 +++++++++++++++++ ietf/name/fixtures/names.json | 45 ++++++++++- 6 files changed, 217 insertions(+), 91 deletions(-) create mode 100644 ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py diff --git a/ietf/liaisons/mails.py b/ietf/liaisons/mails.py index 8708c8a078d..878aada576b 100644 --- a/ietf/liaisons/mails.py +++ b/ietf/liaisons/mails.py @@ -14,7 +14,10 @@ def send_liaison_by_email(request, liaison): subject = 'New Liaison Statement, "%s"' % (liaison.title) from_email = settings.LIAISON_UNIVERSAL_FROM - (to_email, cc) = gather_address_lists('liaison_statement_posted',liaison=liaison) + if liaison.is_outgoing(): + (to_email, cc) = gather_address_lists('liaison_statement_posted_outgoing',liaison=liaison) + else: + (to_email, cc) = gather_address_lists('liaison_statement_posted_incoming',liaison=liaison) bcc = ['statements@ietf.org'] body = render_to_string('liaisons/liaison_mail.txt', dict(liaison=liaison)) diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index fd1c22be775..5478f6c3028 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -112,61 +112,61 @@ def test_help_pages(self): class UnitTests(TestCase): - def test_get_cc(self): - from ietf.liaisons.views import get_cc,EMAIL_ALIASES + def test_get_contacts_for_liaison_messages_for_group_primary(self): + from ietf.mailtrigger.utils import get_contacts_for_liaison_messages_for_group_primary,EMAIL_ALIASES # test IETF - cc = get_cc(Group.objects.get(acronym='ietf')) + cc = get_contacts_for_liaison_messages_for_group_primary(Group.objects.get(acronym='ietf')) self.assertTrue(EMAIL_ALIASES['IESG'] in cc) self.assertTrue(EMAIL_ALIASES['IETFCHAIR'] in cc) # test IAB - cc = get_cc(Group.objects.get(acronym='iab')) + cc = get_contacts_for_liaison_messages_for_group_primary(Group.objects.get(acronym='iab')) self.assertTrue(EMAIL_ALIASES['IAB'] in cc) self.assertTrue(EMAIL_ALIASES['IABCHAIR'] in cc) # test an Area area = Group.objects.filter(type='area').first() - cc = get_cc(area) + cc = get_contacts_for_liaison_messages_for_group_primary(area) self.assertTrue(EMAIL_ALIASES['IETFCHAIR'] in cc) self.assertTrue(contacts_from_roles([area.ad_role()]) in cc) # test a Working Group wg = Group.objects.filter(type='wg').first() - cc = get_cc(wg) + cc = get_contacts_for_liaison_messages_for_group_primary(wg) self.assertTrue(contacts_from_roles([wg.parent.ad_role()]) in cc) self.assertTrue(contacts_from_roles([wg.get_chair()]) in cc) # test an SDO sdo = RoleFactory(name_id='liaiman',group__type_id='sdo',).group - cc = get_cc(sdo) + cc = get_contacts_for_liaison_messages_for_group_primary(sdo) self.assertTrue(contacts_from_roles([sdo.role_set.filter(name='liaiman').first()]) in cc) # test a cc_contact role cc_contact_role = RoleFactory(name_id='liaison_cc_contact', group=sdo) - cc = get_cc(sdo) + cc = get_contacts_for_liaison_messages_for_group_primary(sdo) self.assertIn(contact_email_from_role(cc_contact_role), cc) - def test_get_contacts_for_group(self): - from ietf.liaisons.views import get_contacts_for_group, EMAIL_ALIASES + def test_get_contacts_for_liaison_messages_for_group_secondary(self): + from ietf.mailtrigger.utils import get_contacts_for_liaison_messages_for_group_secondary,EMAIL_ALIASES - # test explicit + # test explicit group contacts sdo = GroupFactory(type_id='sdo') contact_email = RoleFactory(name_id='liaison_contact', group=sdo).email.address - contacts = get_contacts_for_group(sdo) + contacts = get_contacts_for_liaison_messages_for_group_secondary(sdo) self.assertIsNotNone(contact_email) self.assertIn(contact_email, contacts) # test area area = Group.objects.filter(type='area').first() - contacts = get_contacts_for_group(area) + contacts = get_contacts_for_liaison_messages_for_group_secondary(area) self.assertTrue(area.ad_role().email.address in contacts) # test wg wg = Group.objects.filter(type='wg').first() - contacts = get_contacts_for_group(wg) + contacts = get_contacts_for_liaison_messages_for_group_secondary(wg) self.assertTrue(wg.get_chair().email.address in contacts) # test ietf - contacts = get_contacts_for_group(Group.objects.get(acronym='ietf')) + contacts = get_contacts_for_liaison_messages_for_group_secondary(Group.objects.get(acronym='ietf')) self.assertTrue(EMAIL_ALIASES['IETFCHAIR'] in contacts) # test iab - contacts = get_contacts_for_group(Group.objects.get(acronym='iab')) + contacts = get_contacts_for_liaison_messages_for_group_secondary(Group.objects.get(acronym='iab')) self.assertTrue(EMAIL_ALIASES['IABCHAIR'] in contacts) # test iesg - contacts = get_contacts_for_group(Group.objects.get(acronym='iesg')) + contacts = get_contacts_for_liaison_messages_for_group_secondary(Group.objects.get(acronym='iesg')) self.assertTrue(EMAIL_ALIASES['IESG'] in contacts) def test_needs_approval(self): @@ -786,8 +786,11 @@ def test_add_incoming_liaison(self): self.assertTrue("Liaison Statement" in outbox[-1]["Subject"]) self.assertTrue('to_contacts@' in outbox[-1]['To']) + self.assertTrue(submitter.email_address(), outbox[-1]['To']) self.assertTrue('cc@' in outbox[-1]['Cc']) + + def test_add_outgoing_liaison(self): RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman') wg = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group @@ -867,6 +870,8 @@ def test_add_outgoing_liaison(self): self.assertEqual(len(outbox), mailbox_before + 1) self.assertTrue("Liaison Statement" in outbox[-1]["Subject"]) self.assertTrue('aread@' in outbox[-1]['To']) + self.assertTrue(submitter.email_address(), outbox[-1]['Cc']) + def test_add_outgoing_liaison_unapproved_post_only(self): RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman') diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py index 6a6f5797143..f54a0233574 100644 --- a/ietf/liaisons/views.py +++ b/ietf/liaisons/views.py @@ -27,14 +27,6 @@ from ietf.name.models import LiaisonStatementTagName from ietf.utils.response import permission_denied -EMAIL_ALIASES = { - "IETFCHAIR": "The IETF Chair ", - "IESG": "The IESG ", - "IAB": "The IAB ", - "IABCHAIR": "The IAB Chair ", -} - - # ------------------------------------------------- # Helper Functions # ------------------------------------------------- @@ -94,64 +86,6 @@ def contacts_from_roles(roles): emails = [ contact_email_from_role(r) for r in roles ] return ','.join(emails) -def get_cc(group): - '''Returns list of emails to use as CC for group. Simplified refactor of IETFHierarchy - get_cc() and get_from_cc() - ''' - emails = [] - - # role based CCs - if group.acronym in ('ietf','iesg'): - emails.append(EMAIL_ALIASES['IESG']) - emails.append(EMAIL_ALIASES['IETFCHAIR']) - elif group.acronym in ('iab'): - emails.append(EMAIL_ALIASES['IAB']) - emails.append(EMAIL_ALIASES['IABCHAIR']) - elif group.type_id == 'area': - emails.append(EMAIL_ALIASES['IETFCHAIR']) - ad_roles = group.role_set.filter(name='ad') - emails.extend([ contact_email_from_role(r) for r in ad_roles ]) - elif group.type_id == 'wg': - ad_roles = group.parent.role_set.filter(name='ad') - emails.extend([ contact_email_from_role(r) for r in ad_roles ]) - chair_roles = group.role_set.filter(name='chair') - emails.extend([ contact_email_from_role(r) for r in chair_roles ]) - if group.list_email: - emails.append('{} Discussion List <{}>'.format(group.name,group.list_email)) - elif group.type_id == 'sdo': - liaiman_roles = group.role_set.filter(name='liaiman') - emails.extend([ contact_email_from_role(r) for r in liaiman_roles ]) - - # explicit CCs - liaison_cc_roles = group.role_set.filter(name='liaison_cc_contact') - emails.extend([ contact_email_from_role(r) for r in liaison_cc_roles ]) - - return emails - -def get_contacts_for_group(group): - '''Returns default contacts for groups as a comma separated string''' - # use explicit default contacts if defined - explicit_contacts = contacts_from_roles(group.role_set.filter(name='liaison_contact')) - if explicit_contacts: - return explicit_contacts - - # otherwise construct based on group type - contacts = [] - if group.type_id == 'area': - roles = group.role_set.filter(name='ad') - contacts.append(contacts_from_roles(roles)) - elif group.type_id == 'wg': - roles = group.role_set.filter(name='chair') - contacts.append(contacts_from_roles(roles)) - elif group.acronym == 'ietf': - contacts.append(EMAIL_ALIASES['IETFCHAIR']) - elif group.acronym == 'iab': - contacts.append(EMAIL_ALIASES['IABCHAIR']) - elif group.acronym == 'iesg': - contacts.append(EMAIL_ALIASES['IESG']) - - return ','.join(contacts) - def get_details_tabs(stmt, selected): return [ t + (t[0].lower() == selected.lower(),) @@ -207,6 +141,8 @@ def post_only(group,person): # ------------------------------------------------- @can_submit_liaison_required def ajax_get_liaison_info(request): + from ietf.mailtrigger.utils import get_contacts_for_liaison_messages_for_group_primary,get_contacts_for_liaison_messages_for_group_secondary + '''Returns dictionary of info to update entry form given the groups that have been selected ''' @@ -229,14 +165,14 @@ def ajax_get_liaison_info(request): result = {'response_contacts':[],'to_contacts': [], 'cc': [], 'needs_approval': False, 'post_only': False, 'full_list': []} for group in from_groups: - cc.extend(get_cc(group)) + cc.extend(get_contacts_for_liaison_messages_for_group_primary(group)) does_need_approval.append(needs_approval(group,person)) can_post_only.append(post_only(group,person)) - response_contacts.append(get_contacts_for_group(group)) + response_contacts.append(get_contacts_for_liaison_messages_for_group_secondary(group)) for group in to_groups: - cc.extend(get_cc(group)) - to_contacts.append(get_contacts_for_group(group)) + cc.extend(get_contacts_for_liaison_messages_for_group_primary(group)) + to_contacts.append(get_contacts_for_liaison_messages_for_group_secondary(group)) # if there are from_groups and any need approval if does_need_approval: diff --git a/ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py b/ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py new file mode 100644 index 00000000000..189a783a2e9 --- /dev/null +++ b/ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py @@ -0,0 +1,72 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + Mailtrigger = apps.get_model("mailtrigger", "MailTrigger") + Recipient = apps.get_model("mailtrigger", "Recipient") + recipients_to = Recipient.objects.get(pk="liaison_to_contacts") + recipients_cc = list( + Recipient.objects.filter( + slug__in=( + "liaison_cc", + "liaison_coordinators", + "liaison_response_contacts", + "liaison_technical_contacts", + ) + ) + ) + recipient_from = Recipient.objects.get(pk="liaison_from_contact") + + liaison_posted_outgoing = Mailtrigger.objects.create( + slug="liaison_statement_posted_outgoing", + desc="Recipients for a message when a new outgoing liaison statement is posted", + ) + liaison_posted_outgoing.to.add(recipients_to) + liaison_posted_outgoing.cc.add(*recipients_cc) + liaison_posted_outgoing.cc.add(recipient_from) + + liaison_posted_incoming = Mailtrigger.objects.create( + slug="liaison_statement_posted_incoming", + desc="Recipients for a message when a new incoming liaison statement is posted", + ) + liaison_posted_incoming.to.add(recipients_to) + liaison_posted_incoming.cc.add(*recipients_cc) + + Mailtrigger.objects.filter(slug=("liaison_statement_posted")).delete() + + +def reverse(apps, schema_editor): + Mailtrigger = apps.get_model("mailtrigger", "MailTrigger") + Recipient = apps.get_model("mailtrigger", "Recipient") + + Mailtrigger.objects.filter( + slug__in=( + "liaison_statement_posted_outgoing", + "liaison_statement_posted_incoming", + ) + ).delete() + + liaison_statement_posted = Mailtrigger.objects.create( + slug="liaison_statement_posted", + desc="Recipients for a message when a new liaison statement is posted", + ) + + liaison_to_contacts = Recipient.objects.get(slug="liaison_to_contacts") + recipients_ccs = Recipient.objects.filter( + slug__in=( + "liaison_cc", + "liaison_coordinators", + "liaison_response_contacts", + "liaison_technical_contacts", + ) + ) + liaison_statement_posted.to.add(liaison_to_contacts) + liaison_statement_posted.cc.add(*recipients_ccs) + + +class Migration(migrations.Migration): + dependencies = [("mailtrigger", "0007_historicalrecipient_historicalmailtrigger")] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/mailtrigger/utils.py b/ietf/mailtrigger/utils.py index 9915eae3fd9..bcdaf5e44ea 100644 --- a/ietf/mailtrigger/utils.py +++ b/ietf/mailtrigger/utils.py @@ -9,6 +9,14 @@ from ietf.utils.mail import excludeaddrs +EMAIL_ALIASES = { + "IETFCHAIR": "The IETF Chair ", + "IESG": "The IESG ", + "IAB": "The IAB ", + "IABCHAIR": "The IAB Chair ", +} + + class AddrLists(namedtuple("AddrLists", ["to", "cc"])): __slots__ = () @@ -66,6 +74,69 @@ def get_mailtrigger(slug, create_from_slug_if_not_exists, desc_if_not_exists): return mailtrigger +def get_contacts_for_liaison_messages_for_group_primary(group): + from ietf.liaisons.views import contact_email_from_role + + '''Returns list of emails to use in liaison message for group + ''' + emails = [] + + # role based emails + if group.acronym in ('ietf','iesg'): + emails.append(EMAIL_ALIASES['IESG']) + emails.append(EMAIL_ALIASES['IETFCHAIR']) + elif group.acronym in ('iab'): + emails.append(EMAIL_ALIASES['IAB']) + emails.append(EMAIL_ALIASES['IABCHAIR']) + elif group.type_id == 'area': + emails.append(EMAIL_ALIASES['IETFCHAIR']) + ad_roles = group.role_set.filter(name='ad') + emails.extend([ contact_email_from_role(r) for r in ad_roles ]) + elif group.type_id == 'wg': + ad_roles = group.parent.role_set.filter(name='ad') + emails.extend([ contact_email_from_role(r) for r in ad_roles ]) + chair_roles = group.role_set.filter(name='chair') + emails.extend([ contact_email_from_role(r) for r in chair_roles ]) + if group.list_email: + emails.append('{} Discussion List <{}>'.format(group.name,group.list_email)) + elif group.type_id == 'sdo': + liaiman_roles = group.role_set.filter(name='liaiman') + emails.extend([ contact_email_from_role(r) for r in liaiman_roles ]) + + # explicit CCs + liaison_cc_roles = group.role_set.filter(name='liaison_cc_contact') + emails.extend([ contact_email_from_role(r) for r in liaison_cc_roles ]) + + return emails + + +def get_contacts_for_liaison_messages_for_group_secondary(group): + from ietf.liaisons.views import contacts_from_roles + + '''Returns default contacts for groups as a comma separated string''' + # use explicit default contacts if defined + explicit_contacts = contacts_from_roles(group.role_set.filter(name='liaison_contact')) + if explicit_contacts: + return explicit_contacts + + # otherwise construct based on group type + contacts = [] + if group.type_id == 'area': + roles = group.role_set.filter(name='ad') + contacts.append(contacts_from_roles(roles)) + elif group.type_id == 'wg': + roles = group.role_set.filter(name='chair') + contacts.append(contacts_from_roles(roles)) + elif group.acronym == 'ietf': + contacts.append(EMAIL_ALIASES['IETFCHAIR']) + elif group.acronym == 'iab': + contacts.append(EMAIL_ALIASES['IABCHAIR']) + elif group.acronym == 'iesg': + contacts.append(EMAIL_ALIASES['IESG']) + + return ','.join(contacts) + + def gather_relevant_expansions(**kwargs): def starts_with(prefix): return MailTrigger.objects.filter(slug__startswith=prefix).values_list( diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index c94e15a4591..58deb01f0ca 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -2638,11 +2638,24 @@ "order": 0, "slug": "historic", "type": "statement", - "used": true + "used": false }, "model": "doc.state", "pk": 182 }, + { + "fields": { + "desc": "The statement is no longer active", + "name": "Inactive", + "next_states": [], + "order": 0, + "slug": "inactive", + "type": "statement", + "used": true + }, + "model": "doc.state", + "pk": 183 + }, { "fields": { "label": "State" @@ -5520,13 +5533,31 @@ "liaison_response_contacts", "liaison_technical_contacts" ], - "desc": "Recipient for a message when a new liaison statement is posted", + "desc": "Recipients for a message when a new incoming liaison statement is posted", "to": [ + "liaison_from_contact", "liaison_to_contacts" ] }, "model": "mailtrigger.mailtrigger", - "pk": "liaison_statement_posted" + "pk": "liaison_statement_posted_incoming" + }, + { + "fields": { + "cc": [ + "liaison_cc", + "liaison_coordinators", + "liaison_from_contact", + "liaison_response_contacts", + "liaison_technical_contacts" + ], + "desc": "Recipients for a message when a new outgoing liaison statement is posted", + "to": [ + "liaison_to_contacts" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "liaison_statement_posted_outgoing" }, { "fields": { @@ -7068,6 +7099,14 @@ "model": "mailtrigger.recipient", "pk": "liaison_coordinators" }, + { + "fields": { + "desc": "Email address of the formal sender of the statement", + "template": "{{liaison.from_contact}}" + }, + "model": "mailtrigger.recipient", + "pk": "liaison_from_contact" + }, { "fields": { "desc": "The assigned liaison manager for an external group ", From 8fbbc55ec3cb87f528953da33e8c7194c2b75afd Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 7 Oct 2025 15:13:16 -0300 Subject: [PATCH 037/313] fix: keep day visible in timeslot editor (#9653) --- ietf/templates/meeting/timeslot_edit.html | 44 ++++++++++++----------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/ietf/templates/meeting/timeslot_edit.html b/ietf/templates/meeting/timeslot_edit.html index 11691ba6dd9..3259dba9daf 100644 --- a/ietf/templates/meeting/timeslot_edit.html +++ b/ietf/templates/meeting/timeslot_edit.html @@ -11,20 +11,22 @@ {% endcomment %} .timeslot-edit { overflow: auto; height: max(30rem, calc(100vh - 25rem));} .tstable { width: 100%; border-collapse: separate; } {# "separate" to ensure sticky cells keep their borders #} -.tstable thead { position: sticky; top: 0; z-index: 3; background-color: white;} -.tstable th:first-child, .tstable td:first-child { - background-color: white; {# needs to match the lighter of the striped-table colors! #} -position: sticky; -left: 0; - z-index: 2; {# render above other cells / borders but below thead (z-index 3, above) #} -} -.tstable tbody > tr:nth-of-type(odd) > th:first-child { - background-color: rgb(249, 249, 249); {# needs to match the darker of the striped-table colors! #} -} -.tstable th { white-space: nowrap;} -.tstable td { white-space: nowrap;} -.capacity { font-size:80%; font-weight: normal;} -a.new-timeslot-link { color: lightgray; font-size: large;} + .tstable tr th:first-child { min-width: 25rem; max-width: 25rem; overflow: hidden; } + .tstable thead { position: sticky; top: 0; z-index: 3; background-color: white;} + .tstable thead th span.day { position: sticky; left: 25.5rem; } + .tstable th:first-child, .tstable td:first-child { + background-color: white; {# needs to match the lighter of the striped-table colors! #} + position: sticky; + left: 0; + z-index: 2; {# render above other cells / borders but below thead (z-index 3, above) #} + } + .tstable tbody > tr:nth-of-type(odd) > th:first-child { + background-color: rgb(249, 249, 249); {# needs to match the darker of the striped-table colors! #} + } + .tstable th { white-space: nowrap;} + .tstable td { white-space: nowrap;} + .capacity { font-size:80%; font-weight: normal;} + a.new-timeslot-link { color: lightgray; font-size: large;} {% endblock %} {% block content %} {% origin %} @@ -84,12 +86,14 @@

    {% for day in time_slices %} - {{ day|date:'D' }} ({{ day }}) - - + + {{ day|date:'D' }} ({{ day }}) + + + {% endfor %} {% endif %} From 24101bb8ca85cfb3a5c47d7f9ed283cc6fb5bc0e Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 9 Oct 2025 13:49:40 -0500 Subject: [PATCH 038/313] feat: json snapshots of ipr statements (#9684) --- ietf/ipr/tests.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++ ietf/ipr/urls.py | 1 + ietf/ipr/utils.py | 24 ++++++++++++++++++-- ietf/ipr/views.py | 7 +++++- 4 files changed, 86 insertions(+), 3 deletions(-) diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py index 74fa540126c..4146fbd4c16 100644 --- a/ietf/ipr/tests.py +++ b/ietf/ipr/tests.py @@ -3,6 +3,7 @@ import datetime +import json from unittest import mock import re @@ -15,6 +16,8 @@ from django.urls import reverse as urlreverse from django.utils import timezone +from django.db.models import Max + import debug # pyflakes:ignore from ietf.api.views import EmailIngestionError @@ -45,6 +48,7 @@ from ietf.mailtrigger.utils import gather_address_lists from ietf.message.factories import MessageFactory from ietf.message.models import Message +from ietf.person.factories import PersonFactory from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import TestCase, login_testing_unauthorized from ietf.utils.text import text_to_dict @@ -1113,3 +1117,56 @@ def test_patent_details_required_unless_blanket(self): val = self.data.pop(pf) self.assertTrue(HolderIprDisclosureForm(data=self.data).is_valid()) self.data[pf] = val + +class JsonSnapshotTests(TestCase): + def test_json_snapshot(self): + h = HolderIprDisclosureFactory() + url = urlreverse("ietf.ipr.views.json_snapshot", kwargs=dict(id=h.id)) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + dump = json.loads(r.content) + self.assertCountEqual( + [o["model"] for o in dump], + ["ipr.holderiprdisclosure", "ipr.iprdisclosurebase", "person.person"], + ) + h.docs.add(WgRfcFactory()) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + dump = json.loads(r.content) + self.assertCountEqual( + [o["model"] for o in dump], + [ + "ipr.holderiprdisclosure", + "ipr.iprdisclosurebase", + "ipr.iprdocrel", + "person.person", + ], + ) + IprEventFactory( + disclosure=h, + message=MessageFactory(by=PersonFactory()), + in_reply_to=MessageFactory(), + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + dump = json.loads(r.content) + self.assertCountEqual( + [o["model"] for o in dump], + [ + "ipr.holderiprdisclosure", + "ipr.iprdisclosurebase", + "ipr.iprdocrel", + "ipr.iprevent", + "message.message", + "message.message", + "person.person", + "person.person", + "person.person", + "person.person", + ], + ) + no_such_ipr_id = IprDisclosureBase.objects.aggregate(Max("id"))["id__max"] + 1 + url = urlreverse("ietf.ipr.views.json_snapshot", kwargs=dict(id=no_such_ipr_id)) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) diff --git a/ietf/ipr/urls.py b/ietf/ipr/urls.py index 84ed04a66bf..2c8a26c6240 100644 --- a/ietf/ipr/urls.py +++ b/ietf/ipr/urls.py @@ -21,6 +21,7 @@ url(r'^(?P\d+)/notify/(?Pupdate|posted)/$', views.notify), url(r'^(?P\d+)/post/$', views.post), url(r'^(?P\d+)/state/$', views.state), + url(r'^(?P\d+)/json-snapshot/$', views.json_snapshot), url(r'^update/$', RedirectView.as_view(url=reverse_lazy('ietf.ipr.views.showlist'), permanent=True)), url(r'^update/(?P\d+)/$', views.update), url(r'^new-(?P<_type>(specific|generic|general|third-party))/$', views.new), diff --git a/ietf/ipr/utils.py b/ietf/ipr/utils.py index 7e569a1d1df..bcbb0522604 100644 --- a/ietf/ipr/utils.py +++ b/ietf/ipr/utils.py @@ -1,11 +1,16 @@ -# Copyright The IETF Trust 2014-2020, All Rights Reserved +# Copyright The IETF Trust 2014-2025, All Rights Reserved # -*- coding: utf-8 -*- +import json +import debug # pyflakes:ignore + from textwrap import dedent +from django.core import serializers + from ietf.ipr.mail import process_response_email, UndeliverableIprResponseError -import debug # pyflakes:ignore +from ietf.ipr.models import IprDocRel def get_genitive(name): """Return the genitive form of name""" @@ -85,3 +90,18 @@ def ingest_response_email(message: bytes): email_original_message=message, email_attach_traceback=True, ) from err + +def json_dump_disclosure(disclosure): + objs = set() + objs.add(disclosure) + objs.add(disclosure.iprdisclosurebase_ptr) + objs.add(disclosure.by) + objs.update(IprDocRel.objects.filter(disclosure=disclosure)) + objs.update(disclosure.iprevent_set.all()) + objs.update([i.by for i in disclosure.iprevent_set.all()]) + objs.update([i.message for i in disclosure.iprevent_set.all() if i.message ]) + objs.update([i.message.by for i in disclosure.iprevent_set.all() if i.message ]) + objs.update([i.in_reply_to for i in disclosure.iprevent_set.all() if i.in_reply_to ]) + objs.update([i.in_reply_to.by for i in disclosure.iprevent_set.all() if i.in_reply_to ]) + objs = sorted(list(objs),key=lambda o:o.__class__.__name__) + return json.dumps(json.loads(serializers.serialize("json",objs)),indent=4) diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index 08979a39725..8eb3affbc0b 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -32,7 +32,7 @@ NonDocSpecificIprDisclosure, IprDocRel, RelatedIpr,IprEvent) from ietf.ipr.utils import (get_genitive, get_ipr_summary, - iprs_from_docs, related_docs) + iprs_from_docs, json_dump_disclosure, related_docs) from ietf.mailtrigger.utils import gather_address_lists from ietf.message.models import Message from ietf.message.utils import infer_message @@ -901,3 +901,8 @@ def update(request, id): child = ipr.get_child() type = class_to_type[child.__class__.__name__] return new(request, type, updates=id) + +@role_required("Secretariat") +def json_snapshot(request, id): + obj = get_object_or_404(IprDisclosureBase,id=id).get_child() + return HttpResponse(json_dump_disclosure(obj),content_type="application/json") From 9d2fa7a32c6dae35de28c3a5f62ca5d762baef3c Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 9 Oct 2025 17:04:13 -0500 Subject: [PATCH 039/313] feat: track deleted ipr disclosures (#9691) * feat: track deleted ipr disclosures * fix: unique constraint on removed_id --- ietf/ipr/admin.py | 23 ++++++++++++--- .../migrations/0005_removediprdisclosure.py | 28 +++++++++++++++++++ .../migrations/0006_already_removed_ipr.py | 24 ++++++++++++++++ ietf/ipr/models.py | 6 +++- ietf/ipr/resources.py | 19 +++++++++++-- ietf/ipr/tests.py | 22 ++++++++++++++- ietf/ipr/views.py | 11 ++++++-- ietf/templates/ipr/deleted.html | 16 +++++++++++ 8 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 ietf/ipr/migrations/0005_removediprdisclosure.py create mode 100644 ietf/ipr/migrations/0006_already_removed_ipr.py create mode 100644 ietf/templates/ipr/deleted.html diff --git a/ietf/ipr/admin.py b/ietf/ipr/admin.py index afc1952d72d..1a8a908dcdf 100644 --- a/ietf/ipr/admin.py +++ b/ietf/ipr/admin.py @@ -1,13 +1,22 @@ -# Copyright The IETF Trust 2010-2020, All Rights Reserved +# Copyright The IETF Trust 2010-2025, All Rights Reserved # -*- coding: utf-8 -*- from django import forms from django.contrib import admin from ietf.name.models import DocRelationshipName -from ietf.ipr.models import (IprDisclosureBase, IprDocRel, IprEvent, - RelatedIpr, HolderIprDisclosure, ThirdPartyIprDisclosure, GenericIprDisclosure, - NonDocSpecificIprDisclosure, LegacyMigrationIprEvent) +from ietf.ipr.models import ( + IprDisclosureBase, + IprDocRel, + IprEvent, + RelatedIpr, + HolderIprDisclosure, + RemovedIprDisclosure, + ThirdPartyIprDisclosure, + GenericIprDisclosure, + NonDocSpecificIprDisclosure, + LegacyMigrationIprEvent, +) # ------------------------------------------------------ # ModelAdmins @@ -110,3 +119,9 @@ class LegacyMigrationIprEventAdmin(admin.ModelAdmin): list_filter = ['time', 'type', 'response_due'] raw_id_fields = ['by', 'disclosure', 'message', 'in_reply_to'] admin.site.register(LegacyMigrationIprEvent, LegacyMigrationIprEventAdmin) + +class RemovedIprDisclosureAdmin(admin.ModelAdmin): + pass + + +admin.site.register(RemovedIprDisclosure, RemovedIprDisclosureAdmin) diff --git a/ietf/ipr/migrations/0005_removediprdisclosure.py b/ietf/ipr/migrations/0005_removediprdisclosure.py new file mode 100644 index 00000000000..400a2645797 --- /dev/null +++ b/ietf/ipr/migrations/0005_removediprdisclosure.py @@ -0,0 +1,28 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("ipr", "0004_holderiprdisclosure_is_blanket_disclosure"), + ] + + operations = [ + migrations.CreateModel( + name="RemovedIprDisclosure", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("removed_id", models.PositiveBigIntegerField(unique=True)), + ("reason", models.TextField()), + ], + ), + ] diff --git a/ietf/ipr/migrations/0006_already_removed_ipr.py b/ietf/ipr/migrations/0006_already_removed_ipr.py new file mode 100644 index 00000000000..0e2dbc63ebb --- /dev/null +++ b/ietf/ipr/migrations/0006_already_removed_ipr.py @@ -0,0 +1,24 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from django.db import migrations + + +def forward(apps, schema_editor): + RemovedIprDisclosure = apps.get_model("ipr", "RemovedIprDisclosure") + for id in (6544, 6068): + RemovedIprDisclosure.objects.create( + removed_id=id, + reason="This IPR disclosure was removed as objectively false.", + ) + + +def reverse(apps, schema_editor): + RemovedIprDisclosure = apps.get_model("ipr", "RemovedIprDisclosure") + RemovedIprDisclosure.objects.all().delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("ipr", "0005_removediprdisclosure"), + ] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/ipr/models.py b/ietf/ipr/models.py index 2d81eb4b423..ea148c27045 100644 --- a/ietf/ipr/models.py +++ b/ietf/ipr/models.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2023, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -270,3 +270,7 @@ class LegacyMigrationIprEvent(IprEvent): """A subclass of IprEvent specifically for capturing contents of legacy_url_0, the text of a disclosure submitted by email""" pass + +class RemovedIprDisclosure(models.Model): + removed_id = models.PositiveBigIntegerField(unique=True) + reason = models.TextField() diff --git a/ietf/ipr/resources.py b/ietf/ipr/resources.py index 0d8421cdecd..c4d2c436e6b 100644 --- a/ietf/ipr/resources.py +++ b/ietf/ipr/resources.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2015-2019, All Rights Reserved +# Copyright The IETF Trust 2015-2025, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2015-03-21 14:05 PDT @@ -11,7 +11,7 @@ from ietf import api -from ietf.ipr.models import ( IprDisclosureBase, IprDocRel, HolderIprDisclosure, ThirdPartyIprDisclosure, +from ietf.ipr.models import ( IprDisclosureBase, IprDocRel, HolderIprDisclosure, RemovedIprDisclosure, ThirdPartyIprDisclosure, RelatedIpr, NonDocSpecificIprDisclosure, GenericIprDisclosure, IprEvent, LegacyMigrationIprEvent ) from ietf.person.resources import PersonResource @@ -295,3 +295,18 @@ class Meta: } api.ipr.register(LegacyMigrationIprEventResource()) + + +class RemovedIprDisclosureResource(ModelResource): + class Meta: + queryset = RemovedIprDisclosure.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'removediprdisclosure' + ordering = ['id', ] + filtering = { + "id": ALL, + "removed_id": ALL, + "reason": ALL, + } +api.ipr.register(RemovedIprDisclosureResource()) diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py index 4146fbd4c16..53a599e2def 100644 --- a/ietf/ipr/tests.py +++ b/ietf/ipr/tests.py @@ -41,7 +41,7 @@ from ietf.ipr.forms import DraftForm, HolderIprDisclosureForm from ietf.ipr.mail import (process_response_email, get_reply_to, get_update_submitter_emails, get_pseudo_submitter, get_holders, get_update_cc_addrs, UndeliverableIprResponseError) -from ietf.ipr.models import (IprDisclosureBase, GenericIprDisclosure, HolderIprDisclosure, +from ietf.ipr.models import (IprDisclosureBase, GenericIprDisclosure, HolderIprDisclosure, RemovedIprDisclosure, ThirdPartyIprDisclosure, IprEvent) from ietf.ipr.templatetags.ipr_filters import no_revisions_message from ietf.ipr.utils import get_genitive, get_ipr_summary, ingest_response_email @@ -129,6 +129,26 @@ def test_showlist(self): self.assertContains(r, "removed as objectively false") ipr.delete() + def test_show_delete(self): + ipr = HolderIprDisclosureFactory() + removed = RemovedIprDisclosure.objects.create( + removed_id=ipr.pk, reason="Removed for reasons" + ) + url = urlreverse("ietf.ipr.views.show", kwargs=dict(id=removed.removed_id)) + r = self.client.get(url) + self.assertContains(r, "Removed for reasons") + q = PyQuery(r.content) + self.assertEqual(len(q("#deletion_warning")), 0) + self.client.login(username="secretary", password="secretary+password") + r = self.client.get(url) + self.assertContains(r, "Removed for reasons") + q = PyQuery(r.content) + self.assertEqual(len(q("#deletion_warning")), 1) + ipr.delete() + r = self.client.get(url) + self.assertContains(r, "Removed for reasons") + q = PyQuery(r.content) + self.assertEqual(len(q("#deletion_warning")), 0) def test_show_posted(self): ipr = HolderIprDisclosureFactory() diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index 8eb3affbc0b..665c99dc43b 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -28,7 +28,7 @@ AddCommentForm, AddEmailForm, NotifyForm, StateForm, NonDocSpecificIprDisclosureForm, GenericIprDisclosureForm) from ietf.ipr.models import (IprDisclosureStateName, IprDisclosureBase, - HolderIprDisclosure, GenericIprDisclosure, ThirdPartyIprDisclosure, + HolderIprDisclosure, GenericIprDisclosure, RemovedIprDisclosure, ThirdPartyIprDisclosure, NonDocSpecificIprDisclosure, IprDocRel, RelatedIpr,IprEvent) from ietf.ipr.utils import (get_genitive, get_ipr_summary, @@ -817,7 +817,14 @@ def get_details_tabs(ipr, selected): def show(request, id): """View of individual declaration""" - ipr = get_object_or_404(IprDisclosureBase, id=id).get_child() + ipr = IprDisclosureBase.objects.filter(id=id) + removed = RemovedIprDisclosure.objects.filter(removed_id=id) + if removed.exists(): + return render(request, "ipr/deleted.html", {"removed": removed.get(), "ipr": ipr}) + if not ipr.exists(): + raise Http404 + else: + ipr = ipr.get().get_child() if not has_role(request.user, 'Secretariat'): if ipr.state.slug in ['removed', 'removed_objfalse']: return render(request, "ipr/removed.html", { diff --git a/ietf/templates/ipr/deleted.html b/ietf/templates/ipr/deleted.html new file mode 100644 index 00000000000..24f696ebca0 --- /dev/null +++ b/ietf/templates/ipr/deleted.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2015-2023, All Rights Reserved #} +{% load ietf_filters origin %} +{% block title %}Removed IPR Disclosure{% endblock %} +{% block content %} + {% origin %} +

    Removed IPR disclosure

    +

    + {{ removed.reason }} +

    + {% if user|has_role:"Secretariat" and ipr.exists %} +

    + This disclosure has not yet been deleted and parts of its content is available through, e.g, the history view and the /api/v1 views. +

    + {% endif %} + {% endblock %} \ No newline at end of file From ed6b061cfe4279328dbb7b914f5f7f76644521f9 Mon Sep 17 00:00:00 2001 From: Ryan Cross Date: Fri, 10 Oct 2025 05:43:24 -0700 Subject: [PATCH 040/313] chore: merge feat/sreq to main (#9697) * refactor: move session request tool to ietf.meeting and restyle (#9617) * refactor: move session request tool to ietf.meeting and restyle to match standard Datatracker * fix: add redirect for old session request url * fix: move stripe javascript to js file * fix: update copyright lines for modified files * fix: rename javascripts and expand redirects * fix: don't show inactive constraints label when there are none (#9680) * chore: remove unused utility * fix: add test for secr main menu page (#9693) * fix: don't show inactive constraints label when there are none * fix: add test for secr main menu page --------- Co-authored-by: Jennifer Richards Co-authored-by: Robert Sparks --- ietf/meeting/forms.py | 333 +++++++- .../templatetags/ams_filters.py | 2 + .../tests_session_requests.py} | 314 ++++--- ietf/meeting/tests_views.py | 4 +- ietf/meeting/urls.py | 14 +- .../views_session_request.py} | 803 ++++++++++-------- ietf/secr/meetings/views.py | 4 +- ietf/secr/sreq/__init__.py | 0 ietf/secr/sreq/forms.py | 333 -------- ietf/secr/sreq/templatetags/__init__.py | 0 ietf/secr/sreq/urls.py | 20 - ietf/secr/telechat/tests.py | 21 + ietf/secr/templates/includes/activities.html | 23 - .../includes/buttons_next_cancel.html | 6 - .../includes/buttons_submit_cancel.html | 6 - .../templates/includes/sessions_footer.html | 5 - .../includes/sessions_request_form.html | 130 --- .../includes/sessions_request_view.html | 73 -- .../sessions_request_view_formset.html | 32 - .../sessions_request_view_session_set.html | 32 - ietf/secr/templates/index.html | 6 +- ietf/secr/templates/sreq/confirm.html | 57 -- ietf/secr/templates/sreq/edit.html | 39 - ietf/secr/templates/sreq/locked.html | 30 - ietf/secr/templates/sreq/main.html | 65 -- ietf/secr/templates/sreq/new.html | 43 - ietf/secr/templates/sreq/tool_status.html | 42 - ietf/secr/templates/sreq/view.html | 55 -- ietf/secr/urls.py | 13 +- ietf/secr/utils/group.py | 50 -- ietf/settings.py | 1 - ietf/static/js/custom_striped.js | 16 + ietf/{secr => }/static/js/session_form.js | 2 +- .../js/session_request.js} | 12 +- ietf/templates/base/menu.html | 4 +- ietf/templates/group/meetings-row.html | 3 +- ietf/templates/group/meetings.html | 3 +- .../meeting/important_dates_for_meeting.ics | 5 +- ietf/templates/meeting/requests.html | 4 +- .../session_approval_notification.txt | 5 +- .../meeting}/session_cancel_notification.txt | 1 + .../meeting/session_details_form.html | 64 +- .../session_not_meeting_notification.txt} | 1 + .../meeting/session_request_confirm.html | 38 + .../meeting/session_request_form.html | 206 +++++ .../meeting/session_request_info.txt | 26 + .../meeting/session_request_list.html | 65 ++ .../meeting/session_request_locked.html | 21 + .../meeting}/session_request_notification.txt | 3 +- .../meeting/session_request_status.html | 28 + .../meeting/session_request_view.html | 59 ++ .../meeting/session_request_view_formset.html | 49 ++ .../session_request_view_session_set.html | 47 + .../meeting/session_request_view_table.html | 146 ++++ package.json | 5 +- 55 files changed, 1728 insertions(+), 1641 deletions(-) rename ietf/{secr/sreq => meeting}/templatetags/ams_filters.py (96%) rename ietf/{secr/sreq/tests.py => meeting/tests_session_requests.py} (84%) rename ietf/{secr/sreq/views.py => meeting/views_session_request.py} (80%) delete mode 100644 ietf/secr/sreq/__init__.py delete mode 100644 ietf/secr/sreq/forms.py delete mode 100644 ietf/secr/sreq/templatetags/__init__.py delete mode 100644 ietf/secr/sreq/urls.py delete mode 100644 ietf/secr/templates/includes/activities.html delete mode 100644 ietf/secr/templates/includes/buttons_next_cancel.html delete mode 100644 ietf/secr/templates/includes/buttons_submit_cancel.html delete mode 100755 ietf/secr/templates/includes/sessions_footer.html delete mode 100755 ietf/secr/templates/includes/sessions_request_form.html delete mode 100644 ietf/secr/templates/includes/sessions_request_view.html delete mode 100644 ietf/secr/templates/includes/sessions_request_view_formset.html delete mode 100644 ietf/secr/templates/includes/sessions_request_view_session_set.html delete mode 100755 ietf/secr/templates/sreq/confirm.html delete mode 100755 ietf/secr/templates/sreq/edit.html delete mode 100755 ietf/secr/templates/sreq/locked.html delete mode 100755 ietf/secr/templates/sreq/main.html delete mode 100755 ietf/secr/templates/sreq/new.html delete mode 100755 ietf/secr/templates/sreq/tool_status.html delete mode 100644 ietf/secr/templates/sreq/view.html delete mode 100644 ietf/secr/utils/group.py create mode 100644 ietf/static/js/custom_striped.js rename ietf/{secr => }/static/js/session_form.js (92%) rename ietf/{secr/static/js/sessions.js => static/js/session_request.js} (90%) rename ietf/{secr/templates/sreq => templates/meeting}/session_approval_notification.txt (56%) rename ietf/{secr/templates/sreq => templates/meeting}/session_cancel_notification.txt (71%) rename ietf/{secr/templates/sreq/not_meeting_notification.txt => templates/meeting/session_not_meeting_notification.txt} (83%) create mode 100644 ietf/templates/meeting/session_request_confirm.html create mode 100644 ietf/templates/meeting/session_request_form.html create mode 100644 ietf/templates/meeting/session_request_info.txt create mode 100644 ietf/templates/meeting/session_request_list.html create mode 100644 ietf/templates/meeting/session_request_locked.html rename ietf/{secr/templates/sreq => templates/meeting}/session_request_notification.txt (56%) create mode 100644 ietf/templates/meeting/session_request_status.html create mode 100644 ietf/templates/meeting/session_request_view.html create mode 100644 ietf/templates/meeting/session_request_view_formset.html create mode 100644 ietf/templates/meeting/session_request_view_session_set.html create mode 100644 ietf/templates/meeting/session_request_view_table.html diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index b6b1a1591f1..e5b1697f866 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2016-2023, All Rights Reserved +# Copyright The IETF Trust 2016-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,19 +15,24 @@ from django.core import validators from django.core.exceptions import ValidationError from django.forms import BaseInlineFormSet +from django.template.defaultfilters import pluralize from django.utils.functional import cached_property +from django.utils.safestring import mark_safe import debug # pyflakes:ignore from ietf.doc.models import Document, State, NewRevisionDocEvent from ietf.group.models import Group from ietf.group.utils import groups_managed_by -from ietf.meeting.models import Session, Meeting, Schedule, COUNTRIES, TIMEZONES, TimeSlot, Room +from ietf.meeting.models import (Session, Meeting, Schedule, COUNTRIES, TIMEZONES, TimeSlot, Room, + Constraint, ResourceAssociation) from ietf.meeting.helpers import get_next_interim_number, make_materials_directories from ietf.meeting.helpers import is_interim_meeting_approved, get_next_agenda_name from ietf.message.models import Message -from ietf.name.models import TimeSlotTypeName, SessionPurposeName +from ietf.name.models import TimeSlotTypeName, SessionPurposeName, TimerangeName, ConstraintName +from ietf.person.fields import SearchablePersonsField from ietf.person.models import Person +from ietf.utils import log from ietf.utils.fields import ( DatepickerDateField, DatepickerSplitDateTimeWidget, @@ -35,9 +40,14 @@ ModelMultipleChoiceField, MultiEmailField, ) +from ietf.utils.html import clean_text_field from ietf.utils.validators import ( validate_file_size, validate_mime_type, validate_file_extension, validate_no_html_frame) +NUM_SESSION_CHOICES = (('', '--Please select'), ('1', '1'), ('2', '2')) +SESSION_TIME_RELATION_CHOICES = (('', 'No preference'),) + Constraint.TIME_RELATION_CHOICES +JOINT_FOR_SESSION_CHOICES = (('1', 'First session'), ('2', 'Second session'), ('3', 'Third session'), ) + # ------------------------------------------------- # Helpers # ------------------------------------------------- @@ -74,6 +84,27 @@ def duration_string(duration): return string +def allowed_conflicting_groups(): + return Group.objects.filter( + type__in=['wg', 'ag', 'rg', 'rag', 'program', 'edwg'], + state__in=['bof', 'proposed', 'active']) + + +def check_conflict(groups, source_group): + ''' + Takes a string which is a list of group acronyms. Checks that they are all active groups + ''' + # convert to python list (allow space or comma separated lists) + items = groups.replace(',', ' ').split() + active_groups = allowed_conflicting_groups() + for group in items: + if group == source_group.acronym: + raise forms.ValidationError("Cannot declare a conflict with the same group: %s" % group) + + if not active_groups.filter(acronym=group): + raise forms.ValidationError("Invalid or inactive group acronym: %s" % group) + + # ------------------------------------------------- # Forms # ------------------------------------------------- @@ -753,6 +784,9 @@ def __init__(self, group, *args, **kwargs): self.fields['purpose'].queryset = SessionPurposeName.objects.filter(pk__in=session_purposes) if not group.features.acts_like_wg: self.fields['requested_duration'].durations = [datetime.timedelta(minutes=m) for m in range(30, 241, 30)] + # add bootstrap classes + self.fields['purpose'].widget.attrs.update({'class': 'form-select'}) + self.fields['type'].widget.attrs.update({'class': 'form-select', 'aria-label': 'session type'}) class Meta: model = Session @@ -837,3 +871,296 @@ def sessiondetailsformset_factory(min_num=1, max_num=3): max_num=max_num, extra=max_num, # only creates up to max_num total ) + + +class SessionRequestStatusForm(forms.Form): + message = forms.CharField(widget=forms.Textarea(attrs={'rows': '3', 'cols': '80'}), strip=False) + + +class NameModelMultipleChoiceField(ModelMultipleChoiceField): + def label_from_instance(self, name): + return name.desc + + +class SessionRequestForm(forms.Form): + num_session = forms.ChoiceField( + choices=NUM_SESSION_CHOICES, + label="Number of sessions") + # session fields are added in __init__() + session_time_relation = forms.ChoiceField( + choices=SESSION_TIME_RELATION_CHOICES, + required=False, + label="Time between two sessions") + attendees = forms.IntegerField(label="Number of Attendees") + # FIXME: it would cleaner to have these be + # ModelMultipleChoiceField, and just customize the widgetry, that + # way validation comes for free (applies to this CharField and the + # constraints dynamically instantiated in __init__()) + joint_with_groups = forms.CharField(max_length=255, required=False) + joint_with_groups_selector = forms.ChoiceField(choices=[], required=False) # group select widget for prev field + joint_for_session = forms.ChoiceField(choices=JOINT_FOR_SESSION_CHOICES, required=False) + comments = forms.CharField( + max_length=200, + label='Special Requests', + help_text='i.e. restrictions on meeting times / days, etc. (limit 200 characters)', + required=False) + third_session = forms.BooleanField( + required=False, + help_text="Help") + resources = forms.MultipleChoiceField( + widget=forms.CheckboxSelectMultiple, + required=False, + label='Resources Requested') + bethere = SearchablePersonsField( + label="Participants who must be present", + required=False, + help_text=mark_safe('Do not include Area Directors and WG Chairs; the system already tracks their availability.')) + timeranges = NameModelMultipleChoiceField( + widget=forms.CheckboxSelectMultiple, + required=False, + label=mark_safe('Times during which this WG can not meet:
    Please explain any selections in Special Requests below.'), + queryset=TimerangeName.objects.all()) + adjacent_with_wg = forms.ChoiceField( + required=False, + label=mark_safe('Plan session adjacent with another WG:
    (Immediately before or after another WG, no break in between, in the same room.)')) + send_notifications = forms.BooleanField(label="Send notification emails?", required=False, initial=False) + + def __init__(self, group, meeting, data=None, *args, **kwargs): + self.hidden = kwargs.pop('hidden', False) + self.notifications_optional = kwargs.pop('notifications_optional', False) + + self.group = group + formset_class = sessiondetailsformset_factory(max_num=3 if group.features.acts_like_wg else 50) + self.session_forms = formset_class(group=self.group, meeting=meeting, data=data) + super().__init__(data=data, *args, **kwargs) + if not self.notifications_optional: + self.fields['send_notifications'].widget = forms.HiddenInput() + + # Allow additional sessions for non-wg-like groups + if not self.group.features.acts_like_wg: + self.fields['num_session'].choices = ((n, str(n)) for n in range(1, 51)) + + self._add_widget_class(self.fields['third_session'].widget, 'form-check-input') + self.fields['comments'].widget = forms.Textarea(attrs={'rows': '3', 'cols': '65'}) + + other_groups = list(allowed_conflicting_groups().exclude(pk=group.pk).values_list('acronym', 'acronym').order_by('acronym')) + self.fields['adjacent_with_wg'].choices = [('', '--No preference')] + other_groups + group_acronym_choices = [('', '--Select WG(s)')] + other_groups + self.fields['joint_with_groups_selector'].choices = group_acronym_choices + + # Set up constraints for the meeting + self._wg_field_data = [] + for constraintname in meeting.group_conflict_types.all(): + # two fields for each constraint: a CharField for the group list and a selector to add entries + constraint_field = forms.CharField(max_length=255, required=False) + constraint_field.widget.attrs['data-slug'] = constraintname.slug + constraint_field.widget.attrs['data-constraint-name'] = str(constraintname).title() + constraint_field.widget.attrs['aria-label'] = f'{constraintname.slug}_input' + self._add_widget_class(constraint_field.widget, 'wg_constraint') + self._add_widget_class(constraint_field.widget, 'form-control') + + selector_field = forms.ChoiceField(choices=group_acronym_choices, required=False) + selector_field.widget.attrs['data-slug'] = constraintname.slug # used by onchange handler + self._add_widget_class(selector_field.widget, 'wg_constraint_selector') + self._add_widget_class(selector_field.widget, 'form-control') + + cfield_id = 'constraint_{}'.format(constraintname.slug) + cselector_id = 'wg_selector_{}'.format(constraintname.slug) + # keep an eye out for field name conflicts + log.assertion('cfield_id not in self.fields') + log.assertion('cselector_id not in self.fields') + self.fields[cfield_id] = constraint_field + self.fields[cselector_id] = selector_field + self._wg_field_data.append((constraintname, cfield_id, cselector_id)) + + # Show constraints that are not actually used by the meeting so these don't get lost + self._inactive_wg_field_data = [] + inactive_cnames = ConstraintName.objects.filter( + is_group_conflict=True # Only collect group conflicts... + ).exclude( + meeting=meeting # ...that are not enabled for this meeting... + ).filter( + constraint__source=group, # ...but exist for this group... + constraint__meeting=meeting, # ... at this meeting. + ).distinct() + + for inactive_constraint_name in inactive_cnames: + field_id = 'delete_{}'.format(inactive_constraint_name.slug) + self.fields[field_id] = forms.BooleanField(required=False, label='Delete this conflict', help_text='Delete this inactive conflict?') + self._add_widget_class(self.fields[field_id].widget, 'form-control') + constraints = group.constraint_source_set.filter(meeting=meeting, name=inactive_constraint_name) + self._inactive_wg_field_data.append( + (inactive_constraint_name, + ' '.join([c.target.acronym for c in constraints]), + field_id) + ) + + self.fields['joint_with_groups_selector'].widget.attrs['onchange'] = "document.form_post.joint_with_groups.value=document.form_post.joint_with_groups.value + ' ' + this.options[this.selectedIndex].value; return 1;" + self.fields["resources"].choices = [(x.pk, x.desc) for x in ResourceAssociation.objects.filter(name__used=True).order_by('name__order')] + + if self.hidden: + # replace all the widgets to start... + for key in list(self.fields.keys()): + self.fields[key].widget = forms.HiddenInput() + # re-replace a couple special cases + self.fields['resources'].widget = forms.MultipleHiddenInput() + self.fields['timeranges'].widget = forms.MultipleHiddenInput() + # and entirely replace bethere - no need to support searching if input is hidden + self.fields['bethere'] = ModelMultipleChoiceField( + widget=forms.MultipleHiddenInput, required=False, + queryset=Person.objects.all(), + ) + + def wg_constraint_fields(self): + """Iterates over wg constraint fields + + Intended for use in the template. + """ + for cname, cfield_id, cselector_id in self._wg_field_data: + yield cname, self[cfield_id], self[cselector_id] + + def wg_constraint_count(self): + """How many wg constraints are there?""" + return len(self._wg_field_data) + + def wg_constraint_field_ids(self): + """Iterates over wg constraint field IDs""" + for cname, cfield_id, _ in self._wg_field_data: + yield cname, cfield_id + + def inactive_wg_constraints(self): + for cname, value, field_id in self._inactive_wg_field_data: + yield cname, value, self[field_id] + + def inactive_wg_constraint_count(self): + return len(self._inactive_wg_field_data) + + def inactive_wg_constraint_field_ids(self): + """Iterates over wg constraint field IDs""" + for cname, _, field_id in self._inactive_wg_field_data: + yield cname, field_id + + @staticmethod + def _add_widget_class(widget, new_class): + """Add a new class, taking care in case some already exist""" + existing_classes = widget.attrs.get('class', '').split() + widget.attrs['class'] = ' '.join(existing_classes + [new_class]) + + def _join_conflicts(self, cleaned_data, slugs): + """Concatenate constraint fields from cleaned data into a single list""" + conflicts = [] + for cname, cfield_id, _ in self._wg_field_data: + if cname.slug in slugs and cfield_id in cleaned_data: + groups = cleaned_data[cfield_id] + # convert to python list (allow space or comma separated lists) + items = groups.replace(',', ' ').split() + conflicts.extend(items) + return conflicts + + def _validate_duplicate_conflicts(self, cleaned_data): + """Validate that no WGs appear in more than one constraint that does not allow duplicates + + Raises ValidationError + """ + # Only the older constraints (conflict, conflic2, conflic3) need to be mutually exclusive. + all_conflicts = self._join_conflicts(cleaned_data, ['conflict', 'conflic2', 'conflic3']) + seen = [] + duplicated = [] + errors = [] + for c in all_conflicts: + if c not in seen: + seen.append(c) + elif c not in duplicated: # only report once + duplicated.append(c) + errors.append(forms.ValidationError('%s appears in conflicts more than once' % c)) + return errors + + def clean_joint_with_groups(self): + groups = self.cleaned_data['joint_with_groups'] + check_conflict(groups, self.group) + return groups + + def clean_comments(self): + return clean_text_field(self.cleaned_data['comments']) + + def clean_bethere(self): + bethere = self.cleaned_data["bethere"] + if bethere: + extra = set( + Person.objects.filter( + role__group=self.group, role__name__in=["chair", "ad"] + ) + & bethere + ) + if extra: + extras = ", ".join(e.name for e in extra) + raise forms.ValidationError( + ( + f"Please remove the following person{pluralize(len(extra))}, the system " + f"tracks their availability due to their role{pluralize(len(extra))}: {extras}." + ) + ) + return bethere + + def clean_send_notifications(self): + return True if not self.notifications_optional else self.cleaned_data['send_notifications'] + + def is_valid(self): + return super().is_valid() and self.session_forms.is_valid() + + def clean(self): + super(SessionRequestForm, self).clean() + self.session_forms.clean() + + data = self.cleaned_data + + # Validate the individual conflict fields + for _, cfield_id, _ in self._wg_field_data: + try: + check_conflict(data[cfield_id], self.group) + except forms.ValidationError as e: + self.add_error(cfield_id, e) + + # Skip remaining tests if individual field tests had errors, + if self.errors: + return data + + # error if conflicts contain disallowed dupes + for error in self._validate_duplicate_conflicts(data): + self.add_error(None, error) + + # Verify expected number of session entries are present + num_sessions_with_data = len(self.session_forms.forms_to_keep) + num_sessions_expected = -1 + try: + num_sessions_expected = int(data.get('num_session', '')) + except ValueError: + self.add_error('num_session', 'Invalid value for number of sessions') + if num_sessions_with_data < num_sessions_expected: + self.add_error('num_session', 'Must provide data for all sessions') + + # if default (empty) option is selected, cleaned_data won't include num_session key + if num_sessions_expected != 2 and num_sessions_expected is not None: + if data.get('session_time_relation'): + self.add_error( + 'session_time_relation', + forms.ValidationError('Time between sessions can only be used when two sessions are requested.') + ) + + joint_session = data.get('joint_for_session', '') + if joint_session != '': + joint_session = int(joint_session) + if joint_session > num_sessions_with_data: + self.add_error( + 'joint_for_session', + forms.ValidationError( + f'Session {joint_session} can not be the joint session, the session has not been requested.' + ) + ) + + return data + + @property + def media(self): + # get media for our formset + return super().media + self.session_forms.media + forms.Media(js=('ietf/js/session_form.js',)) diff --git a/ietf/secr/sreq/templatetags/ams_filters.py b/ietf/meeting/templatetags/ams_filters.py similarity index 96% rename from ietf/secr/sreq/templatetags/ams_filters.py rename to ietf/meeting/templatetags/ams_filters.py index 3ef872232ab..a8175a81d6d 100644 --- a/ietf/secr/sreq/templatetags/ams_filters.py +++ b/ietf/meeting/templatetags/ams_filters.py @@ -1,3 +1,5 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + from django import template from ietf.person.models import Person diff --git a/ietf/secr/sreq/tests.py b/ietf/meeting/tests_session_requests.py similarity index 84% rename from ietf/secr/sreq/tests.py rename to ietf/meeting/tests_session_requests.py index 847b993e1cd..0cb092d2f87 100644 --- a/ietf/secr/sreq/tests.py +++ b/ietf/meeting/tests_session_requests.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved +# Copyright The IETF Trust 2013-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,30 +15,15 @@ from ietf.name.models import ConstraintName, TimerangeName from ietf.person.factories import PersonFactory from ietf.person.models import Person -from ietf.secr.sreq.forms import SessionForm +from ietf.meeting.forms import SessionRequestForm from ietf.utils.mail import outbox, empty_outbox, get_payload_text, send_mail from ietf.utils.timezone import date_today from pyquery import PyQuery -SECR_USER='secretary' +SECR_USER = 'secretary' -class SreqUrlTests(TestCase): - def test_urls(self): - MeetingFactory(type_id='ietf',date=date_today()) - - self.client.login(username="secretary", password="secretary+password") - - r = self.client.get("/secr/") - self.assertEqual(r.status_code, 200) - - r = self.client.get("/secr/sreq/") - self.assertEqual(r.status_code, 200) - - testgroup=GroupFactory() - r = self.client.get("/secr/sreq/%s/new/" % testgroup.acronym) - self.assertEqual(r.status_code, 200) class SessionRequestTestCase(TestCase): def test_main(self): @@ -46,7 +31,7 @@ def test_main(self): SessionFactory.create_batch(2, meeting=meeting, status_id='sched') SessionFactory.create_batch(2, meeting=meeting, status_id='disappr') # Several unscheduled groups come from make_immutable_base_data - url = reverse('ietf.secr.sreq.views.main') + url = reverse('ietf.meeting.views_session_request.list_view') self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -62,27 +47,27 @@ def test_approve(self): mars = GroupFactory(parent=area, acronym='mars') # create session waiting for approval session = SessionFactory(meeting=meeting, group=mars, status_id='apprw') - url = reverse('ietf.secr.sreq.views.approve', kwargs={'acronym':'mars'}) + url = reverse('ietf.meeting.views_session_request.approve_request', kwargs={'acronym': 'mars'}) self.client.login(username="ad", password="ad+password") r = self.client.get(url) - self.assertRedirects(r,reverse('ietf.secr.sreq.views.view', kwargs={'acronym':'mars'})) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'})) self.assertEqual(SchedulingEvent.objects.filter(session=session).order_by('-id')[0].status_id, 'appr') - + def test_cancel(self): meeting = MeetingFactory(type_id='ietf', date=date_today()) ad = Person.objects.get(user__username='ad') area = RoleFactory(name_id='ad', person=ad, group__type_id='area').group session = SessionFactory(meeting=meeting, group__parent=area, group__acronym='mars', status_id='sched') - url = reverse('ietf.secr.sreq.views.cancel', kwargs={'acronym':'mars'}) + url = reverse('ietf.meeting.views_session_request.cancel_request', kwargs={'acronym': 'mars'}) self.client.login(username="ad", password="ad+password") r = self.client.get(url) - self.assertRedirects(r,reverse('ietf.secr.sreq.views.main')) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view')) self.assertEqual(SchedulingEvent.objects.filter(session=session).order_by('-id')[0].status_id, 'deleted') def test_cancel_notification_msg(self): to = "" subject = "Dummy subject" - template = "sreq/session_cancel_notification.txt" + template = "meeting/session_cancel_notification.txt" meeting = MeetingFactory(type_id="ietf", date=date_today()) requester = PersonFactory(name="James O'Rourke", user__username="jimorourke") context = {"meeting": meeting, "requester": requester} @@ -113,9 +98,9 @@ def test_edit(self): group4 = GroupFactory() iabprog = GroupFactory(type_id='program') - SessionFactory(meeting=meeting,group=mars,status_id='sched') + SessionFactory(meeting=meeting, group=mars, status_id='sched') - url = reverse('ietf.secr.sreq.views.edit', kwargs={'acronym':'mars'}) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs={'acronym': 'mars'}) self.client.login(username="marschairman", password="marschairman+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -123,9 +108,9 @@ def test_edit(self): comments = 'need lights' mars_sessions = meeting.session_set.filter(group__acronym='mars') empty_outbox() - post_data = {'num_session':'2', + post_data = {'num_session': '2', 'attendees': attendees, - 'constraint_chair_conflict':iabprog.acronym, + 'constraint_chair_conflict': iabprog.acronym, 'session_time_relation': 'subsequent-days', 'adjacent_with_wg': group2.acronym, 'joint_with_groups': group3.acronym + ' ' + group4.acronym, @@ -135,7 +120,7 @@ def test_edit(self): 'session_set-INITIAL_FORMS': '1', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', - 'session_set-0-id':mars_sessions[0].pk, + 'session_set-0-id': mars_sessions[0].pk, 'session_set-0-name': mars_sessions[0].name, 'session_set-0-short': mars_sessions[0].short, 'session_set-0-purpose': mars_sessions[0].purpose_id, @@ -169,7 +154,7 @@ def test_edit(self): 'session_set-2-DELETE': 'on', 'submit': 'Continue'} r = self.client.post(url, post_data, HTTP_HOST='example.com') - redirect_url = reverse('ietf.secr.sreq.views.view', kwargs={'acronym': 'mars'}) + redirect_url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'}) self.assertRedirects(r, redirect_url) # Check whether updates were stored in the database @@ -204,17 +189,17 @@ def test_edit(self): # Edit again, changing the joint sessions and clearing some fields. The behaviour of # edit is different depending on whether previous joint sessions were recorded. empty_outbox() - post_data = {'num_session':'2', - 'attendees':attendees, - 'constraint_chair_conflict':'', - 'comments':'need lights', + post_data = {'num_session': '2', + 'attendees': attendees, + 'constraint_chair_conflict': '', + 'comments': 'need lights', 'joint_with_groups': group2.acronym, 'joint_for_session': '1', 'session_set-TOTAL_FORMS': '3', # matches what view actually sends, even with only 2 filled in 'session_set-INITIAL_FORMS': '2', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', - 'session_set-0-id':sessions[0].pk, + 'session_set-0-id': sessions[0].pk, 'session_set-0-name': sessions[0].name, 'session_set-0-short': sessions[0].short, 'session_set-0-purpose': sessions[0].purpose_id, @@ -270,7 +255,6 @@ def test_edit(self): r = self.client.get(redirect_url) self.assertContains(r, 'First session with: {}'.format(group2.acronym)) - def test_edit_constraint_bethere(self): meeting = MeetingFactory(type_id='ietf', date=date_today()) mars = RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars').group @@ -282,7 +266,7 @@ def test_edit_constraint_bethere(self): name_id='bethere', ) self.assertEqual(session.people_constraints.count(), 1) - url = reverse('ietf.secr.sreq.views.edit', kwargs=dict(acronym='mars')) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs=dict(acronym='mars')) self.client.login(username='marschairman', password='marschairman+password') attendees = '10' ad = Person.objects.get(user__username='ad') @@ -290,8 +274,8 @@ def test_edit_constraint_bethere(self): 'num_session': '1', 'attendees': attendees, 'bethere': str(ad.pk), - 'constraint_chair_conflict':'', - 'comments':'', + 'constraint_chair_conflict': '', + 'comments': '', 'joint_with_groups': '', 'joint_for_session': '', 'delete_conflict': 'on', @@ -299,7 +283,7 @@ def test_edit_constraint_bethere(self): 'session_set-INITIAL_FORMS': '1', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', - 'session_set-0-id':session.pk, + 'session_set-0-id': session.pk, 'session_set-0-name': session.name, 'session_set-0-short': session.short, 'session_set-0-purpose': session.purpose_id, @@ -313,8 +297,8 @@ def test_edit_constraint_bethere(self): 'session_set-1-id': '', 'session_set-1-name': '', 'session_set-1-short': '', - 'session_set-1-purpose':'regular', - 'session_set-1-type':'regular', + 'session_set-1-purpose': 'regular', + 'session_set-1-type': 'regular', 'session_set-1-requested_duration': '', 'session_set-1-on_agenda': 'True', 'session_set-1-attendees': attendees, @@ -333,7 +317,7 @@ def test_edit_constraint_bethere(self): 'submit': 'Save', } r = self.client.post(url, post_data, HTTP_HOST='example.com') - redirect_url = reverse('ietf.secr.sreq.views.view', kwargs={'acronym': 'mars'}) + redirect_url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'}) self.assertRedirects(r, redirect_url) self.assertEqual([pc.person for pc in session.people_constraints.all()], [ad]) @@ -350,7 +334,7 @@ def test_edit_inactive_conflicts(self): target=other_group, ) - url = reverse('ietf.secr.sreq.views.edit', kwargs=dict(acronym='mars')) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs=dict(acronym='mars')) self.client.login(username='marschairman', password='marschairman+password') r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -360,17 +344,17 @@ def test_edit_inactive_conflicts(self): found = q('input#id_delete_conflict[type="checkbox"]') self.assertEqual(len(found), 1) delete_checkbox = found[0] - # check that the label on the checkbox is correct - self.assertIn('Delete this conflict', delete_checkbox.tail) + self.assertIn('Delete this conflict', delete_checkbox.label.text) # check that the target is displayed correctly in the UI - self.assertIn(other_group.acronym, delete_checkbox.find('../input[@type="text"]').value) + row = found.parent().parent() + self.assertIn(other_group.acronym, row.find('input[@type="text"]').val()) attendees = '10' post_data = { 'num_session': '1', 'attendees': attendees, - 'constraint_chair_conflict':'', - 'comments':'', + 'constraint_chair_conflict': '', + 'comments': '', 'joint_with_groups': '', 'joint_for_session': '', 'delete_conflict': 'on', @@ -378,7 +362,7 @@ def test_edit_inactive_conflicts(self): 'session_set-INITIAL_FORMS': '1', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', - 'session_set-0-id':session.pk, + 'session_set-0-id': session.pk, 'session_set-0-name': session.name, 'session_set-0-short': session.short, 'session_set-0-purpose': session.purpose_id, @@ -392,28 +376,28 @@ def test_edit_inactive_conflicts(self): 'submit': 'Save', } r = self.client.post(url, post_data, HTTP_HOST='example.com') - redirect_url = reverse('ietf.secr.sreq.views.view', kwargs={'acronym': 'mars'}) + redirect_url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'}) self.assertRedirects(r, redirect_url) self.assertEqual(len(mars.constraint_source_set.filter(name_id='conflict')), 0) def test_tool_status(self): MeetingFactory(type_id='ietf', date=date_today()) - url = reverse('ietf.secr.sreq.views.tool_status') + url = reverse('ietf.meeting.views_session_request.status') self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post(url, {'message':'locked', 'submit':'Lock'}) - self.assertRedirects(r,reverse('ietf.secr.sreq.views.main')) + r = self.client.post(url, {'message': 'locked', 'submit': 'Lock'}) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view')) def test_new_req_constraint_types(self): """Configurable constraint types should be handled correctly in a new request - Relies on SessionForm representing constraint values with element IDs + Relies on SessionRequestForm representing constraint values with element IDs like id_constraint_ """ meeting = MeetingFactory(type_id='ietf', date=date_today()) RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars') - url = reverse('ietf.secr.sreq.views.new', kwargs=dict(acronym='mars')) + url = reverse('ietf.meeting.views_session_request.new_request', kwargs=dict(acronym='mars')) self.client.login(username="marschairman", password="marschairman+password") for expected in [ @@ -441,7 +425,7 @@ def test_edit_req_constraint_types(self): add_to_schedule=False) RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars') - url = reverse('ietf.secr.sreq.views.edit', kwargs=dict(acronym='mars')) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs=dict(acronym='mars')) self.client.login(username='marschairman', password='marschairman+password') for expected in [ @@ -460,6 +444,7 @@ def test_edit_req_constraint_types(self): ['id_constraint_{}'.format(conf_name) for conf_name in expected], ) + class SubmitRequestCase(TestCase): def setUp(self): super(SubmitRequestCase, self).setUp() @@ -476,15 +461,15 @@ def test_submit_request(self): group3 = GroupFactory(parent=area) group4 = GroupFactory(parent=area) session_count_before = Session.objects.filter(meeting=meeting, group=group).count() - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) - confirm_url = reverse('ietf.secr.sreq.views.confirm',kwargs={'acronym':group.acronym}) - main_url = reverse('ietf.secr.sreq.views.main') + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym}) + confirm_url = reverse('ietf.meeting.views_session_request.confirm', kwargs={'acronym': group.acronym}) + main_url = reverse('ietf.meeting.views_session_request.list_view') attendees = '10' comments = 'need projector' - post_data = {'num_session':'1', - 'attendees':attendees, - 'constraint_chair_conflict':'', - 'comments':comments, + post_data = {'num_session': '1', + 'attendees': attendees, + 'constraint_chair_conflict': '', + 'comments': comments, 'adjacent_with_wg': group2.acronym, 'timeranges': ['thursday-afternoon-early', 'thursday-afternoon-late'], 'joint_with_groups': group3.acronym + ' ' + group4.acronym, @@ -506,7 +491,7 @@ def test_submit_request(self): 'session_set-0-DELETE': '', 'submit': 'Continue'} self.client.login(username="secretary", password="secretary+password") - r = self.client.post(url,post_data) + r = self.client.post(url, post_data) self.assertEqual(r.status_code, 200) # Verify the contents of the confirm view @@ -515,13 +500,13 @@ def test_submit_request(self): self.assertContains(r, 'First session with: {} {}'.format(group3.acronym, group4.acronym)) post_data['submit'] = 'Submit' - r = self.client.post(confirm_url,post_data) + r = self.client.post(confirm_url, post_data) self.assertRedirects(r, main_url) session_count_after = Session.objects.filter(meeting=meeting, group=group, type='regular').count() self.assertEqual(session_count_after, session_count_before + 1) # test that second confirm does not add sessions - r = self.client.post(confirm_url,post_data) + r = self.client.post(confirm_url, post_data) self.assertRedirects(r, main_url) session_count_after = Session.objects.filter(meeting=meeting, group=group, type='regular').count() self.assertEqual(session_count_after, session_count_before + 1) @@ -535,42 +520,6 @@ def test_submit_request(self): ) self.assertEqual(set(list(session.joint_with_groups.all())), set([group3, group4])) - def test_submit_request_invalid(self): - MeetingFactory(type_id='ietf', date=date_today()) - ad = Person.objects.get(user__username='ad') - area = RoleFactory(name_id='ad', person=ad, group__type_id='area').group - group = GroupFactory(parent=area) - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) - attendees = '10' - comments = 'need projector' - post_data = { - 'num_session':'2', - 'attendees':attendees, - 'constraint_chair_conflict':'', - 'comments':comments, - 'session_set-TOTAL_FORMS': '1', - 'session_set-INITIAL_FORMS': '1', - 'session_set-MIN_NUM_FORMS': '1', - 'session_set-MAX_NUM_FORMS': '3', - # no 'session_set-0-id' to create a new session - 'session_set-0-name': '', - 'session_set-0-short': '', - 'session_set-0-purpose': 'regular', - 'session_set-0-type': 'regular', - 'session_set-0-requested_duration': '3600', - 'session_set-0-on_agenda': True, - 'session_set-0-remote_instructions': '', - 'session_set-0-attendees': attendees, - 'session_set-0-comments': comments, - 'session_set-0-DELETE': '', - } - self.client.login(username="secretary", password="secretary+password") - r = self.client.post(url,post_data) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertEqual(len(q('#session-request-form')),1) - self.assertContains(r, 'Must provide data for all sessions') - def test_submit_request_check_constraints(self): m1 = MeetingFactory(type_id='ietf', date=date_today() - datetime.timedelta(days=100)) MeetingFactory(type_id='ietf', date=date_today(), @@ -597,7 +546,7 @@ def test_submit_request_check_constraints(self): self.client.login(username="secretary", password="secretary+password") - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym}) r = self.client.get(url + '?previous') self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -607,11 +556,11 @@ def test_submit_request_check_constraints(self): attendees = '10' comments = 'need projector' - post_data = {'num_session':'1', - 'attendees':attendees, + post_data = {'num_session': '1', + 'attendees': attendees, 'constraint_chair_conflict': group.acronym, - 'comments':comments, - 'session_set-TOTAL_FORMS': '1', + 'comments': comments, + 'session_set-TOTAL_FORMS': '3', 'session_set-INITIAL_FORMS': '1', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', @@ -626,11 +575,31 @@ def test_submit_request_check_constraints(self): 'session_set-0-attendees': attendees, 'session_set-0-comments': comments, 'session_set-0-DELETE': '', + 'session_set-1-name': '', + 'session_set-1-short': '', + 'session_set-1-purpose': session.purpose_id, + 'session_set-1-type': session.type_id, + 'session_set-1-requested_duration': '', + 'session_set-1-on_agenda': session.on_agenda, + 'session_set-1-remote_instructions': '', + 'session_set-1-attendees': attendees, + 'session_set-1-comments': '', + 'session_set-1-DELETE': 'on', + 'session_set-2-name': '', + 'session_set-2-short': '', + 'session_set-2-purpose': session.purpose_id, + 'session_set-2-type': session.type_id, + 'session_set-2-requested_duration': '', + 'session_set-2-on_agenda': session.on_agenda, + 'session_set-2-remote_instructions': '', + 'session_set-2-attendees': attendees, + 'session_set-2-comments': '', + 'session_set-2-DELETE': 'on', 'submit': 'Continue'} - r = self.client.post(url,post_data) + r = self.client.post(url, post_data) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#session-request-form')),1) + self.assertEqual(len(q('#session-request-form')), 1) self.assertContains(r, "Cannot declare a conflict with the same group") def test_request_notification(self): @@ -645,18 +614,18 @@ def test_request_notification(self): RoleFactory(name_id='chair', group=group, person__user__username='ameschairman') resource = ResourceAssociation.objects.create(name_id='project') # Bit of a test data hack - the fixture now has no used resources to pick from - resource.name.used=True + resource.name.used = True resource.name.save() - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) - confirm_url = reverse('ietf.secr.sreq.views.confirm',kwargs={'acronym':group.acronym}) + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym}) + confirm_url = reverse('ietf.meeting.views_session_request.confirm', kwargs={'acronym': group.acronym}) len_before = len(outbox) attendees = '10' - post_data = {'num_session':'2', - 'attendees':attendees, - 'bethere':str(ad.pk), - 'constraint_chair_conflict':group4.acronym, - 'comments':'', + post_data = {'num_session': '2', + 'attendees': attendees, + 'bethere': str(ad.pk), + 'constraint_chair_conflict': group4.acronym, + 'comments': '', 'resources': resource.pk, 'session_time_relation': 'subsequent-days', 'adjacent_with_wg': group2.acronym, @@ -692,23 +661,23 @@ def test_request_notification(self): 'submit': 'Continue'} self.client.login(username="ameschairman", password="ameschairman+password") # submit - r = self.client.post(url,post_data) + r = self.client.post(url, post_data) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue('Confirm' in str(q("title")), r.context['form'].errors) # confirm post_data['submit'] = 'Submit' - r = self.client.post(confirm_url,post_data) - self.assertRedirects(r, reverse('ietf.secr.sreq.views.main')) - self.assertEqual(len(outbox),len_before+1) + r = self.client.post(confirm_url, post_data) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view')) + self.assertEqual(len(outbox), len_before + 1) notification = outbox[-1] notification_payload = get_payload_text(notification) - sessions = Session.objects.filter(meeting=meeting,group=group) + sessions = Session.objects.filter(meeting=meeting, group=group) self.assertEqual(len(sessions), 2) session = sessions[0] - self.assertEqual(session.resources.count(),1) - self.assertEqual(session.people_constraints.count(),1) + self.assertEqual(session.resources.count(), 1) + self.assertEqual(session.people_constraints.count(), 1) self.assertEqual(session.constraints().get(name='time_relation').time_relation, 'subsequent-days') self.assertEqual(session.constraints().get(name='wg_adjacent').target.acronym, group2.acronym) self.assertEqual( @@ -731,7 +700,7 @@ def test_request_notification(self): def test_request_notification_msg(self): to = "" subject = "Dummy subject" - template = "sreq/session_request_notification.txt" + template = "meeting/session_request_notification.txt" header = "A new" meeting = MeetingFactory(type_id="ietf", date=date_today()) requester = PersonFactory(name="James O'Rourke", user__username="jimorourke") @@ -767,19 +736,19 @@ def test_request_notification_third_session(self): RoleFactory(name_id='chair', group=group, person__user__username='ameschairman') resource = ResourceAssociation.objects.create(name_id='project') # Bit of a test data hack - the fixture now has no used resources to pick from - resource.name.used=True + resource.name.used = True resource.name.save() - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) - confirm_url = reverse('ietf.secr.sreq.views.confirm',kwargs={'acronym':group.acronym}) + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym}) + confirm_url = reverse('ietf.meeting.views_session_request.confirm', kwargs={'acronym': group.acronym}) len_before = len(outbox) attendees = '10' - post_data = {'num_session':'2', + post_data = {'num_session': '2', 'third_session': 'true', - 'attendees':attendees, - 'bethere':str(ad.pk), - 'constraint_chair_conflict':group4.acronym, - 'comments':'', + 'attendees': attendees, + 'bethere': str(ad.pk), + 'constraint_chair_conflict': group4.acronym, + 'comments': '', 'resources': resource.pk, 'session_time_relation': 'subsequent-days', 'adjacent_with_wg': group2.acronym, @@ -826,23 +795,23 @@ def test_request_notification_third_session(self): 'submit': 'Continue'} self.client.login(username="ameschairman", password="ameschairman+password") # submit - r = self.client.post(url,post_data) + r = self.client.post(url, post_data) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue('Confirm' in str(q("title")), r.context['form'].errors) # confirm post_data['submit'] = 'Submit' - r = self.client.post(confirm_url,post_data) - self.assertRedirects(r, reverse('ietf.secr.sreq.views.main')) - self.assertEqual(len(outbox),len_before+1) + r = self.client.post(confirm_url, post_data) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view')) + self.assertEqual(len(outbox), len_before + 1) notification = outbox[-1] notification_payload = get_payload_text(notification) - sessions = Session.objects.filter(meeting=meeting,group=group) + sessions = Session.objects.filter(meeting=meeting, group=group) self.assertEqual(len(sessions), 3) session = sessions[0] - self.assertEqual(session.resources.count(),1) - self.assertEqual(session.people_constraints.count(),1) + self.assertEqual(session.resources.count(), 1) + self.assertEqual(session.people_constraints.count(), 1) self.assertEqual(session.constraints().get(name='time_relation').time_relation, 'subsequent-days') self.assertEqual(session.constraints().get(name='wg_adjacent').target.acronym, group2.acronym) self.assertEqual( @@ -861,16 +830,17 @@ def test_request_notification_third_session(self): self.assertIn('1 Hour, 1 Hour, 1 Hour', notification_payload) self.assertIn('The third session requires your approval', notification_payload) + class LockAppTestCase(TestCase): def setUp(self): super().setUp() - self.meeting = MeetingFactory(type_id='ietf', date=date_today(),session_request_lock_message='locked') + self.meeting = MeetingFactory(type_id='ietf', date=date_today(), session_request_lock_message='locked') self.group = GroupFactory(acronym='mars') RoleFactory(name_id='chair', group=self.group, person__user__username='marschairman') - SessionFactory(group=self.group,meeting=self.meeting) + SessionFactory(group=self.group, meeting=self.meeting) def test_edit_request(self): - url = reverse('ietf.secr.sreq.views.edit',kwargs={'acronym':self.group.acronym}) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs={'acronym': self.group.acronym}) self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -882,48 +852,49 @@ def test_edit_request(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertEqual(len(q(':disabled[name="submit"]')), 1) - + def test_view_request(self): - url = reverse('ietf.secr.sreq.views.view',kwargs={'acronym':self.group.acronym}) + url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': self.group.acronym}) self.client.login(username="secretary", password="secretary+password") - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertEqual(len(q(':enabled[name="edit"]')), 1) # secretary can edit chair = self.group.role_set.filter(name_id='chair').first().person.user.username self.client.login(username=chair, password=f'{chair}+password') - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertEqual(len(q(':disabled[name="edit"]')), 1) # chair cannot edit def test_new_request(self): - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':self.group.acronym}) - + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': self.group.acronym}) + # try as WG Chair self.client.login(username="marschairman", password="marschairman+password") r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#session-request-form')),0) - + self.assertEqual(len(q('#session-request-form')), 0) + # try as Secretariat self.client.login(username="secretary", password="secretary+password") - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#session-request-form')),1) - + self.assertEqual(len(q('#session-request-form')), 1) + + class NotMeetingCase(TestCase): def test_not_meeting(self): - MeetingFactory(type_id='ietf',date=date_today()) + MeetingFactory(type_id='ietf', date=date_today()) group = GroupFactory(acronym='mars') - url = reverse('ietf.secr.sreq.views.no_session',kwargs={'acronym':group.acronym}) + url = reverse('ietf.meeting.views_session_request.no_session', kwargs={'acronym': group.acronym}) self.client.login(username="secretary", password="secretary+password") empty_outbox() - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) # If the view invoked by that get throws an exception (such as an integrity error), # the traceback from this test will talk about a TransactionManagementError and # yell about executing queries before the end of an 'atomic' block @@ -932,14 +903,15 @@ def test_not_meeting(self): self.assertEqual(r.status_code, 200) self.assertContains(r, 'A message was sent to notify not having a session') - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) self.assertContains(r, 'is already marked as not meeting') - self.assertEqual(len(outbox),1) + self.assertEqual(len(outbox), 1) self.assertTrue('Not having a session' in outbox[0]['Subject']) self.assertTrue('session-request@' in outbox[0]['To']) + class RetrievePreviousCase(TestCase): pass @@ -949,7 +921,7 @@ class RetrievePreviousCase(TestCase): # test access by unauthorized -class SessionFormTest(TestCase): +class SessionRequestFormTest(TestCase): def setUp(self): super().setUp() self.meeting = MeetingFactory(type_id='ietf') @@ -1014,19 +986,19 @@ def setUp(self): 'session_set-2-comments': '', 'session_set-2-DELETE': '', } - + def test_valid(self): # Test with three sessions - form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) self.assertTrue(form.is_valid()) - + # Test with two sessions self.valid_form_data.update({ 'third_session': '', 'session_set-TOTAL_FORMS': '2', 'joint_for_session': '2' }) - form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) self.assertTrue(form.is_valid()) # Test with one session @@ -1036,9 +1008,9 @@ def test_valid(self): 'joint_for_session': '1', 'session_time_relation': '', }) - form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) self.assertTrue(form.is_valid()) - + def test_invalid_groups(self): new_form_data = { 'constraint_chair_conflict': 'doesnotexist', @@ -1057,7 +1029,7 @@ def test_valid_group_appears_in_multiple_conflicts(self): 'constraint_tech_overlap': self.group2.acronym, } self.valid_form_data.update(new_form_data) - form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) self.assertTrue(form.is_valid()) def test_invalid_group_appears_in_multiple_conflicts(self): @@ -1116,7 +1088,7 @@ def test_invalid_joint_for_session(self): 'joint_for_session': [ 'Session 2 can not be the joint session, the session has not been requested.'] }) - + def test_invalid_missing_session_length(self): form = self._invalid_test_helper({ 'session_set-TOTAL_FORMS': '2', @@ -1156,6 +1128,6 @@ def test_invalid_missing_session_length(self): def _invalid_test_helper(self, new_form_data): form_data = dict(self.valid_form_data, **new_form_data) - form = SessionForm(data=form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=form_data, group=self.group1, meeting=self.meeting) self.assertFalse(form.is_valid()) return form diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index bd3ab772fca..b1bbc629071 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2009-2024, All Rights Reserved +# Copyright The IETF Trust 2009-2025, All Rights Reserved # -*- coding: utf-8 -*- import datetime import io @@ -7554,7 +7554,7 @@ def test_meeting_requests(self): ) def _sreq_edit_link(sess): return urlreverse( - 'ietf.secr.sreq.views.edit', + 'ietf.meeting.views_session_request.edit_request', kwargs={ 'num': meeting.number, 'acronym': sess.group.acronym, diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py index 18b123b4d89..af36a6656c3 100644 --- a/ietf/meeting/urls.py +++ b/ietf/meeting/urls.py @@ -1,10 +1,10 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved from django.conf import settings from django.urls import include from django.views.generic import RedirectView -from ietf.meeting import views, views_proceedings +from ietf.meeting import views, views_proceedings, views_session_request from ietf.utils.urls import url class AgendaRedirectView(RedirectView): @@ -108,6 +108,8 @@ def get_redirect_url(self, *args, **kwargs): url(r'^important-dates.(?Pics)$', views.important_dates), url(r'^proceedings/meetinghosts/edit/', views_proceedings.edit_meetinghosts), url(r'^proceedings/meetinghosts/(?P\d+)/logo/$', views_proceedings.meetinghost_logo), + url(r'^session/request/%(acronym)s/edit/$' % settings.URL_REGEXPS, views_session_request.edit_request), + url(r'^session/request/%(acronym)s/view/$' % settings.URL_REGEXPS, views_session_request.view_request), ] urlpatterns = [ @@ -127,6 +129,13 @@ def get_redirect_url(self, *args, **kwargs): url(r'^upcoming/?$', views.upcoming), url(r'^upcoming\.ics/?$', views.upcoming_ical), url(r'^upcoming\.json/?$', views.upcoming_json), + url(r'^session/request/$', views_session_request.list_view), + url(r'^session/request/%(acronym)s/new/$' % settings.URL_REGEXPS, views_session_request.new_request), + url(r'^session/request/%(acronym)s/approve/$' % settings.URL_REGEXPS, views_session_request.approve_request), + url(r'^session/request/%(acronym)s/no_session/$' % settings.URL_REGEXPS, views_session_request.no_session), + url(r'^session/request/%(acronym)s/cancel/$' % settings.URL_REGEXPS, views_session_request.cancel_request), + url(r'^session/request/%(acronym)s/confirm/$' % settings.URL_REGEXPS, views_session_request.confirm), + url(r'^session/request/status/$', views_session_request.status), url(r'^session/(?P\d+)/agenda_materials$', views.session_materials), url(r'^session/(?P\d+)/cancel/?', views.cancel_session), url(r'^session/(?P\d+)/edit/?', views.edit_session), @@ -140,4 +149,3 @@ def get_redirect_url(self, *args, **kwargs): url(r'^(?P\d+)/', include(safe_for_all_meeting_types)), url(r'^(?Pinterim-[a-z0-9-]+)/', include(safe_for_all_meeting_types)), ] - diff --git a/ietf/secr/sreq/views.py b/ietf/meeting/views_session_request.py similarity index 80% rename from ietf/secr/sreq/views.py rename to ietf/meeting/views_session_request.py index eb93168e1ce..a1ef74f1b88 100644 --- a/ietf/secr/sreq/views.py +++ b/ietf/meeting/views_session_request.py @@ -1,29 +1,26 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved # -*- coding: utf-8 -*- - import datetime import inflect from collections import defaultdict, OrderedDict from django.conf import settings from django.contrib import messages +from django.core.exceptions import ObjectDoesNotExist from django.db.models import Q from django.shortcuts import render, get_object_or_404, redirect from django.http import Http404 -import debug # pyflakes:ignore - from ietf.group.models import Group, GroupFeatures from ietf.ietfauth.utils import has_role, role_required -from ietf.meeting.models import Meeting, Session, Constraint, ResourceAssociation, SchedulingEvent from ietf.meeting.helpers import get_meeting +from ietf.meeting.models import Session, Meeting, Constraint, ResourceAssociation, SchedulingEvent from ietf.meeting.utils import add_event_info_to_session_qs -from ietf.name.models import SessionStatusName, ConstraintName -from ietf.secr.sreq.forms import (SessionForm, ToolStatusForm, allowed_conflicting_groups, +from ietf.meeting.forms import (SessionRequestStatusForm, SessionRequestForm, allowed_conflicting_groups, JOINT_FOR_SESSION_CHOICES) +from ietf.name.models import SessionStatusName, ConstraintName from ietf.secr.utils.decorators import check_permissions -from ietf.secr.utils.group import get_my_groups from ietf.utils.mail import send_mail from ietf.mailtrigger.utils import gather_address_lists @@ -31,12 +28,25 @@ # Globals # ------------------------------------------------- # TODO: This needs to be replaced with something that pays attention to groupfeatures -AUTHORIZED_ROLES=('WG Chair','WG Secretary','RG Chair','IAB Group Chair','Area Director','Secretariat','Team Chair','IRTF Chair','Program Chair','Program Lead','Program Secretary', 'EDWG Chair') +AUTHORIZED_ROLES = ( + 'WG Chair', + 'WG Secretary', + 'RG Chair', + 'IAB Group Chair', + 'Area Director', + 'Secretariat', + 'Team Chair', + 'IRTF Chair', + 'Program Chair', + 'Program Lead', + 'Program Secretary', + 'EDWG Chair') # ------------------------------------------------- # Helper Functions # ------------------------------------------------- + def check_app_locked(meeting=None): ''' This function returns True if the application is locked to non-secretariat users. @@ -45,6 +55,54 @@ def check_app_locked(meeting=None): meeting = get_meeting(days=14) return bool(meeting.session_request_lock_message) + +def get_lock_message(meeting=None): + ''' + Returns the message to display to non-secretariat users when the tool is locked. + ''' + if not meeting: + meeting = get_meeting(days=14) + return meeting.session_request_lock_message + + +def get_my_groups(user, conclude=False): + ''' + Takes a Django user object (from request) + Returns a list of groups the user has access to. Rules are as follows + secretariat - has access to all groups + area director - has access to all groups in their area + wg chair or secretary - has access to their own group + chair of irtf has access to all irtf groups + + If user=None than all groups are returned. + concluded=True means include concluded groups. Need this to upload materials for groups + after they've been concluded. it happens. + ''' + my_groups = set() + states = ['bof', 'proposed', 'active'] + if conclude: + states.extend(['conclude', 'bof-conc']) + + all_groups = Group.objects.filter(type__features__has_meetings=True, state__in=states).order_by('acronym') + if user is None or has_role(user, 'Secretariat'): + return all_groups + + try: + person = user.person + except ObjectDoesNotExist: + return list() + + for group in all_groups: + if group.role_set.filter(person=person, name__in=('chair', 'secr', 'ad')): + my_groups.add(group) + continue + if group.parent and group.parent.role_set.filter(person=person, name__in=('ad', 'chair')): + my_groups.add(group) + continue + + return list(my_groups) + + def get_initial_session(sessions, prune_conflicts=False): ''' This function takes a queryset of sessions ordered by 'id' for consistency. It returns @@ -97,13 +155,43 @@ def valid_conflict(conflict): initial['joint_for_session_display'] = dict(JOINT_FOR_SESSION_CHOICES)[initial['joint_for_session']] return initial -def get_lock_message(meeting=None): + +def inbound_session_conflicts_as_string(group, meeting): ''' - Returns the message to display to non-secretariat users when the tool is locked. + Takes a Group object and Meeting object and returns a string of other groups which have + a conflict with this one ''' - if not meeting: - meeting = get_meeting(days=14) - return meeting.session_request_lock_message + constraints = group.constraint_target_set.filter(meeting=meeting, name__is_group_conflict=True) + group_set = set(constraints.values_list('source__acronym', flat=True)) # set to de-dupe + group_list = sorted(group_set) # give a consistent order + return ', '.join(group_list) + + +def get_outbound_conflicts(form: SessionRequestForm): + """extract wg conflict constraint data from a SessionForm""" + outbound_conflicts = [] + for conflictname, cfield_id in form.wg_constraint_field_ids(): + conflict_groups = form.cleaned_data[cfield_id] + if len(conflict_groups) > 0: + outbound_conflicts.append(dict(name=conflictname, groups=conflict_groups)) + return outbound_conflicts + + +def save_conflicts(group, meeting, conflicts, name): + ''' + This function takes a Group, Meeting a string which is a list of Groups acronyms (conflicts), + and the constraint name (conflict|conflic2|conflic3) and creates Constraint records + ''' + constraint_name = ConstraintName.objects.get(slug=name) + acronyms = conflicts.replace(',',' ').split() + for acronym in acronyms: + target = Group.objects.get(acronym=acronym) + + constraint = Constraint(source=group, + target=target, + meeting=meeting, + name=constraint_name) + constraint.save() def get_requester_text(person, group): @@ -129,22 +217,6 @@ def get_requester_text(person, group): ) -def save_conflicts(group, meeting, conflicts, name): - ''' - This function takes a Group, Meeting a string which is a list of Groups acronyms (conflicts), - and the constraint name (conflict|conflic2|conflic3) and creates Constraint records - ''' - constraint_name = ConstraintName.objects.get(slug=name) - acronyms = conflicts.replace(',',' ').split() - for acronym in acronyms: - target = Group.objects.get(acronym=acronym) - - constraint = Constraint(source=group, - target=target, - meeting=meeting, - name=constraint_name) - constraint.save() - def send_notification(group, meeting, login, sreq_data, session_data, action): ''' This function generates email notifications for various session request activities. @@ -152,10 +224,10 @@ def send_notification(group, meeting, login, sreq_data, session_data, action): session_data is an array of data from individual session subforms action argument is a string [new|update]. ''' - (to_email, cc_list) = gather_address_lists('session_requested',group=group,person=login) + (to_email, cc_list) = gather_address_lists('session_requested', group=group, person=login) from_email = (settings.SESSION_REQUEST_FROM_EMAIL) subject = '%s - New Meeting Session Request for IETF %s' % (group.acronym, meeting.number) - template = 'sreq/session_request_notification.txt' + template = 'meeting/session_request_notification.txt' # send email context = {} @@ -164,7 +236,7 @@ def send_notification(group, meeting, login, sreq_data, session_data, action): context['meeting'] = meeting context['login'] = login context['header'] = 'A new' - context['requester'] = get_requester_text(login,group) + context['requester'] = get_requester_text(login, group) # update overrides if action == 'update': @@ -174,10 +246,10 @@ def send_notification(group, meeting, login, sreq_data, session_data, action): # if third session requested approval is required # change headers TO=ADs, CC=session-request, submitter and cochairs if len(session_data) > 2: - (to_email, cc_list) = gather_address_lists('session_requested_long',group=group,person=login) + (to_email, cc_list) = gather_address_lists('session_requested_long', group=group, person=login) subject = '%s - Request for meeting session approval for IETF %s' % (group.acronym, meeting.number) - template = 'sreq/session_approval_notification.txt' - #status_text = 'the %s Directors for approval' % group.parent + template = 'meeting/session_approval_notification.txt' + # status_text = 'the %s Directors for approval' % group.parent context['session_lengths'] = [sd['requested_duration'] for sd in session_data] @@ -189,103 +261,188 @@ def send_notification(group, meeting, login, sreq_data, session_data, action): context, cc=cc_list) -def inbound_session_conflicts_as_string(group, meeting): - ''' - Takes a Group object and Meeting object and returns a string of other groups which have - a conflict with this one - ''' - constraints = group.constraint_target_set.filter(meeting=meeting, name__is_group_conflict=True) - group_set = set(constraints.values_list('source__acronym', flat=True)) # set to de-dupe - group_list = sorted(group_set) # give a consistent order - return ', '.join(group_list) + +def session_changed(session): + latest_event = SchedulingEvent.objects.filter(session=session).order_by('-time', '-id').first() + + if latest_event and latest_event.status_id == "schedw" and session.meeting.schedule is not None: + # send an email to iesg-secretariat to alert to change + pass + + +def status_slug_for_new_session(session, session_number): + if session.group.features.acts_like_wg and session_number == 2: + return 'apprw' + return 'schedw' # ------------------------------------------------- # View Functions # ------------------------------------------------- -@check_permissions -def approve(request, acronym): + + +@role_required(*AUTHORIZED_ROLES) +def list_view(request): ''' - This view approves the third session. For use by ADs or Secretariat. + Display list of groups the user has access to. ''' meeting = get_meeting(days=14) - group = get_object_or_404(Group, acronym=acronym) - session = add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group=group)).filter(current_status='apprw').first() - if session is None: - raise Http404 + # check for locked flag + is_locked = check_app_locked() + if is_locked and not has_role(request.user, 'Secretariat'): + message = get_lock_message() + return render(request, 'meeting/session_request_locked.html', { + 'message': message, + 'meeting': meeting}) - if has_role(request.user,'Secretariat') or group.parent.role_set.filter(name='ad',person=request.user.person): - SchedulingEvent.objects.create( - session=session, - status=SessionStatusName.objects.get(slug='appr'), - by=request.user.person, - ) - session_changed(session) + scheduled_groups = [] + unscheduled_groups = [] - messages.success(request, 'Third session approved') - return redirect('ietf.secr.sreq.views.view', acronym=acronym) - else: - # if an unauthorized user gets here return error - messages.error(request, 'Not authorized to approve the third session') - return redirect('ietf.secr.sreq.views.view', acronym=acronym) + group_types = GroupFeatures.objects.filter(has_meetings=True).values_list('type', flat=True) -@check_permissions -def cancel(request, acronym): - ''' - This view cancels a session request and sends a notification. - To cancel, or withdraw the request set status = deleted. - "canceled" status is used by the secretariat. + my_groups = [g for g in get_my_groups(request.user, conclude=True) if g.type_id in group_types] - NOTE: this function can also be called after a session has been - scheduled during the period when the session request tool is - reopened. In this case be sure to clear the timeslot assignment as well. + sessions_by_group = defaultdict(list) + for s in add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group__in=my_groups)).filter(current_status__in=['schedw', 'apprw', 'appr', 'sched']): + sessions_by_group[s.group_id].append(s) + + for group in my_groups: + group.meeting_sessions = sessions_by_group.get(group.pk, []) + + if group.pk in sessions_by_group: + # include even if concluded as we need to to see that the + # sessions are there + scheduled_groups.append(group) + else: + if group.state_id not in ['conclude', 'bof-conc']: + # too late for unscheduled if concluded + unscheduled_groups.append(group) + + # warn if there are no associated groups + if not scheduled_groups and not unscheduled_groups: + messages.warning(request, 'The account %s is not associated with any groups. If you have multiple Datatracker accounts you may try another or report a problem to %s' % (request.user, settings.SECRETARIAT_ACTION_EMAIL)) + + # add session status messages for use in template + for group in scheduled_groups: + if not group.features.acts_like_wg or (len(group.meeting_sessions) < 3): + group.status_message = group.meeting_sessions[0].current_status + else: + group.status_message = 'First two sessions: %s, Third session: %s' % (group.meeting_sessions[0].current_status, group.meeting_sessions[2].current_status) + + # add not meeting indicators for use in template + for group in unscheduled_groups: + if any(s.current_status == 'notmeet' for s in group.meeting_sessions): + group.not_meeting = True + + return render(request, 'meeting/session_request_list.html', { + 'is_locked': is_locked, + 'meeting': meeting, + 'scheduled_groups': scheduled_groups, + 'unscheduled_groups': unscheduled_groups}, + ) + + +@role_required('Secretariat') +def status(request): + ''' + This view handles locking and unlocking of the session request tool to the public. ''' meeting = get_meeting(days=14) - group = get_object_or_404(Group, acronym=acronym) - sessions = Session.objects.filter(meeting=meeting,group=group).order_by('id') - login = request.user.person + is_locked = check_app_locked(meeting=meeting) - # delete conflicts - Constraint.objects.filter(meeting=meeting,source=group).delete() + if request.method == 'POST': + button_text = request.POST.get('submit', '') + if button_text == 'Back': + return redirect('ietf.meeting.views_session_request.list_view') - # mark sessions as deleted - for session in sessions: - SchedulingEvent.objects.create( - session=session, - status=SessionStatusName.objects.get(slug='deleted'), - by=request.user.person, - ) - session_changed(session) + form = SessionRequestStatusForm(request.POST) - # clear schedule assignments if already scheduled - session.timeslotassignments.all().delete() + if button_text == 'Lock': + if form.is_valid(): + meeting.session_request_lock_message = form.cleaned_data['message'] + meeting.save() + messages.success(request, 'Session Request Tool is now Locked') + return redirect('ietf.meeting.views_session_request.list_view') - # send notifitcation - (to_email, cc_list) = gather_address_lists('session_request_cancelled',group=group,person=login) - from_email = (settings.SESSION_REQUEST_FROM_EMAIL) - subject = '%s - Cancelling a meeting request for IETF %s' % (group.acronym, meeting.number) - send_mail(request, to_email, from_email, subject, 'sreq/session_cancel_notification.txt', - {'requester':get_requester_text(login,group), - 'meeting':meeting}, cc=cc_list) + elif button_text == 'Unlock': + meeting.session_request_lock_message = '' + meeting.save() + messages.success(request, 'Session Request Tool is now Unlocked') + return redirect('ietf.meeting.views_session_request.list_view') - messages.success(request, 'The %s Session Request has been cancelled' % group.acronym) - return redirect('ietf.secr.sreq.views.main') + else: + if is_locked: + message = get_lock_message() + initial = {'message': message} + form = SessionRequestStatusForm(initial=initial) + else: + form = SessionRequestStatusForm() + return render(request, 'meeting/session_request_status.html', { + 'is_locked': is_locked, + 'form': form}, + ) -def status_slug_for_new_session(session, session_number): - if session.group.features.acts_like_wg and session_number == 2: - return 'apprw' - return 'schedw' +@check_permissions +def new_request(request, acronym): + ''' + This view gathers details for a new session request. The user proceeds to confirm() + to create the request. + ''' + group = get_object_or_404(Group, acronym=acronym) + if len(group.features.session_purposes) == 0: + raise Http404(f'Cannot request sessions for group "{acronym}"') + meeting = get_meeting(days=14) + session_conflicts = dict(inbound=inbound_session_conflicts_as_string(group, meeting)) -def get_outbound_conflicts(form: SessionForm): - """extract wg conflict constraint data from a SessionForm""" - outbound_conflicts = [] - for conflictname, cfield_id in form.wg_constraint_field_ids(): - conflict_groups = form.cleaned_data[cfield_id] - if len(conflict_groups) > 0: - outbound_conflicts.append(dict(name=conflictname, groups=conflict_groups)) - return outbound_conflicts + # check if app is locked + is_locked = check_app_locked() + if is_locked and not has_role(request.user, 'Secretariat'): + messages.warning(request, "The Session Request Tool is closed") + return redirect('ietf.meeting.views_session_request.list_view') + + if request.method == 'POST': + button_text = request.POST.get('submit', '') + if button_text == 'Cancel': + return redirect('ietf.meeting.views_session_request.list_view') + + form = SessionRequestForm(group, meeting, request.POST, notifications_optional=has_role(request.user, "Secretariat")) + if form.is_valid(): + return confirm(request, acronym) + + # the "previous" querystring causes the form to be returned + # pre-populated with data from last meeeting's session request + elif request.method == 'GET' and 'previous' in request.GET: + latest_session = add_event_info_to_session_qs(Session.objects.filter(meeting__type_id='ietf', group=group)).exclude(current_status__in=['notmeet', 'deleted', 'canceled',]).order_by('-meeting__date').first() + if latest_session: + previous_meeting = Meeting.objects.get(number=latest_session.meeting.number) + previous_sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=previous_meeting, group=group)).exclude(current_status__in=['notmeet', 'deleted']).order_by('id') + if not previous_sessions: + messages.warning(request, 'This group did not meet at %s' % previous_meeting) + return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym) + else: + messages.info(request, 'Fetched session info from %s' % previous_meeting) + else: + messages.warning(request, 'Did not find any previous meeting') + return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym) + + initial = get_initial_session(previous_sessions, prune_conflicts=True) + if 'resources' in initial: + initial['resources'] = [x.pk for x in initial['resources']] + form = SessionRequestForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + + else: + initial = {} + form = SessionRequestForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + + return render(request, 'meeting/session_request_form.html', { + 'meeting': meeting, + 'form': form, + 'group': group, + 'is_create': True, + 'session_conflicts': session_conflicts}, + ) @role_required(*AUTHORIZED_ROLES) @@ -295,11 +452,11 @@ def confirm(request, acronym): to confirm for submission. ''' # FIXME: this should be using form.is_valid/form.cleaned_data - invalid input will make it crash - group = get_object_or_404(Group,acronym=acronym) + group = get_object_or_404(Group, acronym=acronym) if len(group.features.session_purposes) == 0: raise Http404(f'Cannot request sessions for group "{acronym}"') meeting = get_meeting(days=14) - form = SessionForm(group, meeting, request.POST, hidden=True, notifications_optional=has_role(request.user, "Secretariat")) + form = SessionRequestForm(group, meeting, request.POST, hidden=True, notifications_optional=has_role(request.user, "Secretariat")) form.is_valid() login = request.user.person @@ -307,8 +464,8 @@ def confirm(request, acronym): # check if request already exists for this group if add_event_info_to_session_qs(Session.objects.filter(group=group, meeting=meeting)).filter(Q(current_status__isnull=True) | ~Q(current_status__in=['deleted', 'notmeet'])): messages.warning(request, 'Sessions for working group %s have already been requested once.' % group.acronym) - return redirect('ietf.secr.sreq.views.main') - + return redirect('ietf.meeting.views_session_request.list_view') + session_data = form.data.copy() # use cleaned_data for the 'bethere' field so we get the Person instances session_data['bethere'] = form.cleaned_data['bethere'] if 'bethere' in form.cleaned_data else [] @@ -318,7 +475,7 @@ def confirm(request, acronym): session_data['joint_for_session_display'] = dict(JOINT_FOR_SESSION_CHOICES)[session_data['joint_for_session']] if form.cleaned_data.get('timeranges'): session_data['timeranges_display'] = [t.desc for t in form.cleaned_data['timeranges']] - session_data['resources'] = [ ResourceAssociation.objects.get(pk=pk) for pk in request.POST.getlist('resources') ] + session_data['resources'] = [ResourceAssociation.objects.get(pk=pk) for pk in request.POST.getlist('resources')] # extract wg conflict constraint data for the view / notifications outbound_conflicts = get_outbound_conflicts(form) @@ -326,7 +483,7 @@ def confirm(request, acronym): button_text = request.POST.get('submit', '') if button_text == 'Cancel': messages.success(request, 'Session Request has been cancelled') - return redirect('ietf.secr.sreq.views.main') + return redirect('ietf.meeting.views_session_request.list_view') if request.method == 'POST' and button_text == 'Submit': # delete any existing session records with status = canceled or notmeet @@ -344,10 +501,10 @@ def confirm(request, acronym): if 'resources' in form.data: new_session.resources.set(session_data['resources']) jfs = form.data.get('joint_for_session', '-1') - if not jfs: # jfs might be '' + if not jfs: # jfs might be '' jfs = '-1' if int(jfs) == count + 1: # count is zero-indexed - groups_split = form.cleaned_data.get('joint_with_groups').replace(',',' ').split() + groups_split = form.cleaned_data.get('joint_with_groups').replace(',', ' ').split() joint = Group.objects.filter(acronym__in=groups_split) new_session.joint_with_groups.set(joint) new_session.save() @@ -388,36 +545,105 @@ def confirm(request, acronym): 'new', ) - status_text = 'IETF Agenda to be scheduled' - messages.success(request, 'Your request has been sent to %s' % status_text) - return redirect('ietf.secr.sreq.views.main') + status_text = 'IETF Agenda to be scheduled' + messages.success(request, 'Your request has been sent to %s' % status_text) + return redirect('ietf.meeting.views_session_request.list_view') + + # POST from request submission + session_conflicts = dict( + outbound=outbound_conflicts, # each is a dict with name and groups as keys + inbound=inbound_session_conflicts_as_string(group, meeting), + ) + if form.cleaned_data.get('third_session'): + messages.warning(request, 'Note: Your request for a third session must be approved by an area director before being submitted to agenda@ietf.org. Click "Submit" below to email an approval request to the area directors') + + return render(request, 'meeting/session_request_confirm.html', { + 'form': form, + 'session': session_data, + 'group': group, + 'meeting': meeting, + 'session_conflicts': session_conflicts}, + ) + + +@role_required(*AUTHORIZED_ROLES) +def view_request(request, acronym, num=None): + ''' + This view displays the session request info + ''' + meeting = get_meeting(num, days=14) + group = get_object_or_404(Group, acronym=acronym) + query = Session.objects.filter(meeting=meeting, group=group) + status_is_null = Q(current_status__isnull=True) + status_allowed = ~Q(current_status__in=("canceled", "notmeet", "deleted")) + sessions = ( + add_event_info_to_session_qs(query) + .filter(status_is_null | status_allowed) + .order_by("id") + ) + + # check if app is locked + is_locked = check_app_locked() + if is_locked: + messages.warning(request, "The Session Request Tool is closed") + + # if there are no session requests yet, redirect to new session request page + if not sessions: + if is_locked: + return redirect('ietf.meeting.views_session_request.list_view') + else: + return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym) + + activities = [{ + 'act_date': e.time.strftime('%b %d, %Y'), + 'act_time': e.time.strftime('%H:%M:%S'), + 'activity': e.status.name, + 'act_by': e.by, + } for e in sessions[0].schedulingevent_set.select_related('status', 'by')] + + # gather outbound conflicts + outbound_dict = OrderedDict() + for obc in group.constraint_source_set.filter(meeting=meeting, name__is_group_conflict=True): + if obc.name.slug not in outbound_dict: + outbound_dict[obc.name.slug] = [] + outbound_dict[obc.name.slug].append(obc.target.acronym) - # POST from request submission session_conflicts = dict( - outbound=outbound_conflicts, # each is a dict with name and groups as keys inbound=inbound_session_conflicts_as_string(group, meeting), + outbound=[dict(name=ConstraintName.objects.get(slug=slug), groups=' '.join(groups)) + for slug, groups in outbound_dict.items()], ) - return render(request, 'sreq/confirm.html', { - 'form': form, - 'session': session_data, + + show_approve_button = False + + # if sessions include a 3rd session waiting approval and the user is a secretariat or AD of the group + # display approve button + if any(s.current_status == 'apprw' for s in sessions): + if has_role(request.user, 'Secretariat') or group.parent.role_set.filter(name='ad', person=request.user.person): + show_approve_button = True + + # build session dictionary (like querydict from new session request form) for use in template + session = get_initial_session(sessions) + + return render(request, 'meeting/session_request_view.html', { + 'can_edit': (not is_locked) or has_role(request.user, 'Secretariat'), + 'can_cancel': (not is_locked) or has_role(request.user, 'Secretariat'), + 'session': session, # legacy processed data + 'sessions': sessions, # actual session instances + 'activities': activities, + 'meeting': meeting, 'group': group, - 'session_conflicts': session_conflicts}, + 'session_conflicts': session_conflicts, + 'show_approve_button': show_approve_button}, ) - -def session_changed(session): - latest_event = SchedulingEvent.objects.filter(session=session).order_by('-time', '-id').first() - - if latest_event and latest_event.status_id == "schedw" and session.meeting.schedule != None: - # send an email to iesg-secretariat to alert to change - pass @check_permissions -def edit(request, acronym, num=None): +def edit_request(request, acronym, num=None): ''' This view allows the user to edit details of the session request ''' - meeting = get_meeting(num,days=14) + meeting = get_meeting(num, days=14) group = get_object_or_404(Group, acronym=acronym) if len(group.features.session_purposes) == 0: raise Http404(f'Cannot request sessions for group "{acronym}"') @@ -443,15 +669,15 @@ def edit(request, acronym, num=None): login = request.user.person first_session = Session() - if(len(sessions) > 0): + if (len(sessions) > 0): first_session = sessions[0] if request.method == 'POST': button_text = request.POST.get('submit', '') if button_text == 'Cancel': - return redirect('ietf.secr.sreq.views.view', acronym=acronym) + return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym) - form = SessionForm(group, meeting, request.POST, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + form = SessionRequestForm(group, meeting, request.POST, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) if form.is_valid(): if form.has_changed(): changed_session_forms = [sf for sf in form.session_forms.forms_to_keep if sf.has_changed()] @@ -513,11 +739,11 @@ def edit(request, acronym, num=None): if 'resources' in form.changed_data: new_resource_ids = form.cleaned_data['resources'] - new_resources = [ ResourceAssociation.objects.get(pk=a) - for a in new_resource_ids] + new_resources = [ResourceAssociation.objects.get(pk=a) + for a in new_resource_ids] first_session.resources = new_resources - if 'bethere' in form.changed_data and set(form.cleaned_data['bethere'])!=set(initial['bethere']): + if 'bethere' in form.changed_data and set(form.cleaned_data['bethere']) != set(initial['bethere']): first_session.constraints().filter(name='bethere').delete() bethere_cn = ConstraintName.objects.get(slug='bethere') for p in form.cleaned_data['bethere']: @@ -539,7 +765,7 @@ def edit(request, acronym, num=None): # deprecated # log activity - #add_session_activity(group,'Session Request was updated',meeting,user) + # add_session_activity(group,'Session Request was updated',meeting,user) # send notification if form.cleaned_data.get("send_notifications"): @@ -556,7 +782,7 @@ def edit(request, acronym, num=None): ) messages.success(request, 'Session Request updated') - return redirect('ietf.secr.sreq.views.view', acronym=acronym) + return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym) else: # method is not POST # gather outbound conflicts for initial value @@ -567,142 +793,46 @@ def edit(request, acronym, num=None): initial['constraint_{}'.format(slug)] = ' '.join(groups) if not sessions: - return redirect('ietf.secr.sreq.views.new', acronym=acronym) - form = SessionForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym) + form = SessionRequestForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) - return render(request, 'sreq/edit.html', { - 'is_locked': is_locked and not has_role(request.user,'Secretariat'), + return render(request, 'meeting/session_request_form.html', { + 'is_locked': is_locked and not has_role(request.user, 'Secretariat'), 'meeting': meeting, 'form': form, 'group': group, + 'is_create': False, 'session_conflicts': session_conflicts}, ) -@role_required(*AUTHORIZED_ROLES) -def main(request): - ''' - Display list of groups the user has access to. - - Template variables - form: a select box populated with unscheduled groups - meeting: the current meeting - scheduled_sessions: - ''' - # check for locked flag - is_locked = check_app_locked() - - if is_locked and not has_role(request.user,'Secretariat'): - message = get_lock_message() - return render(request, 'sreq/locked.html', { - 'message': message}, - ) - - meeting = get_meeting(days=14) - - scheduled_groups = [] - unscheduled_groups = [] - - group_types = GroupFeatures.objects.filter(has_meetings=True).values_list('type', flat=True) - - my_groups = [g for g in get_my_groups(request.user, conclude=True) if g.type_id in group_types] - - sessions_by_group = defaultdict(list) - for s in add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group__in=my_groups)).filter(current_status__in=['schedw', 'apprw', 'appr', 'sched']): - sessions_by_group[s.group_id].append(s) - - for group in my_groups: - group.meeting_sessions = sessions_by_group.get(group.pk, []) - - if group.pk in sessions_by_group: - # include even if concluded as we need to to see that the - # sessions are there - scheduled_groups.append(group) - else: - if group.state_id not in ['conclude', 'bof-conc']: - # too late for unscheduled if concluded - unscheduled_groups.append(group) - - # warn if there are no associated groups - if not scheduled_groups and not unscheduled_groups: - messages.warning(request, 'The account %s is not associated with any groups. If you have multiple Datatracker accounts you may try another or report a problem to %s' % (request.user, settings.SECRETARIAT_ACTION_EMAIL)) - - # add session status messages for use in template - for group in scheduled_groups: - if not group.features.acts_like_wg or (len(group.meeting_sessions) < 3): - group.status_message = group.meeting_sessions[0].current_status - else: - group.status_message = 'First two sessions: %s, Third session: %s' % (group.meeting_sessions[0].current_status, group.meeting_sessions[2].current_status) - - # add not meeting indicators for use in template - for group in unscheduled_groups: - if any(s.current_status == 'notmeet' for s in group.meeting_sessions): - group.not_meeting = True - - return render(request, 'sreq/main.html', { - 'is_locked': is_locked, - 'meeting': meeting, - 'scheduled_groups': scheduled_groups, - 'unscheduled_groups': unscheduled_groups}, - ) @check_permissions -def new(request, acronym): +def approve_request(request, acronym): ''' - This view gathers details for a new session request. The user proceeds to confirm() - to create the request. + This view approves the third session. For use by ADs or Secretariat. ''' - group = get_object_or_404(Group, acronym=acronym) - if len(group.features.session_purposes) == 0: - raise Http404(f'Cannot request sessions for group "{acronym}"') meeting = get_meeting(days=14) - session_conflicts = dict(inbound=inbound_session_conflicts_as_string(group, meeting)) - - # check if app is locked - is_locked = check_app_locked() - if is_locked and not has_role(request.user,'Secretariat'): - messages.warning(request, "The Session Request Tool is closed") - return redirect('ietf.secr.sreq.views.main') - - if request.method == 'POST': - button_text = request.POST.get('submit', '') - if button_text == 'Cancel': - return redirect('ietf.secr.sreq.views.main') - - form = SessionForm(group, meeting, request.POST, notifications_optional=has_role(request.user, "Secretariat")) - if form.is_valid(): - return confirm(request, acronym) + group = get_object_or_404(Group, acronym=acronym) - # the "previous" querystring causes the form to be returned - # pre-populated with data from last meeeting's session request - elif request.method == 'GET' and 'previous' in request.GET: - latest_session = add_event_info_to_session_qs(Session.objects.filter(meeting__type_id='ietf', group=group)).exclude(current_status__in=['notmeet', 'deleted', 'canceled',]).order_by('-meeting__date').first() - if latest_session: - previous_meeting = Meeting.objects.get(number=latest_session.meeting.number) - previous_sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=previous_meeting, group=group)).exclude(current_status__in=['notmeet', 'deleted']).order_by('id') - if not previous_sessions: - messages.warning(request, 'This group did not meet at %s' % previous_meeting) - return redirect('ietf.secr.sreq.views.new', acronym=acronym) - else: - messages.info(request, 'Fetched session info from %s' % previous_meeting) - else: - messages.warning(request, 'Did not find any previous meeting') - return redirect('ietf.secr.sreq.views.new', acronym=acronym) + session = add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group=group)).filter(current_status='apprw').first() + if session is None: + raise Http404 - initial = get_initial_session(previous_sessions, prune_conflicts=True) - if 'resources' in initial: - initial['resources'] = [x.pk for x in initial['resources']] - form = SessionForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + if has_role(request.user, 'Secretariat') or group.parent.role_set.filter(name='ad', person=request.user.person): + SchedulingEvent.objects.create( + session=session, + status=SessionStatusName.objects.get(slug='appr'), + by=request.user.person, + ) + session_changed(session) + messages.success(request, 'Third session approved') + return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym) else: - initial={} - form = SessionForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + # if an unauthorized user gets here return error + messages.error(request, 'Not authorized to approve the third session') + return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym) - return render(request, 'sreq/new.html', { - 'meeting': meeting, - 'form': form, - 'group': group, - 'session_conflicts': session_conflicts}, - ) @check_permissions def no_session(request, acronym): @@ -722,7 +852,7 @@ def no_session(request, acronym): # skip if state is already notmeet if add_event_info_to_session_qs(Session.objects.filter(group=group, meeting=meeting)).filter(current_status='notmeet'): messages.info(request, 'The group %s is already marked as not meeting' % group.acronym) - return redirect('ietf.secr.sreq.views.main') + return redirect('ietf.meeting.views_session_request.list_view') session = Session.objects.create( group=group, @@ -740,125 +870,62 @@ def no_session(request, acronym): session_changed(session) # send notification - (to_email, cc_list) = gather_address_lists('session_request_not_meeting',group=group,person=login) + (to_email, cc_list) = gather_address_lists('session_request_not_meeting', group=group, person=login) from_email = (settings.SESSION_REQUEST_FROM_EMAIL) subject = '%s - Not having a session at IETF %s' % (group.acronym, meeting.number) - send_mail(request, to_email, from_email, subject, 'sreq/not_meeting_notification.txt', - {'login':login, - 'group':group, - 'meeting':meeting}, cc=cc_list) + send_mail(request, to_email, from_email, subject, 'meeting/session_not_meeting_notification.txt', + {'login': login, + 'group': group, + 'meeting': meeting}, cc=cc_list) # deprecated? # log activity - #text = 'A message was sent to notify not having a session at IETF %d' % meeting.meeting_num - #add_session_activity(group,text,meeting,request.person) + # text = 'A message was sent to notify not having a session at IETF %d' % meeting.meeting_num + # add_session_activity(group,text,meeting,request.person) # redirect messages.success(request, 'A message was sent to notify not having a session at IETF %s' % meeting.number) - return redirect('ietf.secr.sreq.views.main') - -@role_required('Secretariat') -def tool_status(request): - ''' - This view handles locking and unlocking of the tool to the public. - ''' - meeting = get_meeting(days=14) - is_locked = check_app_locked(meeting=meeting) - - if request.method == 'POST': - button_text = request.POST.get('submit', '') - if button_text == 'Back': - return redirect('ietf.secr.sreq.views.main') - - form = ToolStatusForm(request.POST) - - if button_text == 'Lock': - if form.is_valid(): - meeting.session_request_lock_message = form.cleaned_data['message'] - meeting.save() - messages.success(request, 'Session Request Tool is now Locked') - return redirect('ietf.secr.sreq.views.main') - - elif button_text == 'Unlock': - meeting.session_request_lock_message = '' - meeting.save() - messages.success(request, 'Session Request Tool is now Unlocked') - return redirect('ietf.secr.sreq.views.main') - - else: - if is_locked: - message = get_lock_message() - initial = {'message': message} - form = ToolStatusForm(initial=initial) - else: - form = ToolStatusForm() + return redirect('ietf.meeting.views_session_request.list_view') - return render(request, 'sreq/tool_status.html', { - 'is_locked': is_locked, - 'form': form}, - ) -@role_required(*AUTHORIZED_ROLES) -def view(request, acronym, num = None): +@check_permissions +def cancel_request(request, acronym): ''' - This view displays the session request info + This view cancels a session request and sends a notification. + To cancel, or withdraw the request set status = deleted. + "canceled" status is used by the secretariat. + + NOTE: this function can also be called after a session has been + scheduled during the period when the session request tool is + reopened. In this case be sure to clear the timeslot assignment as well. ''' - meeting = get_meeting(num,days=14) + meeting = get_meeting(days=14) group = get_object_or_404(Group, acronym=acronym) - sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group=group)).filter(Q(current_status__isnull=True) | ~Q(current_status__in=('canceled','notmeet','deleted'))).order_by('id') - - # check if app is locked - is_locked = check_app_locked() - if is_locked: - messages.warning(request, "The Session Request Tool is closed") - - # if there are no session requests yet, redirect to new session request page - if not sessions: - if is_locked: - return redirect('ietf.secr.sreq.views.main') - else: - return redirect('ietf.secr.sreq.views.new', acronym=acronym) - - activities = [{ - 'act_date': e.time.strftime('%b %d, %Y'), - 'act_time': e.time.strftime('%H:%M:%S'), - 'activity': e.status.name, - 'act_by': e.by, - } for e in sessions[0].schedulingevent_set.select_related('status', 'by')] - - # gather outbound conflicts - outbound_dict = OrderedDict() - for obc in group.constraint_source_set.filter(meeting=meeting, name__is_group_conflict=True): - if obc.name.slug not in outbound_dict: - outbound_dict[obc.name.slug] = [] - outbound_dict[obc.name.slug].append(obc.target.acronym) - - session_conflicts = dict( - inbound=inbound_session_conflicts_as_string(group, meeting), - outbound=[dict(name=ConstraintName.objects.get(slug=slug), groups=' '.join(groups)) - for slug, groups in outbound_dict.items()], - ) + sessions = Session.objects.filter(meeting=meeting, group=group).order_by('id') + login = request.user.person - show_approve_button = False + # delete conflicts + Constraint.objects.filter(meeting=meeting, source=group).delete() - # if sessions include a 3rd session waiting approval and the user is a secretariat or AD of the group - # display approve button - if any(s.current_status == 'apprw' for s in sessions): - if has_role(request.user,'Secretariat') or group.parent.role_set.filter(name='ad',person=request.user.person): - show_approve_button = True + # mark sessions as deleted + for session in sessions: + SchedulingEvent.objects.create( + session=session, + status=SessionStatusName.objects.get(slug='deleted'), + by=request.user.person, + ) + session_changed(session) - # build session dictionary (like querydict from new session request form) for use in template - session = get_initial_session(sessions) + # clear schedule assignments if already scheduled + session.timeslotassignments.all().delete() - return render(request, 'sreq/view.html', { - 'can_edit': (not is_locked) or has_role(request.user, 'Secretariat'), - 'can_cancel': (not is_locked) or has_role(request.user, 'Secretariat'), - 'session': session, # legacy processed data - 'sessions': sessions, # actual session instances - 'activities': activities, - 'meeting': meeting, - 'group': group, - 'session_conflicts': session_conflicts, - 'show_approve_button': show_approve_button}, - ) + # send notifitcation + (to_email, cc_list) = gather_address_lists('session_request_cancelled', group=group, person=login) + from_email = (settings.SESSION_REQUEST_FROM_EMAIL) + subject = '%s - Cancelling a meeting request for IETF %s' % (group.acronym, meeting.number) + send_mail(request, to_email, from_email, subject, 'meeting/session_cancel_notification.txt', + {'requester': get_requester_text(login, group), + 'meeting': meeting}, cc=cc_list) + messages.success(request, 'The %s Session Request has been cancelled' % group.acronym) + return redirect('ietf.meeting.views_session_request.list_view') diff --git a/ietf/secr/meetings/views.py b/ietf/secr/meetings/views.py index 47f7b7ffa57..1f6f2f3297a 100644 --- a/ietf/secr/meetings/views.py +++ b/ietf/secr/meetings/views.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2023, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved # -*- coding: utf-8 -*- import datetime @@ -20,12 +20,12 @@ from ietf.meeting.helpers import make_materials_directories, populate_important_dates from ietf.meeting.models import Meeting, Session, Room, TimeSlot, SchedTimeSessAssignment, Schedule, SchedulingEvent from ietf.meeting.utils import add_event_info_to_session_qs +from ietf.meeting.views_session_request import get_initial_session from ietf.name.models import SessionStatusName from ietf.group.models import Group, GroupEvent from ietf.secr.meetings.forms import ( BaseMeetingRoomFormSet, MeetingModelForm, MeetingSelectForm, MeetingRoomForm, MiscSessionForm, TimeSlotForm, RegularSessionEditForm, MeetingRoomOptionsForm ) -from ietf.secr.sreq.views import get_initial_session from ietf.secr.utils.meeting import get_session, get_timeslot from ietf.mailtrigger.utils import gather_address_lists from ietf.utils.timezone import make_aware diff --git a/ietf/secr/sreq/__init__.py b/ietf/secr/sreq/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/ietf/secr/sreq/forms.py b/ietf/secr/sreq/forms.py deleted file mode 100644 index 4a0f449b2a4..00000000000 --- a/ietf/secr/sreq/forms.py +++ /dev/null @@ -1,333 +0,0 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved -# -*- coding: utf-8 -*- - - -from django import forms -from django.template.defaultfilters import pluralize - -import debug # pyflakes:ignore - -from ietf.name.models import TimerangeName, ConstraintName -from ietf.group.models import Group -from ietf.meeting.forms import sessiondetailsformset_factory -from ietf.meeting.models import ResourceAssociation, Constraint -from ietf.person.fields import SearchablePersonsField -from ietf.person.models import Person -from ietf.utils.fields import ModelMultipleChoiceField -from ietf.utils.html import clean_text_field -from ietf.utils import log - -# ------------------------------------------------- -# Globals -# ------------------------------------------------- - -NUM_SESSION_CHOICES = (('','--Please select'),('1','1'),('2','2')) -SESSION_TIME_RELATION_CHOICES = (('', 'No preference'),) + Constraint.TIME_RELATION_CHOICES -JOINT_FOR_SESSION_CHOICES = (('1', 'First session'), ('2', 'Second session'), ('3', 'Third session'), ) - -# ------------------------------------------------- -# Helper Functions -# ------------------------------------------------- -def allowed_conflicting_groups(): - return Group.objects.filter(type__in=['wg', 'ag', 'rg', 'rag', 'program', 'edwg'], state__in=['bof', 'proposed', 'active']) - -def check_conflict(groups, source_group): - ''' - Takes a string which is a list of group acronyms. Checks that they are all active groups - ''' - # convert to python list (allow space or comma separated lists) - items = groups.replace(',',' ').split() - active_groups = allowed_conflicting_groups() - for group in items: - if group == source_group.acronym: - raise forms.ValidationError("Cannot declare a conflict with the same group: %s" % group) - - if not active_groups.filter(acronym=group): - raise forms.ValidationError("Invalid or inactive group acronym: %s" % group) - -# ------------------------------------------------- -# Forms -# ------------------------------------------------- - -class GroupSelectForm(forms.Form): - group = forms.ChoiceField() - - def __init__(self,*args,**kwargs): - choices = kwargs.pop('choices') - super(GroupSelectForm, self).__init__(*args,**kwargs) - self.fields['group'].widget.choices = choices - - -class NameModelMultipleChoiceField(ModelMultipleChoiceField): - def label_from_instance(self, name): - return name.desc - - -class SessionForm(forms.Form): - num_session = forms.ChoiceField(choices=NUM_SESSION_CHOICES) - # session fields are added in __init__() - session_time_relation = forms.ChoiceField(choices=SESSION_TIME_RELATION_CHOICES, required=False) - attendees = forms.IntegerField() - # FIXME: it would cleaner to have these be - # ModelMultipleChoiceField, and just customize the widgetry, that - # way validation comes for free (applies to this CharField and the - # constraints dynamically instantiated in __init__()) - joint_with_groups = forms.CharField(max_length=255,required=False) - joint_with_groups_selector = forms.ChoiceField(choices=[], required=False) # group select widget for prev field - joint_for_session = forms.ChoiceField(choices=JOINT_FOR_SESSION_CHOICES, required=False) - comments = forms.CharField(max_length=200,required=False) - third_session = forms.BooleanField(required=False) - resources = forms.MultipleChoiceField(widget=forms.CheckboxSelectMultiple,required=False) - bethere = SearchablePersonsField(label="Must be present", required=False) - timeranges = NameModelMultipleChoiceField(widget=forms.CheckboxSelectMultiple, required=False, - queryset=TimerangeName.objects.all()) - adjacent_with_wg = forms.ChoiceField(required=False) - send_notifications = forms.BooleanField(label="Send notification emails?", required=False, initial=False) - - def __init__(self, group, meeting, data=None, *args, **kwargs): - self.hidden = kwargs.pop('hidden', False) - self.notifications_optional = kwargs.pop('notifications_optional', False) - - self.group = group - formset_class = sessiondetailsformset_factory(max_num=3 if group.features.acts_like_wg else 50) - self.session_forms = formset_class(group=self.group, meeting=meeting, data=data) - super(SessionForm, self).__init__(data=data, *args, **kwargs) - if not self.notifications_optional: - self.fields['send_notifications'].widget = forms.HiddenInput() - - # Allow additional sessions for non-wg-like groups - if not self.group.features.acts_like_wg: - self.fields['num_session'].choices = ((n, str(n)) for n in range(1, 51)) - - self.fields['comments'].widget = forms.Textarea(attrs={'rows':'3','cols':'65'}) - - other_groups = list(allowed_conflicting_groups().exclude(pk=group.pk).values_list('acronym', 'acronym').order_by('acronym')) - self.fields['adjacent_with_wg'].choices = [('', '--No preference')] + other_groups - group_acronym_choices = [('','--Select WG(s)')] + other_groups - self.fields['joint_with_groups_selector'].choices = group_acronym_choices - - # Set up constraints for the meeting - self._wg_field_data = [] - for constraintname in meeting.group_conflict_types.all(): - # two fields for each constraint: a CharField for the group list and a selector to add entries - constraint_field = forms.CharField(max_length=255, required=False) - constraint_field.widget.attrs['data-slug'] = constraintname.slug - constraint_field.widget.attrs['data-constraint-name'] = str(constraintname).title() - self._add_widget_class(constraint_field.widget, 'wg_constraint') - - selector_field = forms.ChoiceField(choices=group_acronym_choices, required=False) - selector_field.widget.attrs['data-slug'] = constraintname.slug # used by onchange handler - self._add_widget_class(selector_field.widget, 'wg_constraint_selector') - - cfield_id = 'constraint_{}'.format(constraintname.slug) - cselector_id = 'wg_selector_{}'.format(constraintname.slug) - # keep an eye out for field name conflicts - log.assertion('cfield_id not in self.fields') - log.assertion('cselector_id not in self.fields') - self.fields[cfield_id] = constraint_field - self.fields[cselector_id] = selector_field - self._wg_field_data.append((constraintname, cfield_id, cselector_id)) - - # Show constraints that are not actually used by the meeting so these don't get lost - self._inactive_wg_field_data = [] - inactive_cnames = ConstraintName.objects.filter( - is_group_conflict=True # Only collect group conflicts... - ).exclude( - meeting=meeting # ...that are not enabled for this meeting... - ).filter( - constraint__source=group, # ...but exist for this group... - constraint__meeting=meeting, # ... at this meeting. - ).distinct() - - for inactive_constraint_name in inactive_cnames: - field_id = 'delete_{}'.format(inactive_constraint_name.slug) - self.fields[field_id] = forms.BooleanField(required=False, label='Delete this conflict', help_text='Delete this inactive conflict?') - constraints = group.constraint_source_set.filter(meeting=meeting, name=inactive_constraint_name) - self._inactive_wg_field_data.append( - (inactive_constraint_name, - ' '.join([c.target.acronym for c in constraints]), - field_id) - ) - - self.fields['joint_with_groups_selector'].widget.attrs['onchange'] = "document.form_post.joint_with_groups.value=document.form_post.joint_with_groups.value + ' ' + this.options[this.selectedIndex].value; return 1;" - self.fields["resources"].choices = [(x.pk,x.desc) for x in ResourceAssociation.objects.filter(name__used=True).order_by('name__order') ] - - if self.hidden: - # replace all the widgets to start... - for key in list(self.fields.keys()): - self.fields[key].widget = forms.HiddenInput() - # re-replace a couple special cases - self.fields['resources'].widget = forms.MultipleHiddenInput() - self.fields['timeranges'].widget = forms.MultipleHiddenInput() - # and entirely replace bethere - no need to support searching if input is hidden - self.fields['bethere'] = ModelMultipleChoiceField( - widget=forms.MultipleHiddenInput, required=False, - queryset=Person.objects.all(), - ) - - def wg_constraint_fields(self): - """Iterates over wg constraint fields - - Intended for use in the template. - """ - for cname, cfield_id, cselector_id in self._wg_field_data: - yield cname, self[cfield_id], self[cselector_id] - - def wg_constraint_count(self): - """How many wg constraints are there?""" - return len(self._wg_field_data) - - def wg_constraint_field_ids(self): - """Iterates over wg constraint field IDs""" - for cname, cfield_id, _ in self._wg_field_data: - yield cname, cfield_id - - def inactive_wg_constraints(self): - for cname, value, field_id in self._inactive_wg_field_data: - yield cname, value, self[field_id] - - def inactive_wg_constraint_count(self): - return len(self._inactive_wg_field_data) - - def inactive_wg_constraint_field_ids(self): - """Iterates over wg constraint field IDs""" - for cname, _, field_id in self._inactive_wg_field_data: - yield cname, field_id - - @staticmethod - def _add_widget_class(widget, new_class): - """Add a new class, taking care in case some already exist""" - existing_classes = widget.attrs.get('class', '').split() - widget.attrs['class'] = ' '.join(existing_classes + [new_class]) - - def _join_conflicts(self, cleaned_data, slugs): - """Concatenate constraint fields from cleaned data into a single list""" - conflicts = [] - for cname, cfield_id, _ in self._wg_field_data: - if cname.slug in slugs and cfield_id in cleaned_data: - groups = cleaned_data[cfield_id] - # convert to python list (allow space or comma separated lists) - items = groups.replace(',',' ').split() - conflicts.extend(items) - return conflicts - - def _validate_duplicate_conflicts(self, cleaned_data): - """Validate that no WGs appear in more than one constraint that does not allow duplicates - - Raises ValidationError - """ - # Only the older constraints (conflict, conflic2, conflic3) need to be mutually exclusive. - all_conflicts = self._join_conflicts(cleaned_data, ['conflict', 'conflic2', 'conflic3']) - seen = [] - duplicated = [] - errors = [] - for c in all_conflicts: - if c not in seen: - seen.append(c) - elif c not in duplicated: # only report once - duplicated.append(c) - errors.append(forms.ValidationError('%s appears in conflicts more than once' % c)) - return errors - - def clean_joint_with_groups(self): - groups = self.cleaned_data['joint_with_groups'] - check_conflict(groups, self.group) - return groups - - def clean_comments(self): - return clean_text_field(self.cleaned_data['comments']) - - def clean_bethere(self): - bethere = self.cleaned_data["bethere"] - if bethere: - extra = set( - Person.objects.filter( - role__group=self.group, role__name__in=["chair", "ad"] - ) - & bethere - ) - if extra: - extras = ", ".join(e.name for e in extra) - raise forms.ValidationError( - ( - f"Please remove the following person{pluralize(len(extra))}, the system " - f"tracks their availability due to their role{pluralize(len(extra))}: {extras}." - ) - ) - return bethere - - def clean_send_notifications(self): - return True if not self.notifications_optional else self.cleaned_data['send_notifications'] - - def is_valid(self): - return super().is_valid() and self.session_forms.is_valid() - - def clean(self): - super(SessionForm, self).clean() - self.session_forms.clean() - - data = self.cleaned_data - - # Validate the individual conflict fields - for _, cfield_id, _ in self._wg_field_data: - try: - check_conflict(data[cfield_id], self.group) - except forms.ValidationError as e: - self.add_error(cfield_id, e) - - # Skip remaining tests if individual field tests had errors, - if self.errors: - return data - - # error if conflicts contain disallowed dupes - for error in self._validate_duplicate_conflicts(data): - self.add_error(None, error) - - # Verify expected number of session entries are present - num_sessions_with_data = len(self.session_forms.forms_to_keep) - num_sessions_expected = -1 - try: - num_sessions_expected = int(data.get('num_session', '')) - except ValueError: - self.add_error('num_session', 'Invalid value for number of sessions') - if num_sessions_with_data < num_sessions_expected: - self.add_error('num_session', 'Must provide data for all sessions') - - # if default (empty) option is selected, cleaned_data won't include num_session key - if num_sessions_expected != 2 and num_sessions_expected is not None: - if data.get('session_time_relation'): - self.add_error( - 'session_time_relation', - forms.ValidationError('Time between sessions can only be used when two sessions are requested.') - ) - - joint_session = data.get('joint_for_session', '') - if joint_session != '': - joint_session = int(joint_session) - if joint_session > num_sessions_with_data: - self.add_error( - 'joint_for_session', - forms.ValidationError( - f'Session {joint_session} can not be the joint session, the session has not been requested.' - ) - ) - - return data - - @property - def media(self): - # get media for our formset - return super().media + self.session_forms.media + forms.Media(js=('secr/js/session_form.js',)) - - -# Used for totally virtual meetings during COVID-19 to omit the expected -# number of attendees since there were no room size limitations -# -# class VirtualSessionForm(SessionForm): -# '''A SessionForm customized for special virtual meeting requirements''' -# attendees = forms.IntegerField(required=False) - - -class ToolStatusForm(forms.Form): - message = forms.CharField(widget=forms.Textarea(attrs={'rows':'3','cols':'80'}), strip=False) - diff --git a/ietf/secr/sreq/templatetags/__init__.py b/ietf/secr/sreq/templatetags/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/ietf/secr/sreq/urls.py b/ietf/secr/sreq/urls.py deleted file mode 100644 index 7e0db8117a0..00000000000 --- a/ietf/secr/sreq/urls.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright The IETF Trust 2007-2019, All Rights Reserved - -from django.conf import settings - -from ietf.secr.sreq import views -from ietf.utils.urls import url - -urlpatterns = [ - url(r'^$', views.main), - url(r'^status/$', views.tool_status), - url(r'^%(acronym)s/$' % settings.URL_REGEXPS, views.view), - url(r'^(?P[A-Za-z0-9_\-\+]+)/%(acronym)s/view/$' % settings.URL_REGEXPS, views.view), - url(r'^%(acronym)s/approve/$' % settings.URL_REGEXPS, views.approve), - url(r'^%(acronym)s/cancel/$' % settings.URL_REGEXPS, views.cancel), - url(r'^%(acronym)s/confirm/$' % settings.URL_REGEXPS, views.confirm), - url(r'^%(acronym)s/edit/$' % settings.URL_REGEXPS, views.edit), - url(r'^%(acronym)s/new/$' % settings.URL_REGEXPS, views.new), - url(r'^%(acronym)s/no_session/$' % settings.URL_REGEXPS, views.no_session), - url(r'^(?P[A-Za-z0-9_\-\+]+)/%(acronym)s/edit/$' % settings.URL_REGEXPS, views.edit), -] diff --git a/ietf/secr/telechat/tests.py b/ietf/secr/telechat/tests.py index 39949b83a2b..fa26d33a5c5 100644 --- a/ietf/secr/telechat/tests.py +++ b/ietf/secr/telechat/tests.py @@ -13,6 +13,7 @@ IndividualDraftFactory, ConflictReviewFactory) from ietf.doc.models import BallotDocEvent, BallotType, BallotPositionDocEvent, State, Document from ietf.doc.utils import update_telechat, create_ballot_if_not_open +from ietf.meeting.factories import MeetingFactory from ietf.utils.test_utils import TestCase from ietf.utils.timezone import date_today, datetime_today from ietf.iesg.models import TelechatDate @@ -25,6 +26,26 @@ def augment_data(): TelechatDate.objects.create(date=date_today()) +class SecrUrlTests(TestCase): + def test_urls(self): + MeetingFactory(type_id='ietf', date=date_today()) + + # check public options + response = self.client.get("/secr/") + self.assertEqual(response.status_code, 200) + q = PyQuery(response.content) + links = q('div.secr-menu a') + self.assertEqual(len(links), 1) + self.assertEqual(PyQuery(links[0]).text(), 'Announcements') + + # check secretariat only options + self.client.login(username="secretary", password="secretary+password") + response = self.client.get("/secr/") + self.assertEqual(response.status_code, 200) + q = PyQuery(response.content) + links = q('div.secr-menu a') + self.assertEqual(len(links), 4) + class SecrTelechatTestCase(TestCase): def test_main(self): "Main Test" diff --git a/ietf/secr/templates/includes/activities.html b/ietf/secr/templates/includes/activities.html deleted file mode 100644 index 1304b7c48db..00000000000 --- a/ietf/secr/templates/includes/activities.html +++ /dev/null @@ -1,23 +0,0 @@ -

    Activities Log

    -
    - - - - - - - - - - - {% for entry in activities %} - - - - - - - {% endfor %} - -
    DateTimeActionName
    {{ entry.act_date }}{{ entry.act_time }}{{ entry.activity }}{{ entry.act_by }}
    -
    diff --git a/ietf/secr/templates/includes/buttons_next_cancel.html b/ietf/secr/templates/includes/buttons_next_cancel.html deleted file mode 100644 index 95d25f55bcc..00000000000 --- a/ietf/secr/templates/includes/buttons_next_cancel.html +++ /dev/null @@ -1,6 +0,0 @@ -
    -
      -
      • -
      • -
    -
    diff --git a/ietf/secr/templates/includes/buttons_submit_cancel.html b/ietf/secr/templates/includes/buttons_submit_cancel.html deleted file mode 100644 index df40c982559..00000000000 --- a/ietf/secr/templates/includes/buttons_submit_cancel.html +++ /dev/null @@ -1,6 +0,0 @@ -
    -
      -
      • -
      • -
    -
    diff --git a/ietf/secr/templates/includes/sessions_footer.html b/ietf/secr/templates/includes/sessions_footer.html deleted file mode 100755 index 2a26440047f..00000000000 --- a/ietf/secr/templates/includes/sessions_footer.html +++ /dev/null @@ -1,5 +0,0 @@ - \ No newline at end of file diff --git a/ietf/secr/templates/includes/sessions_request_form.html b/ietf/secr/templates/includes/sessions_request_form.html deleted file mode 100755 index 61b16733572..00000000000 --- a/ietf/secr/templates/includes/sessions_request_form.html +++ /dev/null @@ -1,130 +0,0 @@ -* Required Field -{% csrf_token %} - {{ form.session_forms.management_form }} - {% if form.non_field_errors %} - {{ form.non_field_errors }} - {% endif %} - - - - - - {% if group.features.acts_like_wg %} - - {% if not is_virtual %} - - {% endif %} - - {% else %}{# else not group.features.acts_like_wg #} - {% for session_form in form.session_forms %} - - {% endfor %} - {% endif %} - - - - - - - - - - {% if not is_virtual %} - - - - - - - - - - - - - - - - - - - {% endif %} - - - - - - {% if form.notifications_optional %} - - - - - {% endif %} - -
    Working Group Name:{{ group.name }} ({{ group.acronym }})
    Area Name:{% if group.parent %}{{ group.parent.name }} ({{ group.parent.acronym }}){% endif %}
    Number of Sessions:*{{ form.num_session.errors }}{{ form.num_session }}
    Session 1:*{% include 'meeting/session_details_form.html' with form=form.session_forms.0 hide_onsite_tool_prompt=True only %}
    Session 2:*{% include 'meeting/session_details_form.html' with form=form.session_forms.1 hide_onsite_tool_prompt=True only %}
    Time between two sessions:{{ form.session_time_relation.errors }}{{ form.session_time_relation }}
    Additional Session Request:{{ form.third_session }} Check this box to request an additional session.
    - Additional slot may be available after agenda scheduling has closed and with the approval of an Area Director.
    -
    - Third Session: - {% include 'meeting/session_details_form.html' with form=form.session_forms.2 hide_onsite_tool_prompt=True only %} -
    -
    Session {{ forloop.counter }}:*{% include 'meeting/session_details_form.html' with form=session_form only %}
    Number of Attendees:{% if not is_virtual %}*{% endif %}{{ form.attendees.errors }}{{ form.attendees }}
    Participants who must be present: - {{ form.bethere.errors }} - {{ form.bethere }} -

    - Do not include Area Directors and WG Chairs; the system already tracks their availability. -

    -
    Conflicts to Avoid: - - - - - - - {% for cname, cfield, cselector in form.wg_constraint_fields %} - - {% if forloop.first %}{% endif %} - - - - {% empty %}{# shown if there are no constraint fields #} - - {% endfor %} - {% if form.inactive_wg_constraints %} - {% for cname, value, field in form.inactive_wg_constraints %} - - {% if forloop.first %} - - {% endif %} - - - - {% endfor %} - {% endif %} - - - - - -
    Other WGs that included {{ group.name }} in their conflict lists:{{ session_conflicts.inbound|default:"None" }}
    WG Sessions:
    You may select multiple WGs within each category    		{{ cname|title }}{{ cselector }} -
    - {{ cfield.errors }}{{ cfield }} -
    No constraints are enabled for this meeting.
    - Disabled for this meeting - {{ cname|title }}
    {{ field }} {{ field.label }}
    BOF Sessions:If the sessions can not be found in the fields above, please enter free form requests in the Special Requests field below.
    -
    Resources requested: - {{ form.resources.errors }} {{ form.resources }} -
    Times during which this WG can not meet:
    Please explain any selections in Special Requests below.    		{{ form.timeranges.errors }}{{ form.timeranges }}
    - Plan session adjacent with another WG:
    - (Immediately before or after another WG, no break in between, in the same room.) -     		{{ form.adjacent_with_wg.errors }}{{ form.adjacent_with_wg }}
    - Joint session with:
    - (To request one session for multiple WGs together.) -     		To request a joint session with another group, please contact the secretariat.
    Special Requests:
     
    i.e. restrictions on meeting times / days, etc.
    (limit 200 characters)    		{{ form.comments.errors }}{{ form.comments }}
    {{ form.send_notifications.label }}{{ form.send_notifications.errors }}{{ form.send_notifications }}
    - -
    -
      -
      • -
      • -
    -
    - \ No newline at end of file diff --git a/ietf/secr/templates/includes/sessions_request_view.html b/ietf/secr/templates/includes/sessions_request_view.html deleted file mode 100644 index bc6aef06116..00000000000 --- a/ietf/secr/templates/includes/sessions_request_view.html +++ /dev/null @@ -1,73 +0,0 @@ -{% load ams_filters %} - - - - - - {% if form %} - {% include 'includes/sessions_request_view_formset.html' with formset=form.session_forms group=group session=session only %} - {% else %} - {% include 'includes/sessions_request_view_session_set.html' with session_set=sessions group=group session=session only %} - {% endif %} - - - - - - - - - - {% if not is_virtual %} - - - - - {% endif %} - - - - - - - - - {% if not is_virtual %} - - - - - - - - - {% endif %} - - {% if form and form.notifications_optional %} - - - - - {% endif %} - -
    Working Group Name:{{ group.name }} ({{ group.acronym }})
    Area Name:{{ group.parent }}
    Number of Sessions Requested:{% if session.third_session %}3{% else %}{{ session.num_session }}{% endif %}
    Number of Attendees:{{ session.attendees }}
    Conflicts to Avoid: - {% if session_conflicts.outbound %} - - - {% for conflict in session_conflicts.outbound %} - - {% endfor %} - -
    {{ conflict.name|title }}: {{ conflict.groups }}
    - {% else %}None{% endif %} -
    Other WGs that included {{ group }} in their conflict list:{% if session_conflicts.inbound %}{{ session_conflicts.inbound }}{% else %}None so far{% endif %}
    Resources requested:{% if session.resources %}
      {% for resource in session.resources %}
    • {{ resource.desc }}
      • {% endfor %}
    {% else %}None so far{% endif %}
    Participants who must be present:{% if session.bethere %}
      {% for person in session.bethere %}
    • {{ person }}
      • {% endfor %}
    {% else %}None{% endif %}
    Can not meet on:{% if session.timeranges_display %}{{ session.timeranges_display|join:', ' }}{% else %}No constraints{% endif %}
    Adjacent with WG:{{ session.adjacent_with_wg|default:'No preference' }}
    Joint session: - {% if session.joint_with_groups %} - {{ session.joint_for_session_display }} with: {{ session.joint_with_groups }} - {% else %} - Not a joint session - {% endif %} -
    Special Requests:{{ session.comments }}
    - {{ form.send_notifications.label}} - - {% if form.cleaned_data.send_notifications %}Yes{% else %}No{% endif %} -
    \ No newline at end of file diff --git a/ietf/secr/templates/includes/sessions_request_view_formset.html b/ietf/secr/templates/includes/sessions_request_view_formset.html deleted file mode 100644 index 80cad8d829e..00000000000 --- a/ietf/secr/templates/includes/sessions_request_view_formset.html +++ /dev/null @@ -1,32 +0,0 @@ -{% load ams_filters %}{# keep this in sync with sessions_request_view_session_set.html #} -{% for sess_form in formset %}{% if sess_form.cleaned_data and not sess_form.cleaned_data.DELETE %} - - Session {{ forloop.counter }}: - -
    -
    Length
    -
    {{ sess_form.cleaned_data.requested_duration.total_seconds|display_duration }}
    - {% if sess_form.cleaned_data.name %} -
    Name
    -
    {{ sess_form.cleaned_data.name }}
    {% endif %} - {% if sess_form.cleaned_data.purpose.slug != 'regular' %} -
    Purpose
    -
    - {{ sess_form.cleaned_data.purpose }} - {% if sess_form.cleaned_data.purpose.timeslot_types|length > 1 %}({{ sess_form.cleaned_data.type }} - ){% endif %} -
    -
    Onsite tool?
    -
    {{ sess_form.cleaned_data.has_onsite_tool|yesno }}
    - {% endif %} -
    - - - {% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %} - - Time between sessions: - {% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No - preference{% endif %} - - {% endif %} -{% endif %}{% endfor %} \ No newline at end of file diff --git a/ietf/secr/templates/includes/sessions_request_view_session_set.html b/ietf/secr/templates/includes/sessions_request_view_session_set.html deleted file mode 100644 index a434b9d22b5..00000000000 --- a/ietf/secr/templates/includes/sessions_request_view_session_set.html +++ /dev/null @@ -1,32 +0,0 @@ -{% load ams_filters %}{# keep this in sync with sessions_request_view_formset.html #} -{% for sess in session_set %} - - Session {{ forloop.counter }}: - -
    -
    Length
    -
    {{ sess.requested_duration.total_seconds|display_duration }}
    - {% if sess.name %} -
    Name
    -
    {{ sess.name }}
    {% endif %} - {% if sess.purpose.slug != 'regular' %} -
    Purpose
    -
    - {{ sess.purpose }} - {% if sess.purpose.timeslot_types|length > 1 %}({{ sess.type }} - ){% endif %} -
    -
    Onsite tool?
    -
    {{ sess.has_onsite_tool|yesno }}
    - {% endif %} -
    - - - {% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %} - - Time between sessions: - {% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No - preference{% endif %} - - {% endif %} -{% endfor %} \ No newline at end of file diff --git a/ietf/secr/templates/index.html b/ietf/secr/templates/index.html index 05fa3db41f0..9ea70212793 100644 --- a/ietf/secr/templates/index.html +++ b/ietf/secr/templates/index.html @@ -1,11 +1,11 @@ -{# Copyright The IETF Trust 2007, All Rights Reserved #} +{# Copyright The IETF Trust 2007-2025, All Rights Reserved #} {% extends "base.html" %} {% load static %} {% load ietf_filters %} {% block title %}Secretariat Dashboard{% endblock %} {% block content %}

    Secretariat Dashboard

    -
    +
    {% if user|has_role:"Secretariat" %}

    IESG

      @@ -20,12 +20,10 @@

      IDs and WGs Process

      Meetings and Proceedings

      {% else %} {% endif %} diff --git a/ietf/secr/templates/sreq/confirm.html b/ietf/secr/templates/sreq/confirm.html deleted file mode 100755 index 025375af32d..00000000000 --- a/ietf/secr/templates/sreq/confirm.html +++ /dev/null @@ -1,57 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions - Confirm{% endblock %} - -{% block extrastyle %} - -{% endblock %} - -{% block extrahead %}{{ block.super }} - - {{ form.media }} -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » New - » Session Request Confirmation -{% endblock %} - -{% block content %} - -
      -

      Sessions - Confirm

      - - {% include "includes/sessions_request_view.html" %} - - {% if group.features.acts_like_wg and form.session_forms.forms_to_keep|length > 2 %} -
      -

      - - Note: Your request for a third session must be approved by an area director before - being submitted to agenda@ietf.org. Click "Submit" below to email an approval - request to the area directors. - -

      -
      - {% endif %} - -
      - {% csrf_token %} - {{ form }} - {{ form.session_forms.management_form }} - {% for sf in form.session_forms %} - {% include 'meeting/session_details_form.html' with form=sf hidden=True only %} - {% endfor %} - {% include "includes/buttons_submit_cancel.html" %} -
      - -
      - -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/edit.html b/ietf/secr/templates/sreq/edit.html deleted file mode 100755 index f6e62104b0d..00000000000 --- a/ietf/secr/templates/sreq/edit.html +++ /dev/null @@ -1,39 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} -{% block title %}Sessions - Edit{% endblock %} - -{% block extrahead %}{{ block.super }} - - - {{ form.media }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » {{ group.acronym }} - » Edit -{% endblock %} - -{% block instructions %} - Instructions -{% endblock %} - -{% block content %} -
      -

      IETF {{ meeting.number }}: Edit Session Request

      -
      -
      - - {% include "includes/sessions_request_form.html" %} - -
      -
      -{% endblock %} - -{% block footer-extras %} - {% include "includes/sessions_footer.html" %} -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/locked.html b/ietf/secr/templates/sreq/locked.html deleted file mode 100755 index c27cf578ed2..00000000000 --- a/ietf/secr/templates/sreq/locked.html +++ /dev/null @@ -1,30 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions (Locked) -{% endblock %} - -{% block content %} -

      » View list of timeslot requests

      -
      -

      Sessions - Status

      - -

      {{ message }}

      - -
      -
        -
        • -
      -
      - - -
      - -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/main.html b/ietf/secr/templates/sreq/main.html deleted file mode 100755 index a6695cd4f3c..00000000000 --- a/ietf/secr/templates/sreq/main.html +++ /dev/null @@ -1,65 +0,0 @@ -{% extends "base_site.html" %} -{% load ietf_filters %} -{% load static %} - -{% block title %}Sessions{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions -{% endblock %} -{% block instructions %} - Instructions -{% endblock %} - -{% block content %} -

      » View list of timeslot requests

      -
      -

      - Sessions Request Tool: IETF {{ meeting.number }} - {% if user|has_role:"Secretariat" %} - {% if is_locked %} - Tool Status: Locked - {% else %} - Tool Status: Unlocked - {% endif %} - {% endif %} -

      -
      -

      Request New Session

      -

      The list below includes those working groups that you currently chair which do not already have a session scheduled. You can click on an acronym to initiate a request for a new session at the upcoming IETF meeting. Click "Group will not meet" to send a notification that the group does not plan to meet.

      -
        - {% for group in unscheduled_groups %} -
      • - {{ group.acronym }} - {% if group.not_meeting %} - (Currently, this group does not plan to hold a session at IETF {{ meeting.number }}) - {% endif %} -
        • - {% empty %} -
      • NONE
        • - {% endfor %} -
      -
      -
      -
      -

      Edit / Cancel Previously Requested Sessions

      -

      The list below includes those working groups for which you or your co-chair has requested sessions at the upcoming IETF meeting. You can click on an acronym to initiate changes to a session, or cancel a session.

      - -
      -
      - -{% endblock %} - -{% block footer-extras %} - {% include "includes/sessions_footer.html" %} -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/new.html b/ietf/secr/templates/sreq/new.html deleted file mode 100755 index 3f46e6f897c..00000000000 --- a/ietf/secr/templates/sreq/new.html +++ /dev/null @@ -1,43 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions- New{% endblock %} - -{% block extrahead %}{{ block.super }} - - - {{ form.media }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » New Session Request -{% endblock %} - -{% block instructions %} - Instructions -{% endblock %} - -{% block content %} -
      -

      IETF {{ meeting.number }}: New Session Request

      -
      -
      -
        -
        • -
        • -
      -
      - {% include "includes/sessions_request_form.html" %} - -
      - -{% endblock %} - -{% block footer-extras %} - {% include "includes/sessions_footer.html" %} -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/tool_status.html b/ietf/secr/templates/sreq/tool_status.html deleted file mode 100755 index b91e73a129e..00000000000 --- a/ietf/secr/templates/sreq/tool_status.html +++ /dev/null @@ -1,42 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » Session Status -{% endblock %} - -{% block content %} - -
      -

      Sessions - Status

      -

      Enter the message that you would like displayed to the WG Chair when this tool is locked.

      -
      {% csrf_token %} - - - - {{ form.as_table }} - -
      -
      -
        - {% if is_locked %} -
        • - {% else %} -
        • - {% endif %} -
        • -
      -
      - -
      - -
      - -{% endblock %} diff --git a/ietf/secr/templates/sreq/view.html b/ietf/secr/templates/sreq/view.html deleted file mode 100644 index 9a0a3b01c16..00000000000 --- a/ietf/secr/templates/sreq/view.html +++ /dev/null @@ -1,55 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions - View{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block extrastyle %} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » {{ group.acronym }} -{% endblock %} - -{% block instructions %} - Instructions -{% endblock %} - -{% block content %} - -
      -

      Sessions - View (meeting: {{ meeting.number }})

      - - {% include "includes/sessions_request_view.html" %} - -
      - - {% include "includes/activities.html" %} - -
      -
        -
        • - {% if show_approve_button %} -
        • - {% endif %} -
        • -
        • -
      -
      -
      - -{% endblock %} - -{% block footer-extras %} - {% include "includes/sessions_footer.html" %} -{% endblock %} diff --git a/ietf/secr/urls.py b/ietf/secr/urls.py index 4a3e5b0363f..ab210466544 100644 --- a/ietf/secr/urls.py +++ b/ietf/secr/urls.py @@ -1,11 +1,22 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.conf import settings from django.urls import re_path, include from django.views.generic import TemplateView +from django.views.generic.base import RedirectView urlpatterns = [ re_path(r'^$', TemplateView.as_view(template_name='index.html'), name='ietf.secr'), re_path(r'^announcement/', include('ietf.secr.announcement.urls')), re_path(r'^meetings/', include('ietf.secr.meetings.urls')), re_path(r'^rolodex/', include('ietf.secr.rolodex.urls')), - re_path(r'^sreq/', include('ietf.secr.sreq.urls')), + # remove these redirects after 125 + re_path(r'^sreq/$', RedirectView.as_view(url='/meeting/session/request/', permanent=True)), + re_path(r'^sreq/%(acronym)s/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/session/request/%(acronym)s/view/', permanent=True)), + re_path(r'^sreq/%(acronym)s/edit/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/session/request/%(acronym)s/edit/', permanent=True)), + re_path(r'^sreq/%(acronym)s/new/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/session/request/%(acronym)s/new/', permanent=True)), + re_path(r'^sreq/(?P[A-Za-z0-9_\-\+]+)/%(acronym)s/view/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/%(num)s/session/request/%(acronym)s/view/', permanent=True)), + re_path(r'^sreq/(?P[A-Za-z0-9_\-\+]+)/%(acronym)s/edit/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/%(num)s/session/request/%(acronym)s/edit/', permanent=True)), + # --------------------------------- re_path(r'^telechat/', include('ietf.secr.telechat.urls')), ] diff --git a/ietf/secr/utils/group.py b/ietf/secr/utils/group.py deleted file mode 100644 index 40a9065ace2..00000000000 --- a/ietf/secr/utils/group.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright The IETF Trust 2013-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -# Python imports - -# Django imports -from django.core.exceptions import ObjectDoesNotExist - -# Datatracker imports -from ietf.group.models import Group -from ietf.ietfauth.utils import has_role - - -def get_my_groups(user,conclude=False): - ''' - Takes a Django user object (from request) - Returns a list of groups the user has access to. Rules are as follows - secretariat - has access to all groups - area director - has access to all groups in their area - wg chair or secretary - has access to their own group - chair of irtf has access to all irtf groups - - If user=None than all groups are returned. - concluded=True means include concluded groups. Need this to upload materials for groups - after they've been concluded. it happens. - ''' - my_groups = set() - states = ['bof','proposed','active'] - if conclude: - states.extend(['conclude','bof-conc']) - - all_groups = Group.objects.filter(type__features__has_meetings=True, state__in=states).order_by('acronym') - if user == None or has_role(user,'Secretariat'): - return all_groups - - try: - person = user.person - except ObjectDoesNotExist: - return list() - - for group in all_groups: - if group.role_set.filter(person=person,name__in=('chair','secr','ad')): - my_groups.add(group) - continue - if group.parent and group.parent.role_set.filter(person=person,name__in=('ad','chair')): - my_groups.add(group) - continue - - return list(my_groups) diff --git a/ietf/settings.py b/ietf/settings.py index d6be1d1e0f5..9a213c1a733 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -537,7 +537,6 @@ def skip_unreadable_post(record): 'ietf.secr.announcement', 'ietf.secr.meetings', 'ietf.secr.rolodex', - 'ietf.secr.sreq', 'ietf.secr.telechat', ] diff --git a/ietf/static/js/custom_striped.js b/ietf/static/js/custom_striped.js new file mode 100644 index 00000000000..480ad7cf828 --- /dev/null +++ b/ietf/static/js/custom_striped.js @@ -0,0 +1,16 @@ +// Copyright The IETF Trust 2025, All Rights Reserved + +document.addEventListener('DOMContentLoaded', () => { + // add stripes + const firstRow = document.querySelector('.custom-stripe .row') + if (firstRow) { + const parent = firstRow.parentElement; + const allRows = Array.from(parent.children).filter(child => child.classList.contains('row')) + allRows.forEach((row, index) => { + row.classList.remove('bg-light') + if (index % 2 === 1) { + row.classList.add('bg-light') + } + }) + } +}) diff --git a/ietf/secr/static/js/session_form.js b/ietf/static/js/session_form.js similarity index 92% rename from ietf/secr/static/js/session_form.js rename to ietf/static/js/session_form.js index 6f28f16db41..bd61293d7c5 100644 --- a/ietf/secr/static/js/session_form.js +++ b/ietf/static/js/session_form.js @@ -1,4 +1,4 @@ -/* Copyright The IETF Trust 2021, All Rights Reserved +/* Copyright The IETF Trust 2021-2025, All Rights Reserved * * JS support for the SessionForm * */ diff --git a/ietf/secr/static/js/sessions.js b/ietf/static/js/session_request.js similarity index 90% rename from ietf/secr/static/js/sessions.js rename to ietf/static/js/session_request.js index a2770e62622..dfb169f6752 100644 --- a/ietf/secr/static/js/sessions.js +++ b/ietf/static/js/session_request.js @@ -1,4 +1,4 @@ -// Copyright The IETF Trust 2015-2021, All Rights Reserved +// Copyright The IETF Trust 2015-2025, All Rights Reserved /* global alert */ var ietf_sessions; // public interface @@ -38,7 +38,7 @@ var ietf_sessions; // public interface const only_one_session = (val === 1); if (document.form_post.session_time_relation) { document.form_post.session_time_relation.disabled = only_one_session; - document.form_post.session_time_relation.closest('tr').hidden = only_one_session; + document.form_post.session_time_relation.closest('div.row').hidden = only_one_session; } if (document.form_post.joint_for_session) { document.form_post.joint_for_session.disabled = only_one_session; @@ -129,6 +129,11 @@ var ietf_sessions; // public interface } } + function wg_constraint_delete_clicked(event) { + const constraint_name = event.currentTarget.dataset.constraint_name; + delete_last_wg_constraint(constraint_name); + } + /* Initialization */ function on_load() { // Attach event handler to session count select @@ -146,6 +151,9 @@ var ietf_sessions; // public interface selectors[index].addEventListener('change', wg_constraint_selector_changed, false) } + // Attach event handler to constraint delete buttons + document.querySelectorAll('.wg_constraint_delete') + .forEach(btn => btn.addEventListener('click', wg_constraint_delete_clicked)); } // initialize after page loads diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html index bd8c0bf3cd3..1e7c1688ff7 100644 --- a/ietf/templates/base/menu.html +++ b/ietf/templates/base/menu.html @@ -1,4 +1,4 @@ -{# Copyright The IETF Trust 2015-2022, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} {% load origin %} {% origin %} {% load ietf_filters managed_groups wg_menu active_groups_menu group_filters cache meetings_filters %} @@ -304,7 +304,7 @@
    • + href="{% url 'ietf.meeting.views_session_request.list_view' %}"> Request a session
      • diff --git a/ietf/templates/group/meetings-row.html b/ietf/templates/group/meetings-row.html index 25605ec0f12..8927eb61a26 100644 --- a/ietf/templates/group/meetings-row.html +++ b/ietf/templates/group/meetings-row.html @@ -1,3 +1,4 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} {% load origin tz %} {% origin %} {% for s in sessions %} @@ -25,7 +26,7 @@ {% if show_request and s.meeting.type_id == 'ietf' %} {% if can_edit %} + href="{% url 'ietf.meeting.views_session_request.view_request' num=s.meeting.number acronym=s.group.acronym %}"> Edit Session Request {% endif %} diff --git a/ietf/templates/group/meetings.html b/ietf/templates/group/meetings.html index bee81110257..30f478da131 100644 --- a/ietf/templates/group/meetings.html +++ b/ietf/templates/group/meetings.html @@ -1,3 +1,4 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} {% extends "group/group_base.html" %} {% load origin static %} {% block title %} @@ -9,7 +10,7 @@ Session requests {% if can_edit or can_always_edit %} - Request a session + Request a session Request an interim meeting diff --git a/ietf/templates/meeting/important_dates_for_meeting.ics b/ietf/templates/meeting/important_dates_for_meeting.ics index df5fe468189..e6d403da932 100644 --- a/ietf/templates/meeting/important_dates_for_meeting.ics +++ b/ietf/templates/meeting/important_dates_for_meeting.ics @@ -1,3 +1,4 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} {% load tz ietf_filters %}{% for d in meeting.important_dates %}BEGIN:VEVENT UID:ietf-{{ meeting.number }}-{{ d.name_id }}-{{ d.date.isoformat }} SUMMARY:IETF {{ meeting.number }}: {{ d.name.name }} @@ -8,11 +9,11 @@ TRANSP:TRANSPARENT DESCRIPTION:{{ d.name.desc }}{% if first and d.name.slug == 'openreg' or first and d.name.slug == 'earlybird' %}\n Register here: https://www.ietf.org/how/meetings/register/{% endif %}{% if d.name.slug == 'opensched' %}\n To request a Working Group session, use the IETF Meeting Session Request Tool:\n - {{ request.scheme }}://{{ request.get_host}}{% url 'ietf.secr.sreq.views.main' %}\n + {{ request.scheme }}://{{ request.get_host}}{% url 'ietf.meeting.views_session_request.list_view' %}\n If you are working on a BOF request, it is highly recommended to tell the IESG\n now by sending an email to iesg@ietf.org to get advance help with the request.{% endif %}{% if d.name.slug == 'cutoffwgreq' %}\n To request a Working Group session, use the IETF Meeting Session Request Tool:\n - {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.secr.sreq.views.main' %}{% endif %}{% if d.name.slug == 'cutoffbofreq' %}\n + {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.meeting.views_session_request.list_view' %}{% endif %}{% if d.name.slug == 'cutoffbofreq' %}\n To request a BOF, please see instructions on Requesting a BOF:\n https://www.ietf.org/how/bofs/bof-procedures/{% endif %}{% if d.name.slug == 'idcutoff' %}\n Upload using the I-D Submission Tool:\n diff --git a/ietf/templates/meeting/requests.html b/ietf/templates/meeting/requests.html index 3008ceb6629..0abee95887b 100644 --- a/ietf/templates/meeting/requests.html +++ b/ietf/templates/meeting/requests.html @@ -1,5 +1,5 @@ {% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} {% load origin %} {% load ietf_filters static person_filters textfilters %} {% block pagehead %} @@ -151,7 +151,7 @@

      {% endifchanged %} - + {{ session.group.acronym }} {% if session.purpose_id != "regular" and session.purpose_id != "none" %} diff --git a/ietf/secr/templates/sreq/session_approval_notification.txt b/ietf/templates/meeting/session_approval_notification.txt similarity index 56% rename from ietf/secr/templates/sreq/session_approval_notification.txt rename to ietf/templates/meeting/session_approval_notification.txt index 7bb63aa3fa0..74eca09bd8b 100644 --- a/ietf/secr/templates/sreq/session_approval_notification.txt +++ b/ietf/templates/meeting/session_approval_notification.txt @@ -1,3 +1,4 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} Dear {{ group.parent }} Director(s): {{ header }} meeting session request has just been @@ -5,11 +6,11 @@ submitted by {{ requester }}. The third session requires your approval. To approve the session go to the session request view here: -{{ settings.IDTRACKER_BASE_URL }}{% url "ietf.secr.sreq.views.view" acronym=group.acronym %} +{{ settings.IDTRACKER_BASE_URL }}{% url "ietf.meeting.views_session_request.view_request" acronym=group.acronym %} and click "Approve Third Session". Regards, The IETF Secretariat. -{% include "includes/session_info.txt" %} +{% include "meeting/session_request_info.txt" %} diff --git a/ietf/secr/templates/sreq/session_cancel_notification.txt b/ietf/templates/meeting/session_cancel_notification.txt similarity index 71% rename from ietf/secr/templates/sreq/session_cancel_notification.txt rename to ietf/templates/meeting/session_cancel_notification.txt index 8aee6c89db7..3de67fc8f48 100644 --- a/ietf/secr/templates/sreq/session_cancel_notification.txt +++ b/ietf/templates/meeting/session_cancel_notification.txt @@ -1,3 +1,4 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} {% autoescape off %}{% load ams_filters %} A request to cancel a meeting session has just been submitted by {{ requester }}.{% endautoescape %} diff --git a/ietf/templates/meeting/session_details_form.html b/ietf/templates/meeting/session_details_form.html index 6b59e7dacd3..9cd1b6e85c9 100644 --- a/ietf/templates/meeting/session_details_form.html +++ b/ietf/templates/meeting/session_details_form.html @@ -1,42 +1,48 @@ -{# Copyright The IETF Trust 2007-2020, All Rights Reserved #} +{# Copyright The IETF Trust 2007-2025, All Rights Reserved #} +{% load django_bootstrap5 %} +
      {% if hidden %} {{ form.name.as_hidden }}{{ form.purpose.as_hidden }}{{ form.type.as_hidden }}{{ form.requested_duration.as_hidden }} {{ form.has_onsite_tool.as_hidden }} {% else %} - - {% comment %} The form-group class is used by session_details_form.js to identify the correct element to hide the name / purpose / type fields when not needed. This is a bootstrap class - the secr app does not use it, so this (and the hidden class, also needed by session_details_form.js) are defined in edit.html and new.html as a kludge to make this work. {% endcomment %} - - - - - - - - - - - - - {% if not hide_onsite_tool_prompt %} - - - - - {% endif %} - -
      {{ form.name.label_tag }}{{ form.name }}{{ form.purpose.errors }}
      {{ form.purpose.label_tag }} - {{ form.purpose }}
      {{ form.type }}
      - {{ form.purpose.errors }}{{ form.type.errors }} -
      {{ form.requested_duration.label_tag }}{{ form.requested_duration }}{{ form.requested_duration.errors }}
      {{ form.has_onsite_tool.label_tag }}{{ form.has_onsite_tool }}{{ form.has_onsite_tool.errors }}
      - {% if hide_onsite_tool_prompt %}{{ form.has_onsite_tool.as_hidden }}{% endif %} + +
      + {% bootstrap_field form.name layout="horizontal" %} +
      + +
      +
      + +
      {{ form.purpose }}
      +
      {{ form.type }}
      + {{ form.purpose.errors }}{{ form.type.errors }} +
      +
      + + {% bootstrap_field form.requested_duration layout="horizontal" %} + {% if not hide_onsite_tool_prompt %} + {% bootstrap_field form.has_onsite_tool layout="horizontal" %} + {% endif %} + + {% if hide_onsite_tool_prompt %} + {{ form.has_onsite_tool.as_hidden }} + {% endif %} {% endif %} + {# hidden fields included whether or not the whole form is hidden #} - {{ form.attendees.as_hidden }}{{ form.comments.as_hidden }}{{ form.id.as_hidden }}{{ form.on_agenda.as_hidden }}{{ form.DELETE.as_hidden }}{{ form.remote_instructions.as_hidden }}{{ form.short.as_hidden }}{{ form.agenda_note.as_hidden }} -
      \ No newline at end of file + {{ form.attendees.as_hidden }} + {{ form.comments.as_hidden }} + {{ form.id.as_hidden }} + {{ form.on_agenda.as_hidden }} + {{ form.DELETE.as_hidden }} + {{ form.remote_instructions.as_hidden }} + {{ form.short.as_hidden }} + {{ form.agenda_note.as_hidden }} +

    diff --git a/ietf/secr/templates/sreq/not_meeting_notification.txt b/ietf/templates/meeting/session_not_meeting_notification.txt similarity index 83% rename from ietf/secr/templates/sreq/not_meeting_notification.txt rename to ietf/templates/meeting/session_not_meeting_notification.txt index 1120f8480c1..0e5c9407089 100644 --- a/ietf/secr/templates/sreq/not_meeting_notification.txt +++ b/ietf/templates/meeting/session_not_meeting_notification.txt @@ -1,3 +1,4 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} {% load ams_filters %} {{ login|smart_login }} {{ group.acronym }} working group, indicated that the {{ group.acronym }} working group does not plan to hold a session at IETF {{ meeting.number }}. diff --git a/ietf/templates/meeting/session_request_confirm.html b/ietf/templates/meeting/session_request_confirm.html new file mode 100644 index 00000000000..09043d3d0cb --- /dev/null +++ b/ietf/templates/meeting/session_request_confirm.html @@ -0,0 +1,38 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static ietf_filters django_bootstrap5 %} +{% block title %}Confirm Session Request{% endblock %} + +{% block content %} +

    Confirm Session Request - IETF {{ meeting.number }}

    + +
    + Instructions +
    + +
    + +
    + + {% include "meeting/session_request_view_table.html" %} + +
    + {% csrf_token %} + {{ form }} + {{ form.session_forms.management_form }} + {% for sf in form.session_forms %} + {% include 'meeting/session_details_form.html' with form=sf hidden=True only %} + {% endfor %} + + + + +
    + +
    + +{% endblock %} + +{% block js %} + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_request_form.html b/ietf/templates/meeting/session_request_form.html new file mode 100644 index 00000000000..ecf5cb7268c --- /dev/null +++ b/ietf/templates/meeting/session_request_form.html @@ -0,0 +1,206 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static ietf_filters django_bootstrap5 %} +{% block title %}{% if is_create %}New {% else %}Edit {% endif %}Session Request{% endblock %} +{% block morecss %}{{ block.super }} + .hidden {display: none !important;} + div.form-group {display: inline;} +{% endblock %} +{% block content %} +

    {% if is_create %}New {% else %}Edit {% endif %}Session Request

    + + {% if is_create %} +
    + Send a notification that the group does not plan to hold a session at IETF {{ meeting.number }} + Retrieve all information from previous meeting +
    + {% endif %} + +
    + +
    + {% csrf_token %} + {{ form.session_forms.management_form }} + {% if form.non_field_errors %} +
    {{ form.non_field_errors }}
    + {% endif %} + +
    + +
    + +
    +
    + +
    + +
    + +
    +
    + + {% bootstrap_field form.num_session layout="horizontal" %} + + {% if group.features.acts_like_wg %} + +
    +
    Session 1
    +
    + {% include 'meeting/session_details_form.html' with form=form.session_forms.0 hide_onsite_tool_prompt=True only %} +
    +
    + +
    +
    Session 2
    +
    + {% include 'meeting/session_details_form.html' with form=form.session_forms.1 hide_onsite_tool_prompt=True only %} +
    +
    + + {% if not is_virtual %} + {% bootstrap_field form.session_time_relation layout="horizontal" %} + {% endif %} + +
    +
    Additional Session Request
    +
    +
    + {{ form.third_session }} + +
    Additional slot may be available after agenda scheduling has closed and with the approval of an Area Director.
    +
    + +
    +
    + +
    +
    Third session request
    +
    + {% include 'meeting/session_details_form.html' with form=form.session_forms.2 hide_onsite_tool_prompt=True only %} +
    +
    + + {% else %}{# else not group.features.acts_like_wg #} + {% for session_form in form.session_forms %} +
    +
    Session {{ forloop.counter }}
    +
    + {% include 'meeting/session_details_form.html' with form=session_form only %} +
    +
    + {% endfor %} + {% endif %} + + {% bootstrap_field form.attendees layout="horizontal" %} + + {% bootstrap_field form.bethere layout="horizontal" %} + +
    +
    Conflicts to avoid
    +
    +
    +
    Other WGs that included {{ group.acronym }} in their conflict lists
    +
    {{ session_conflicts.inbound|default:"None" }}
    +
    +
    +
    WG Sessions
    You may select multiple WGs within each category
    +
    + {% for cname, cfield, cselector in form.wg_constraint_fields %} +
    +
    +
    +
    +
    + {{ cselector }} +
    +
    + +
    +
    +
    +
    + {{ cfield.errors }}{{ cfield }} +
    +
    +
    +
    + {% empty %}{# shown if there are no constraint fields #} +
    +
    No constraints are enabled for this meeting.
    + {% endfor %} +
    +
    + + {% if form.inactive_wg_constraint_count %} +
    +
    Disabled for this meeting
    +
    + {% for cname, value, field in form.inactive_wg_constraints %} +
    +
    {{ cname|title }}
    +
    +
    +
    + +
    +
    + + +
    +
    +
    +
    + {% endfor %} +
    +
    + {% endif %} + +
    +
    BOF Sessions
    +
    If the sessions can not be found in the fields above, please enter free form requests in the Special Requests field below.
    +
    +
    +
    + + {% if not is_virtual %} + + {% bootstrap_field form.resources layout="horizontal" %} + + {% bootstrap_field form.timeranges layout="horizontal" %} + + {% bootstrap_field form.adjacent_with_wg layout="horizontal" %} + +
    +
    Joint session with: (To request one session for multiple WGs together)
    +
    To request a joint session with another group, please contact the secretariat.
    +
    + + {% endif %} + + {% bootstrap_field form.comments layout="horizontal" %} + + {% if form.notifications_optional %} +
    + +
    +
    + + +
    +
    +
    + {% endif %} + + + Cancel +
    + +{% endblock %} +{% block js %} + + {{ form.media }} +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_request_info.txt b/ietf/templates/meeting/session_request_info.txt new file mode 100644 index 00000000000..2e96efb31f1 --- /dev/null +++ b/ietf/templates/meeting/session_request_info.txt @@ -0,0 +1,26 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load ams_filters %} +--------------------------------------------------------- +Working Group Name: {{ group.name }} +Area Name: {{ group.parent }} +Session Requester: {{ login }} +{% if session.joint_with_groups %}{{ session.joint_for_session_display }} joint with: {{ session.joint_with_groups }}{% endif %} + +Number of Sessions: {{ session.num_session }} +Length of Session(s): {% for session_length in session_lengths %}{{ session_length.total_seconds|display_duration }}{% if not forloop.last %}, {% endif %}{% endfor %} +Number of Attendees: {{ session.attendees }} +Conflicts to Avoid: +{% for line in session.outbound_conflicts %} {{line}} +{% endfor %}{% if session.session_time_relation_display %} {{ session.session_time_relation_display }}{% endif %} +{% if session.adjacent_with_wg %} Adjacent with WG: {{ session.adjacent_with_wg }}{% endif %} +{% if session.timeranges_display %} Can't meet: {{ session.timeranges_display|join:", " }}{% endif %} + +Participants who must be present: +{% for person in session.bethere %} {{ person.ascii_name }} +{% endfor %} +Resources Requested: +{% for resource in session.resources %} {{ resource.desc }} +{% endfor %} +Special Requests: + {{ session.comments }} +--------------------------------------------------------- diff --git a/ietf/templates/meeting/session_request_list.html b/ietf/templates/meeting/session_request_list.html new file mode 100644 index 00000000000..789b7006e5d --- /dev/null +++ b/ietf/templates/meeting/session_request_list.html @@ -0,0 +1,65 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load ietf_filters %} +{% load django_bootstrap5 %} +{% block title %}Session Requests{% endblock %} +{% block content %} +

    Session Requests IETF {{ meeting.number }}

    + +
    + Instructions + + View list of timeslot requests + {% if user|has_role:"Secretariat" %} + {% if is_locked %} + Unlock Tool + {% else %} + Lock Tool + {% endif %} + {% endif %} +
    + +
    +
    + Request New Session +
    +
    +

    The list below includes those working groups that you currently chair which do not already have a session scheduled. You can click on an acronym to complete a request for a new session at the upcoming IETF meeting. Click "Group will not meet" to send a notification that the group does not plan to meet.

    +
      + {% for group in unscheduled_groups %} +
    • + {{ group.acronym }} + {% if group.not_meeting %} + (Currently, this group does not plan to hold a session at IETF {{ meeting.number }}) + {% endif %} +
      • + {% empty %} +
    • NONE
      • + {% endfor %} +
    +
    +
    + + +
    +
    + Edit / Cancel Previously Requested Sessions +
    +
    +

    The list below includes those working groups for which you or your co-chair has requested sessions at the upcoming IETF meeting. You can click on an acronym to initiate changes to a session, or cancel a session.

    + +
    +
    + +{% endblock %} + +{% block footer-extras %} + {% include "includes/sessions_footer.html" %} +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_request_locked.html b/ietf/templates/meeting/session_request_locked.html new file mode 100644 index 00000000000..15c023ce332 --- /dev/null +++ b/ietf/templates/meeting/session_request_locked.html @@ -0,0 +1,21 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static ietf_filters django_bootstrap5 %} +{% block title %}Session Request{% endblock %} + +{% block content %} +

    Session Request - IETF {{ meeting.number }}

    + + View list of timeslot requests + +
    + +
    +

    {{ message }}

    + +
    + +
    +
    + +{% endblock %} diff --git a/ietf/secr/templates/sreq/session_request_notification.txt b/ietf/templates/meeting/session_request_notification.txt similarity index 56% rename from ietf/secr/templates/sreq/session_request_notification.txt rename to ietf/templates/meeting/session_request_notification.txt index 75f2cbbae4c..49dbbfc42ca 100644 --- a/ietf/secr/templates/sreq/session_request_notification.txt +++ b/ietf/templates/meeting/session_request_notification.txt @@ -1,5 +1,6 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} {% autoescape off %}{% load ams_filters %} {% filter wordwrap:78 %}{{ header }} meeting session request has just been submitted by {{ requester }}.{% endfilter %} -{% include "includes/session_info.txt" %}{% endautoescape %} +{% include "meeting/session_request_info.txt" %}{% endautoescape %} diff --git a/ietf/templates/meeting/session_request_status.html b/ietf/templates/meeting/session_request_status.html new file mode 100644 index 00000000000..65e98d6d238 --- /dev/null +++ b/ietf/templates/meeting/session_request_status.html @@ -0,0 +1,28 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load ietf_filters %} +{% load django_bootstrap5 %} +{% block title %}Session Request Status{% endblock %} +{% block content %} +

    Session Request Status

    + +
    +
    + Session Request Status +
    +
    +

    Enter the message that you would like displayed to the WG Chair when this tool is locked.

    +
    {% csrf_token %} + {% bootstrap_form form %} + {% if is_locked %} + + {% else %} + + {% endif %} + +
    +
    +
    + +{% endblock %} diff --git a/ietf/templates/meeting/session_request_view.html b/ietf/templates/meeting/session_request_view.html new file mode 100644 index 00000000000..3db16f56cbf --- /dev/null +++ b/ietf/templates/meeting/session_request_view.html @@ -0,0 +1,59 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static ietf_filters django_bootstrap5 %} +{% block title %}Session Request{% endblock %} + +{% block content %} +

    Session Request - IETF {{ meeting.number }}

    + +
    + Instructions +
    + +
    + +
    + + {% include "meeting/session_request_view_table.html" %} + +
    + +

    Activities Log

    +
    + + + + + + + + + + + {% for entry in activities %} + + + + + + + {% endfor %} + +
    DateTimeActionName
    {{ entry.act_date }}{{ entry.act_time }}{{ entry.activity }}{{ entry.act_by }}
    +
    + + + + {% if show_approve_button %} + Approve Third Session + {% endif %} + + Back + +
    + +{% endblock %} + +{% block js %} + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_request_view_formset.html b/ietf/templates/meeting/session_request_view_formset.html new file mode 100644 index 00000000000..72811b8c2c4 --- /dev/null +++ b/ietf/templates/meeting/session_request_view_formset.html @@ -0,0 +1,49 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load ams_filters %}{# keep this in sync with sessions_request_view_session_set.html #} +{% for sess_form in formset %} + {% if sess_form.cleaned_data and not sess_form.cleaned_data.DELETE %} +
    +
    + Session {{ forloop.counter }} +
    +
    +
    +
    Length
    +
    {{ sess_form.cleaned_data.requested_duration.total_seconds|display_duration }}
    +
    + {% if sess_form.cleaned_data.name %} +
    +
    Name
    +
    {{ sess_form.cleaned_data.name }}
    +
    + {% endif %} + {% if sess_form.cleaned_data.purpose.slug != 'regular' %} +
    +
    Purpose
    +
    + {{ sess_form.cleaned_data.purpose }} + {% if sess_form.cleaned_data.purpose.timeslot_types|length > 1 %}({{ sess_form.cleaned_data.type }} + ){% endif %} +
    +
    +
    +
    Onsite tool?
    +
    {{ sess_form.cleaned_data.has_onsite_tool|yesno }}
    +
    + {% endif %} +
    +
    + + {% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %} +
    +
    + Time between sessions +
    +
    + {% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No + preference{% endif %} +
    +
    + {% endif %} + {% endif %} +{% endfor %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_request_view_session_set.html b/ietf/templates/meeting/session_request_view_session_set.html new file mode 100644 index 00000000000..0b8412b04f3 --- /dev/null +++ b/ietf/templates/meeting/session_request_view_session_set.html @@ -0,0 +1,47 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load ams_filters %}{# keep this in sync with sessions_request_view_formset.html #} +{% for sess in session_set %} +
    +
    + Session {{ forloop.counter }} +
    +
    +
    +
    Length
    +
    {{ sess.requested_duration.total_seconds|display_duration }}
    +
    + {% if sess.name %} +
    +
    Name
    +
    {{ sess.name }}
    +
    + {% endif %} + {% if sess.purpose.slug != 'regular' %} +
    +
    Purpose
    +
    + {{ sess.purpose }} + {% if sess.purpose.timeslot_types|length > 1 %}({{ sess.type }}){% endif %} +
    +
    +
    +
    Onsite tool?
    +
    {{ sess.has_onsite_tool|yesno }}
    +
    + {% endif %} +
    +
    + +{% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %} +
    +
    + Time between sessions +
    +
    + {% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No + preference{% endif %} +
    +
    +{% endif %} + +{% endfor %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_request_view_table.html b/ietf/templates/meeting/session_request_view_table.html new file mode 100644 index 00000000000..a5cb85c2520 --- /dev/null +++ b/ietf/templates/meeting/session_request_view_table.html @@ -0,0 +1,146 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load ams_filters %} + +
    +
    + Working Group Name +
    +
    + {{ group.name }} ({{ group.acronym }}) +
    +
    + +
    +
    + Area Name +
    +
    + {{ group.parent }} +
    +
    + +
    +
    + Number of Sessions Requested +
    +
    + {% if session.third_session %}3{% else %}{{ session.num_session }}{% endif %} +
    +
    + +{% if form %} + {% include 'meeting/session_request_view_formset.html' with formset=form.session_forms group=group session=session only %} +{% else %} + {% include 'meeting/session_request_view_session_set.html' with session_set=sessions group=group session=session only %} +{% endif %} + + +
    +
    + Number of Attendees +
    +
    + {{ session.attendees }} +
    +
    + +
    +
    + Conflicts to Avoid +
    +
    + {% if session_conflicts.outbound %} + {% for conflict in session_conflicts.outbound %} +
    +
    + {{ conflict.name|title }} +
    +
    + {{ conflict.groups }} +
    +
    + {% endfor %} + {% else %}None{% endif %} +
    +
    + +
    +
    + Other WGs that included {{ group }} in their conflict list +
    +
    + {% if session_conflicts.inbound %}{{ session_conflicts.inbound }}{% else %}None so far{% endif %} +
    +
    + +{% if not is_virtual %} +
    +
    + Resources requested +
    +
    + {% if session.resources %}
      {% for resource in session.resources %}
    • {{ resource.desc }}
      • {% endfor %}
    {% else %}None so far{% endif %} +
    +
    +{% endif %} + +
    +
    + Participants who must be present +
    +
    + {% if session.bethere %}
      {% for person in session.bethere %}
    • {{ person }}
      • {% endfor %}
    {% else %}None{% endif %} +
    +
    + +
    +
    + Can not meet on +
    +
    + {% if session.timeranges_display %}{{ session.timeranges_display|join:', ' }}{% else %}No constraints{% endif %} +
    +
    + +{% if not is_virtual %} +
    +
    + Adjacent with WG +
    +
    + {{ session.adjacent_with_wg|default:'No preference' }} +
    +
    +
    +
    + Joint session +
    +
    + {% if session.joint_with_groups %} + {{ session.joint_for_session_display }} with: {{ session.joint_with_groups }} + {% else %} + Not a joint session + {% endif %} +
    +
    +{% endif %} + +
    +
    + Special Requests +
    +
    + {{ session.comments }} +
    +
    + +{% if form and form.notifications_optional %} +
    +
    + {{ form.send_notifications.label}} +
    +
    + {% if form.cleaned_data.send_notifications %}Yes{% else %}No{% endif %} +
    +
    +{% endif %} diff --git a/package.json b/package.json index e3e89288e77..e2e6fd7dab7 100644 --- a/package.json +++ b/package.json @@ -118,6 +118,7 @@ "ietf/static/js/complete-review.js", "ietf/static/js/create_timeslot.js", "ietf/static/js/create_timeslot.js", + "ietf/static/js/custom_striped.js", "ietf/static/js/d3.js", "ietf/static/js/datepicker.js", "ietf/static/js/doc-search.js", @@ -148,6 +149,8 @@ "ietf/static/js/password_strength.js", "ietf/static/js/select2.js", "ietf/static/js/session_details_form.js", + "ietf/static/js/session_form.js", + "ietf/static/js/session_request.js", "ietf/static/js/sortable.js", "ietf/static/js/stats.js", "ietf/static/js/status-change-edit-relations.js", @@ -208,8 +211,6 @@ "ietf/secr/static/images/tooltag-arrowright.webp", "ietf/secr/static/images/tooltag-arrowright_over.webp", "ietf/secr/static/js/dynamic_inlines.js", - "ietf/secr/static/js/session_form.js", - "ietf/secr/static/js/sessions.js", "ietf/secr/static/js/utils.js" ] } From 4961f376756de40ca1fe1d2db6a4ec7ff32b92a9 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 14 Oct 2025 14:51:43 -0500 Subject: [PATCH 041/313] feat: preview ballot email before save (#9646) (#9721) * feat: skeleton for modal email preview * fix: rudimentary transfer of the comment and discuss text * fix: put. the buttons. back. * fix: transfer of the data on the current form to the modal * fix: pull cc_select and additional_cc foward in the workflow UI * fix: refactor where ballot email is sent * fix: refactor build_position_email * chore: remove abandoned imports * chore: remove abandoned template --- ietf/doc/tests_ballot.py | 103 +++----- ietf/doc/tests_draft.py | 77 +++++- ietf/doc/tests_irsg_ballot.py | 63 +++-- ietf/doc/tests_rsab_ballot.py | 51 +--- ietf/doc/urls.py | 3 +- ietf/doc/views_ballot.py | 247 ++++++++++-------- ietf/mailtrigger/forms.py | 1 + ietf/templates/doc/ballot/edit_position.html | 89 ++++++- .../doc/ballot/send_ballot_comment.html | 44 ---- 9 files changed, 393 insertions(+), 285 deletions(-) delete mode 100644 ietf/templates/doc/ballot/send_ballot_comment.html diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py index 810ee598f65..8420e411e29 100644 --- a/ietf/doc/tests_ballot.py +++ b/ietf/doc/tests_ballot.py @@ -25,7 +25,6 @@ from ietf.group.models import Group, Role from ietf.group.factories import GroupFactory, RoleFactory, ReviewTeamFactory from ietf.ipr.factories import HolderIprDisclosureFactory -from ietf.name.models import BallotPositionName from ietf.iesg.models import TelechatDate from ietf.person.models import Person from ietf.person.factories import PersonFactory, PersonalApiKeyFactory @@ -37,9 +36,18 @@ class EditPositionTests(TestCase): + + # N.B. This test needs to be rewritten to exercise all types of ballots (iesg, irsg, rsab) + # and test against the output of the mailtriggers instead of looking for hardcoded values + # in the To and CC results. See #7864 def test_edit_position(self): ad = Person.objects.get(user__username="ad") - draft = IndividualDraftFactory(ad=ad,stream_id='ietf') + draft = WgDraftFactory( + ad=ad, + stream_id="ietf", + notify="somebody@example.com", + group__acronym="mars", + ) ballot = create_ballot_if_not_open(None, draft, ad, 'approve') url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) @@ -55,11 +63,20 @@ def test_edit_position(self): self.assertEqual(len(q('form textarea[name=comment]')), 1) # vote + empty_outbox() events_before = draft.docevent_set.count() - - r = self.client.post(url, dict(position="discuss", - discuss=" This is a discussion test. \n ", - comment=" This is a test. \n ")) + + r = self.client.post( + url, + dict( + position="discuss", + discuss=" This is a discussion test. \n ", + comment=" This is a test. \n ", + additional_cc="test298347@example.com", + cc_choices=["doc_notify", "doc_group_chairs"], + send_mail=1, + ), + ) self.assertEqual(r.status_code, 302) pos = draft.latest_event(BallotPositionDocEvent, balloter=ad) @@ -70,6 +87,22 @@ def test_edit_position(self): self.assertTrue(pos.comment_time != None) self.assertTrue("New position" in pos.desc) self.assertEqual(draft.docevent_set.count(), events_before + 3) + self.assertEqual(len(outbox),1) + m = outbox[0] + self.assertTrue("COMMENT" in m['Subject']) + self.assertTrue("DISCUSS" in m['Subject']) + self.assertTrue(draft.name in m['Subject']) + self.assertTrue("This is a discussion test." in str(m)) + self.assertTrue("This is a test" in str(m)) + self.assertTrue("iesg@" in m['To']) + # cc_choice doc_group_chairs + self.assertTrue("mars-chairs@" in m['Cc']) + # cc_choice doc_notify + self.assertTrue("somebody@example.com" in m['Cc']) + # cc_choice doc_group_email_list was not selected + self.assertFalse(draft.group.list_email in m['Cc']) + # extra-cc + self.assertTrue("test298347@example.com" in m['Cc']) # recast vote events_before = draft.docevent_set.count() @@ -230,64 +263,6 @@ def test_cannot_edit_position_as_pre_ad(self): r = self.client.post(url, dict(position="discuss", discuss="Test discuss text")) self.assertEqual(r.status_code, 403) - # N.B. This test needs to be rewritten to exercise all types of ballots (iesg, irsg, rsab) - # and test against the output of the mailtriggers instead of looking for hardcoded values - # in the To and CC results. See #7864 - def test_send_ballot_comment(self): - ad = Person.objects.get(user__username="ad") - draft = WgDraftFactory(ad=ad,group__acronym='mars') - draft.notify = "somebody@example.com" - draft.save_with_history([DocEvent.objects.create(doc=draft, rev=draft.rev, type="changed_document", by=Person.objects.get(user__username="secretary"), desc="Test")]) - - ballot = create_ballot_if_not_open(None, draft, ad, 'approve') - - BallotPositionDocEvent.objects.create( - doc=draft, rev=draft.rev, type="changed_ballot_position", - by=ad, balloter=ad, ballot=ballot, pos=BallotPositionName.objects.get(slug="discuss"), - discuss="This draft seems to be lacking a clearer title?", - discuss_time=timezone.now(), - comment="Test!", - comment_time=timezone.now()) - - url = urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=draft.name, - ballot_id=ballot.pk)) - login_testing_unauthorized(self, "ad", url) - - # normal get - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertTrue(len(q('form input[name="extra_cc"]')) > 0) - - # send - mailbox_before = len(outbox) - - r = self.client.post(url, dict(extra_cc="test298347@example.com", cc_choices=['doc_notify','doc_group_chairs'])) - self.assertEqual(r.status_code, 302) - - self.assertEqual(len(outbox), mailbox_before + 1) - m = outbox[-1] - self.assertTrue("COMMENT" in m['Subject']) - self.assertTrue("DISCUSS" in m['Subject']) - self.assertTrue(draft.name in m['Subject']) - self.assertTrue("clearer title" in str(m)) - self.assertTrue("Test!" in str(m)) - self.assertTrue("iesg@" in m['To']) - # cc_choice doc_group_chairs - self.assertTrue("mars-chairs@" in m['Cc']) - # cc_choice doc_notify - self.assertTrue("somebody@example.com" in m['Cc']) - # cc_choice doc_group_email_list was not selected - self.assertFalse(draft.group.list_email in m['Cc']) - # extra-cc - self.assertTrue("test298347@example.com" in m['Cc']) - - r = self.client.post(url, dict(cc="")) - self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), mailbox_before + 2) - m = outbox[-1] - self.assertTrue("iesg@" in m['To']) - self.assertFalse(m['Cc'] and draft.group.list_email in m['Cc']) class BallotWriteupsTests(TestCase): diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index ab33acebe6d..4d262c5a2fb 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- +import json import os import datetime import io @@ -11,7 +12,7 @@ from pathlib import Path from pyquery import PyQuery -from django.db.models import Q +from django.db.models import Max, Q from django.urls import reverse as urlreverse from django.conf import settings from django.utils import timezone @@ -2391,3 +2392,77 @@ def test_editorial_metadata(self): top_level_metadata_headings = q("tbody>tr>th:first-child").text() self.assertNotIn("IESG", top_level_metadata_headings) self.assertNotIn("IANA", top_level_metadata_headings) + +class BallotEmailAjaxTests(TestCase): + def test_ajax_build_position_email(self): + def _post_json(self, url, json_to_post): + r = self.client.post( + url, json.dumps(json_to_post), content_type="application/json" + ) + self.assertEqual(r.status_code, 200) + return json.loads(r.content) + + doc = WgDraftFactory() + ad = RoleFactory( + name_id="ad", group=doc.group, person__name="Some Areadirector" + ).person + url = urlreverse("ietf.doc.views_ballot.ajax_build_position_email") + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 405) + response = _post_json(self, url, {}) + self.assertFalse(response["success"]) + self.assertEqual(response["errors"], ["post_data not provided"]) + response = _post_json(self, url, {"dictis": "not empty"}) + self.assertFalse(response["success"]) + self.assertEqual(response["errors"], ["post_data not provided"]) + response = _post_json(self, url, {"post_data": {}}) + self.assertFalse(response["success"]) + self.assertEqual(len(response["errors"]), 7) + response = _post_json( + self, + url, + { + "post_data": { + "discuss": "aaaaaa", + "comment": "bbbbbb", + "position": "discuss", + "balloter": Person.objects.aggregate(maxpk=Max("pk") + 1)["maxpk"], + "docname": "this-draft-does-not-exist", + "cc_choices": ["doc_group_mail_list"], + "additional_cc": "foo@example.com", + } + }, + ) + self.assertFalse(response["success"]) + self.assertEqual( + response["errors"], + ["No person found matching balloter", "No document found matching docname"], + ) + response = _post_json( + self, + url, + { + "post_data": { + "discuss": "aaaaaa", + "comment": "bbbbbb", + "position": "discuss", + "balloter": ad.pk, + "docname": doc.name, + "cc_choices": ["doc_group_mail_list"], + "additional_cc": "foo@example.com", + } + }, + ) + self.assertTrue(response["success"]) + for snippet in [ + "aaaaaa", + "bbbbbb", + "DISCUSS", + ad.plain_name(), + doc.name, + doc.group.list_email, + "foo@example.com", + ]: + self.assertIn(snippet, response["text"]) + diff --git a/ietf/doc/tests_irsg_ballot.py b/ietf/doc/tests_irsg_ballot.py index aa62d8aaf9c..d96cf9dbefe 100644 --- a/ietf/doc/tests_irsg_ballot.py +++ b/ietf/doc/tests_irsg_ballot.py @@ -355,28 +355,35 @@ def test_issue_ballot(self): def test_take_and_email_position(self): draft = RgDraftFactory() ballot = IRSGBallotDocEventFactory(doc=draft) - url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) + self.balloter + url = ( + urlreverse( + "ietf.doc.views_ballot.edit_position", + kwargs=dict(name=draft.name, ballot_id=ballot.pk), + ) + + self.balloter + ) empty_outbox() login_testing_unauthorized(self, self.username, url) r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post(url, dict(position='yes', comment='oib239sb', send_mail='Save and send email')) + empty_outbox() + r = self.client.post( + url, + dict( + position="yes", + comment="oib239sb", + send_mail="Save and send email", + cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"], + ), + ) self.assertEqual(r.status_code, 302) e = draft.latest_event(BallotPositionDocEvent) - self.assertEqual(e.pos.slug,'yes') - self.assertEqual(e.comment, 'oib239sb') - - url = urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) + self.balloter - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - - r = self.client.post(url, dict(cc_choices=['doc_authors','doc_group_chairs','doc_group_mail_list'], body="Stuff")) - self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox),1) - self.assertNotIn('discuss-criteria', get_payload_text(outbox[0])) + self.assertEqual(e.pos.slug, "yes") + self.assertEqual(e.comment, "oib239sb") + self.assertEqual(len(outbox), 1) + self.assertNotIn("discuss-criteria", get_payload_text(outbox[0])) def test_close_ballot(self): draft = RgDraftFactory() @@ -482,27 +489,31 @@ def test_cant_take_position_on_iesg_ballot(self): def test_take_and_email_position(self): draft = RgDraftFactory() ballot = IRSGBallotDocEventFactory(doc=draft) - url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) + url = urlreverse( + "ietf.doc.views_ballot.edit_position", + kwargs=dict(name=draft.name, ballot_id=ballot.pk), + ) empty_outbox() login_testing_unauthorized(self, self.username, url) r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post(url, dict(position='yes', comment='oib239sb', send_mail='Save and send email')) + r = self.client.post( + url, + dict( + position="yes", + comment="oib239sb", + send_mail="Save and send email", + cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"], + ), + ) self.assertEqual(r.status_code, 302) e = draft.latest_event(BallotPositionDocEvent) - self.assertEqual(e.pos.slug,'yes') - self.assertEqual(e.comment, 'oib239sb') - - url = urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - - r = self.client.post(url, dict(cc_choices=['doc_authors','doc_group_chairs','doc_group_mail_list'], body="Stuff")) + self.assertEqual(e.pos.slug, "yes") + self.assertEqual(e.comment, "oib239sb") self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox),1) + self.assertEqual(len(outbox), 1) class IESGMemberTests(TestCase): diff --git a/ietf/doc/tests_rsab_ballot.py b/ietf/doc/tests_rsab_ballot.py index 028f548232e..9086106ba94 100644 --- a/ietf/doc/tests_rsab_ballot.py +++ b/ietf/doc/tests_rsab_ballot.py @@ -333,34 +333,19 @@ def test_take_and_email_position(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post( - url, - dict(position="yes", comment="oib239sb", send_mail="Save and send email"), - ) - self.assertEqual(r.status_code, 302) - e = draft.latest_event(BallotPositionDocEvent) - self.assertEqual(e.pos.slug, "yes") - self.assertEqual(e.comment, "oib239sb") - - url = ( - urlreverse( - "ietf.doc.views_ballot.send_ballot_comment", - kwargs=dict(name=draft.name, ballot_id=ballot.pk), - ) - + self.balloter - ) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - r = self.client.post( url, dict( + position="yes", + comment="oib239sb", + send_mail="Save and send email", cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"], - body="Stuff", ), ) self.assertEqual(r.status_code, 302) + e = draft.latest_event(BallotPositionDocEvent) + self.assertEqual(e.pos.slug, "yes") + self.assertEqual(e.comment, "oib239sb") self.assertEqual(len(outbox), 1) self.assertNotIn("discuss-criteria", get_payload_text(outbox[0])) @@ -532,31 +517,19 @@ def test_take_and_email_position(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post( - url, - dict(position="yes", comment="oib239sb", send_mail="Save and send email"), - ) - self.assertEqual(r.status_code, 302) - e = draft.latest_event(BallotPositionDocEvent) - self.assertEqual(e.pos.slug, "yes") - self.assertEqual(e.comment, "oib239sb") - - url = urlreverse( - "ietf.doc.views_ballot.send_ballot_comment", - kwargs=dict(name=draft.name, ballot_id=ballot.pk), - ) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - r = self.client.post( url, dict( + position="yes", + comment="oib239sb", + send_mail="Save and send email", cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"], - body="Stuff", ), ) self.assertEqual(r.status_code, 302) + e = draft.latest_event(BallotPositionDocEvent) + self.assertEqual(e.pos.slug, "yes") + self.assertEqual(e.comment, "oib239sb") self.assertEqual(len(outbox), 1) diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py index 7b444782d77..8e9c0569e22 100644 --- a/ietf/doc/urls.py +++ b/ietf/doc/urls.py @@ -93,6 +93,8 @@ url(r'^ballots/irsg/$', views_ballot.irsg_ballot_status), url(r'^ballots/rsab/$', views_ballot.rsab_ballot_status), + url(r'^build-position-email/$', views_ballot.ajax_build_position_email), + url(r'^(?P(bcp|std|fyi))/?$', views_search.index_subseries), url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main), @@ -111,7 +113,6 @@ url(r'^%(name)s/ballot/rsab/$' % settings.URL_REGEXPS, views_doc.document_rsab_ballot), url(r'^%(name)s/ballot/(?P[0-9]+)/$' % settings.URL_REGEXPS, views_doc.document_ballot), url(r'^%(name)s/ballot/(?P[0-9]+)/position/$' % settings.URL_REGEXPS, views_ballot.edit_position), - url(r'^%(name)s/ballot/(?P[0-9]+)/emailposition/$' % settings.URL_REGEXPS, views_ballot.send_ballot_comment), url(r'^%(name)s/(?:%(rev)s/)?doc.json$' % settings.URL_REGEXPS, views_doc.document_json), url(r'^%(name)s/ballotpopup/(?P[0-9]+)/$' % settings.URL_REGEXPS, views_doc.ballot_popup), url(r'^(?P[A-Za-z0-9._+-]+)/reviewrequest/', include("ietf.doc.urls_review")), diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py index 0ba340890d6..03cf01a4a16 100644 --- a/ietf/doc/views_ballot.py +++ b/ietf/doc/views_ballot.py @@ -4,18 +4,18 @@ # Directors and Secretariat -import datetime, json +import datetime +import json from django import forms from django.conf import settings -from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseBadRequest +from django.http import HttpResponse, HttpResponseNotAllowed, HttpResponseRedirect, Http404, HttpResponseBadRequest from django.shortcuts import render, get_object_or_404, redirect from django.template.defaultfilters import striptags from django.template.loader import render_to_string from django.urls import reverse as urlreverse from django.views.decorators.csrf import csrf_exempt from django.utils.html import escape -from urllib.parse import urlencode as urllib_urlencode import debug # pyflakes:ignore @@ -34,14 +34,15 @@ from ietf.doc.templatetags.ietf_filters import can_ballot from ietf.iesg.models import TelechatDate from ietf.ietfauth.utils import has_role, role_required, is_authorized_in_doc_stream +from ietf.mailtrigger.models import Recipient from ietf.mailtrigger.utils import gather_address_lists from ietf.mailtrigger.forms import CcSelectForm from ietf.message.utils import infer_message from ietf.name.models import BallotPositionName, DocTypeName from ietf.person.models import Person -from ietf.utils.fields import ModelMultipleChoiceField +from ietf.utils.fields import ModelMultipleChoiceField, MultiEmailField from ietf.utils.http import validate_return_to_path -from ietf.utils.mail import send_mail_text, send_mail_preformatted +from ietf.utils.mail import decode_header_value, send_mail_text, send_mail_preformatted from ietf.utils.decorators import require_api_key from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, datetime_from_date, DEADLINE_TZINFO @@ -179,6 +180,9 @@ def save_position(form, doc, ballot, balloter, login=None, send_email=False): return pos +class AdditionalCCForm(forms.Form): + additional_cc = MultiEmailField(required=False) + @role_required("Area Director", "Secretariat", "IRSG Member", "RSAB Member") def edit_position(request, name, ballot_id): """Vote and edit discuss and comment on document""" @@ -199,50 +203,67 @@ def edit_position(request, name, ballot_id): raise Http404 balloter = get_object_or_404(Person, pk=balloter_id) + if doc.stream_id == 'irtf': + mailtrigger_slug='irsg_ballot_saved' + elif doc.stream_id == 'editorial': + mailtrigger_slug='rsab_ballot_saved' + else: + mailtrigger_slug='iesg_ballot_saved' + if request.method == 'POST': old_pos = None if not has_role(request.user, "Secretariat") and not can_ballot(request.user, doc): # prevent pre-ADs from taking a position permission_denied(request, "Must be an active member (not a pre-AD for example) of the balloting body to take a position") + if request.POST.get("Defer") and doc.stream.slug != "irtf": + return redirect('ietf.doc.views_ballot.defer_ballot', name=doc) + elif request.POST.get("Undefer") and doc.stream.slug != "irtf": + return redirect('ietf.doc.views_ballot.undefer_ballot', name=doc) + form = EditPositionForm(request.POST, ballot_type=ballot.ballot_type) - if form.is_valid(): + cc_select_form = CcSelectForm(data=request.POST,mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc}) + additional_cc_form = AdditionalCCForm(request.POST) + if form.is_valid() and cc_select_form.is_valid() and additional_cc_form.is_valid(): send_mail = True if request.POST.get("send_mail") else False - save_position(form, doc, ballot, balloter, login, send_mail) - + pos = save_position(form, doc, ballot, balloter, login, send_mail) if send_mail: - query = {} - if request.GET.get('balloter'): - query['balloter'] = request.GET.get('balloter') - if request.GET.get('ballot_edit_return_point'): - query['ballot_edit_return_point'] = request.GET.get('ballot_edit_return_point') - qstr = "" - if len(query) > 0: - qstr = "?" + urllib_urlencode(query, safe='/') - return HttpResponseRedirect(urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=doc.name, ballot_id=ballot_id)) + qstr) - elif request.POST.get("Defer") and doc.stream.slug != "irtf": - return redirect('ietf.doc.views_ballot.defer_ballot', name=doc) - elif request.POST.get("Undefer") and doc.stream.slug != "irtf": - return redirect('ietf.doc.views_ballot.undefer_ballot', name=doc) - else: - return HttpResponseRedirect(return_to_url) + addrs, frm, subject, body = build_position_email(balloter, doc, pos) + if doc.stream_id == 'irtf': + mailtrigger_slug='irsg_ballot_saved' + elif doc.stream_id == 'editorial': + mailtrigger_slug='rsab_ballot_saved' + else: + mailtrigger_slug='iesg_ballot_saved' + cc = [] + cc.extend(cc_select_form.get_selected_addresses()) + extra_cc = additional_cc_form.cleaned_data["additional_cc"] + if extra_cc: + cc.extend(extra_cc) + cc_set = set(cc) + cc_set.discard("") + cc = sorted(list(cc_set)) + send_mail_text(request, addrs.to, frm, subject, body, cc=", ".join(cc)) + return redirect(return_to_url) else: initial = {} old_pos = doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position", balloter=balloter, ballot=ballot) if old_pos: initial['position'] = old_pos.pos.slug initial['discuss'] = old_pos.discuss - initial['comment'] = old_pos.comment - + initial['comment'] = old_pos.comment form = EditPositionForm(initial=initial, ballot_type=ballot.ballot_type) + cc_select_form = CcSelectForm(mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc}) + additional_cc_form = AdditionalCCForm() blocking_positions = dict((p.pk, p.name) for p in form.fields["position"].queryset.all() if p.blocking) - ballot_deferred = doc.active_defer_event() return render(request, 'doc/ballot/edit_position.html', dict(doc=doc, form=form, + cc_select_form=cc_select_form, + additional_cc_form=additional_cc_form, balloter=balloter, return_to_url=return_to_url, old_pos=old_pos, @@ -301,21 +322,98 @@ def err(code, text): ) -def build_position_email(balloter, doc, pos): +@role_required("Area Director", "Secretariat") +@csrf_exempt +def ajax_build_position_email(request): + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + errors = list() + try: + json_body = json.loads(request.body) + except json.decoder.JSONDecodeError: + errors.append("Post body is not valid json") + if len(errors) == 0: + post_data = json_body.get("post_data") + if post_data is None: + errors.append("post_data not provided") + else: + for key in [ + "discuss", + "comment", + "position", + "balloter", + "docname", + "cc_choices", + "additional_cc", + ]: + if key not in post_data: + errors.append(f"{key} not found in post_data") + if len(errors) == 0: + person = Person.objects.filter(pk=post_data.get("balloter")).first() + if person is None: + errors.append("No person found matching balloter") + doc = Document.objects.filter(name=post_data.get("docname")).first() + if doc is None: + errors.append("No document found matching docname") + if len(errors) > 0: + response = { + "success": False, + "errors": errors, + } + else: + wanted = dict() # consider named tuple instead + wanted["discuss"] = post_data.get("discuss") + wanted["comment"] = post_data.get("comment") + wanted["position_name"] = post_data.get("position") + wanted["balloter"] = person + wanted["doc"] = doc + addrs, frm, subject, body = build_position_email_from_dict(wanted) + + recipient_slugs = post_data.get("cc_choices") + # Consider refactoring gather_address_lists so this isn't duplicated from there + cc_addrs = set() + for r in Recipient.objects.filter(slug__in=recipient_slugs): + cc_addrs.update(r.gather(doc=doc)) + additional_cc = post_data.get("additional_cc") + for addr in additional_cc.split(","): + cc_addrs.add(addr.strip()) + cc_addrs.discard("") + cc_addrs = sorted(list(cc_addrs)) + + response_text = "\n".join( + [ + f"From: {decode_header_value(frm)}", + f"To: {', '.join([decode_header_value(addr) for addr in addrs.to])}", + f"Cc: {', '.join([decode_header_value(addr) for addr in cc_addrs])}", + f"Subject: {subject}", + "", + body, + ] + ) + + response = { + "success": True, + "text": response_text, + } + return HttpResponse(json.dumps(response), content_type="application/json") + +def build_position_email_from_dict(pos_dict): + doc = pos_dict["doc"] subj = [] d = "" blocking_name = "DISCUSS" - if pos.pos.blocking and pos.discuss: - d = pos.discuss - blocking_name = pos.pos.name.upper() + pos_name = BallotPositionName.objects.filter(slug=pos_dict["position_name"]).first() + if pos_name.blocking and pos_dict.get("discuss"): + d = pos_dict.get("discuss") + blocking_name = pos_name.name.upper() subj.append(blocking_name) c = "" - if pos.comment: - c = pos.comment + if pos_dict.get("comment"): + c = pos_dict.get("comment") subj.append("COMMENT") - + balloter = pos_dict.get("balloter") balloter_name_genitive = balloter.plain_name() + "'" if balloter.plain_name().endswith('s') else balloter.plain_name() + "'s" - subject = "%s %s on %s" % (balloter_name_genitive, pos.pos.name if pos.pos else "No Position", doc.name + "-" + doc.rev) + subject = "%s %s on %s" % (balloter_name_genitive, pos_name.name if pos_name else "No Position", doc.name + "-" + doc.rev) if subj: subject += ": (with %s)" % " and ".join(subj) @@ -324,7 +422,7 @@ def build_position_email(balloter, doc, pos): comment=c, balloter=balloter.plain_name(), doc=doc, - pos=pos.pos, + pos=pos_name, blocking_name=blocking_name, settings=settings)) frm = balloter.role_email("ad").formatted_email() @@ -338,79 +436,16 @@ def build_position_email(balloter, doc, pos): return addrs, frm, subject, body -@role_required('Area Director','Secretariat','IRSG Member', 'RSAB Member') -def send_ballot_comment(request, name, ballot_id): - """Email document ballot position discuss/comment for Area Director.""" - doc = get_object_or_404(Document, name=name) - ballot = get_object_or_404(BallotDocEvent, type="created_ballot", pk=ballot_id, doc=doc) - if not has_role(request.user, 'Secretariat'): - if any([ - doc.stream_id == 'ietf' and not has_role(request.user, 'Area Director'), - doc.stream_id == 'irtf' and not has_role(request.user, 'IRSG Member'), - doc.stream_id == 'editorial' and not has_role(request.user, 'RSAB Member'), - ]): - raise Http404 - - balloter = request.user.person - - try: - return_to_url = parse_ballot_edit_return_point(request.GET.get('ballot_edit_return_point'), doc.name, ballot_id) - except ValueError: - return HttpResponseBadRequest('ballot_edit_return_point is invalid') - - if 'HTTP_REFERER' in request.META: - back_url = request.META['HTTP_REFERER'] - else: - back_url = urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name, ballot_id=ballot_id)) - - # if we're in the Secretariat, we can select a balloter (such as an AD) to act as stand-in for - if has_role(request.user, "Secretariat"): - balloter_id = request.GET.get('balloter') - if not balloter_id: - raise Http404 - balloter = get_object_or_404(Person, pk=balloter_id) - - pos = doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position", balloter=balloter, ballot=ballot) - if not pos: - raise Http404 - - addrs, frm, subject, body = build_position_email(balloter, doc, pos) - - if doc.stream_id == 'irtf': - mailtrigger_slug='irsg_ballot_saved' - elif doc.stream_id == 'editorial': - mailtrigger_slug='rsab_ballot_saved' - else: - mailtrigger_slug='iesg_ballot_saved' - - if request.method == 'POST': - cc = [] - cc_select_form = CcSelectForm(data=request.POST,mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc}) - if cc_select_form.is_valid(): - cc.extend(cc_select_form.get_selected_addresses()) - extra_cc = [x.strip() for x in request.POST.get("extra_cc","").split(',') if x.strip()] - if extra_cc: - cc.extend(extra_cc) - - send_mail_text(request, addrs.to, frm, subject, body, cc=", ".join(cc)) - - return HttpResponseRedirect(return_to_url) - - else: +def build_position_email(balloter, doc, pos): - cc_select_form = CcSelectForm(mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc}) - - return render(request, 'doc/ballot/send_ballot_comment.html', - dict(doc=doc, - subject=subject, - body=body, - frm=frm, - to=addrs.as_strings().to, - balloter=balloter, - back_url=back_url, - cc_select_form = cc_select_form, - )) + pos_dict=dict() + pos_dict["doc"]=doc + pos_dict["position_name"]=pos.pos.slug + pos_dict["discuss"]=pos.discuss + pos_dict["comment"]=pos.comment + pos_dict["balloter"]=balloter + return build_position_email_from_dict(pos_dict) @role_required('Area Director','Secretariat') def clear_ballot(request, name, ballot_type_slug): diff --git a/ietf/mailtrigger/forms.py b/ietf/mailtrigger/forms.py index 366c429d8c5..8d13c5edf3e 100644 --- a/ietf/mailtrigger/forms.py +++ b/ietf/mailtrigger/forms.py @@ -11,6 +11,7 @@ class CcSelectForm(forms.Form): expansions = dict() # type: Dict[str, List[str]] cc_choices = forms.MultipleChoiceField( + required=False, label='Cc', choices=[], widget=forms.CheckboxSelectMultiple(), diff --git a/ietf/templates/doc/ballot/edit_position.html b/ietf/templates/doc/ballot/edit_position.html index 293c186112d..b57e9a36522 100644 --- a/ietf/templates/doc/ballot/edit_position.html +++ b/ietf/templates/doc/ballot/edit_position.html @@ -20,24 +20,48 @@

    Ballot deferred by {{ ballot_deferred.by }} on {{ ballot_deferred.time|date:"Y-m-d" }}.

    {% endif %} +
    +
    + {% if form.errors or cc_select_form.errors or additional_cc_form.errors %} +
    + There were errors in the submitted form -- see below. Please correct these and resubmit. +
    + {% if form.errors %} +
    Position entry
    + {% bootstrap_form_errors form %} + {% endif %} + {% if cc_select_form.errors %} +
    CC selection
    + {% bootstrap_form_errors cc_select_form %} + {% endif %} + {% if additional_cc_form.errors %} +
    Additional Cc Addresses
    + {% bootstrap_form_errors additional_cc_form %} + {% endif %} + {% endif %}
    {% csrf_token %} {% for field in form %} {% if field.name == "discuss" %}
    {% endif %} {% bootstrap_field field %} {% if field.name == "discuss" and old_pos and old_pos.discuss_time %} -
    Last edited {{ old_pos.discuss_time }}
    +
    Last saved {{ old_pos.discuss_time }}
    {% elif field.name == "comment" and old_pos and old_pos.comment_time %} -
    Last edited {{ old_pos.comment_time }}
    +
    Last saved {{ old_pos.comment_time }}
    {% endif %} {% if field.name == "discuss" %}
    {% endif %} {% endfor %} + {% bootstrap_form cc_select_form %} + {% bootstrap_form additional_cc_form %}
    + - + {% if doc.type_id == "draft" or doc.type_id == "conflrev" %} {% if doc.stream.slug != "irtf" %} {% if ballot_deferred %} @@ -58,7 +82,24 @@

    Back

    -
    + + + {% endblock %} {% block js %} + + {% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/ballot/send_ballot_comment.html b/ietf/templates/doc/ballot/send_ballot_comment.html deleted file mode 100644 index 1c5f5218599..00000000000 --- a/ietf/templates/doc/ballot/send_ballot_comment.html +++ /dev/null @@ -1,44 +0,0 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin %} -{% load django_bootstrap5 %} -{% load ietf_filters %} -{% block title %}Send ballot position for {{ balloter }} on {{ doc }}{% endblock %} -{% block content %} - {% origin %} -

    - Send ballot position for {{ balloter }} -
    - {{ doc }} -

    -
    - {% csrf_token %} -
    - - -
    -
    - - -
    - {% bootstrap_form cc_select_form %} -
    - - -
    Separate email addresses with commas.
    -
    -
    - - -
    -
    -

    Body

    - 
    {{ body|maybewordwrap }}
    -
    - - - Back - -
    -{% endblock %} From 8f2feef631acbd8b181a845140647c2c83a9299f Mon Sep 17 00:00:00 2001 From: NGPixel Date: Tue, 14 Oct 2025 18:57:50 -0400 Subject: [PATCH 042/313] ci: update build workflow to deploy to dev --- .github/workflows/build.yml | 71 ++--- dev/k8s-get-deploy-name/.editorconfig | 7 + dev/k8s-get-deploy-name/.gitignore | 1 + dev/k8s-get-deploy-name/.npmrc | 3 + dev/k8s-get-deploy-name/README.md | 16 ++ dev/k8s-get-deploy-name/cli.js | 22 ++ dev/k8s-get-deploy-name/package-lock.json | 303 ++++++++++++++++++++++ dev/k8s-get-deploy-name/package.json | 8 + 8 files changed, 396 insertions(+), 35 deletions(-) create mode 100644 dev/k8s-get-deploy-name/.editorconfig create mode 100644 dev/k8s-get-deploy-name/.gitignore create mode 100644 dev/k8s-get-deploy-name/.npmrc create mode 100644 dev/k8s-get-deploy-name/README.md create mode 100644 dev/k8s-get-deploy-name/cli.js create mode 100644 dev/k8s-get-deploy-name/package-lock.json create mode 100644 dev/k8s-get-deploy-name/package.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8567446cae5..15eaba48d1a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,13 +16,13 @@ on: - Skip - Staging Only - Staging + Prod - sandbox: - description: 'Deploy to Sandbox' + dev: + description: 'Deploy to Dev' default: true required: true type: boolean - sandboxNoDbRefresh: - description: 'Sandbox Disable Daily DB Refresh' + devNoDbRefresh: + description: 'Dev Disable Daily DB Refresh' default: false required: true type: boolean @@ -392,44 +392,45 @@ jobs: value: "Failed" # ----------------------------------------------------------------- - # SANDBOX + # DEV # ----------------------------------------------------------------- - sandbox: - name: Deploy to Sandbox - if: ${{ !failure() && !cancelled() && github.event.inputs.sandbox == 'true' }} + dev: + name: Deploy to Dev + if: ${{ !failure() && !cancelled() && github.event.inputs.dev == 'true' }} needs: [prepare, release] - runs-on: [self-hosted, dev-server] + runs-on: ubuntu-latest environment: - name: sandbox + name: dev env: PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} steps: - - uses: actions/checkout@v4 - - - name: Download a Release Artifact - uses: actions/download-artifact@v4.3.0 - with: - name: release-${{ env.PKG_VERSION }} - - - name: Deploy to containers - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Reset production flags in settings.py..." - sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = True/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'development'/" ietf/settings.py - echo "Install Deploy to Container CLI dependencies..." - cd dev/deploy-to-container - npm ci - cd ../.. - echo "Start Deploy..." - node ./dev/deploy-to-container/cli.js --branch ${{ github.ref_name }} --domain dev.ietf.org --appversion ${{ env.PKG_VERSION }} --commit ${{ github.sha }} --ghrunid ${{ github.run_id }} --nodbrefresh ${{ github.event.inputs.sandboxNoDbRefresh }} - - - name: Cleanup old docker resources - env: - DEBIAN_FRONTEND: noninteractive - run: | - docker image prune -a -f + - uses: actions/checkout@v4 + with: + ref: main + + - name: Get Deploy Name + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "Install Get Deploy Name CLI dependencies..." + cd dev/k8s-get-deploy-name + npm ci + echo "Get Deploy Name..." + echo "DEPLOY_NAMESPACE=$(node cli.js --branch ${{ github.ref_name }})" >> "$GITHUB_ENV" + + - name: Deploy to dev + uses: the-actions-org/workflow-dispatch@v4 + with: + workflow: deploy-dev.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}" }' + wait-for-completion: true + wait-for-completion-timeout: 30m + wait-for-completion-interval: 30s + display-workflow-run-url: false # ----------------------------------------------------------------- # STAGING diff --git a/dev/k8s-get-deploy-name/.editorconfig b/dev/k8s-get-deploy-name/.editorconfig new file mode 100644 index 00000000000..fec5c665197 --- /dev/null +++ b/dev/k8s-get-deploy-name/.editorconfig @@ -0,0 +1,7 @@ +[*] +indent_size = 2 +indent_style = space +charset = utf-8 +trim_trailing_whitespace = false +end_of_line = lf +insert_final_newline = true diff --git a/dev/k8s-get-deploy-name/.gitignore b/dev/k8s-get-deploy-name/.gitignore new file mode 100644 index 00000000000..07e6e472cc7 --- /dev/null +++ b/dev/k8s-get-deploy-name/.gitignore @@ -0,0 +1 @@ +/node_modules diff --git a/dev/k8s-get-deploy-name/.npmrc b/dev/k8s-get-deploy-name/.npmrc new file mode 100644 index 00000000000..580a68c499d --- /dev/null +++ b/dev/k8s-get-deploy-name/.npmrc @@ -0,0 +1,3 @@ +audit = false +fund = false +save-exact = true diff --git a/dev/k8s-get-deploy-name/README.md b/dev/k8s-get-deploy-name/README.md new file mode 100644 index 00000000000..a6605e4dd2c --- /dev/null +++ b/dev/k8s-get-deploy-name/README.md @@ -0,0 +1,16 @@ +# Datatracker Get Deploy Name + +This tool process and slugify a git branch into an appropriate subdomain name. + +## Usage + +1. From the `dev/k8s-get-deploy-name` directory, install the dependencies: +```sh +npm install +``` +2. Run the command: (replacing the `branch` argument) +```sh +node /cli.js --branch feat/fooBar-123 +``` + +The subdomain name will be output. It can then be used in a workflow as a namespace name and subdomain value. diff --git a/dev/k8s-get-deploy-name/cli.js b/dev/k8s-get-deploy-name/cli.js new file mode 100644 index 00000000000..b6c3b5119ef --- /dev/null +++ b/dev/k8s-get-deploy-name/cli.js @@ -0,0 +1,22 @@ +#!/usr/bin/env node + +import yargs from 'yargs/yargs' +import { hideBin } from 'yargs/helpers' +import slugify from 'slugify' + +const argv = yargs(hideBin(process.argv)).argv + +let branch = argv.branch +if (!branch) { + throw new Error('Missing --branch argument!') +} +if (branch.indexOf('/') >= 0) { + branch = branch.split('/').slice(1).join('-') +} +branch = slugify(branch, { lower: true, strict: true }) +if (branch.length < 1) { + throw new Error('Branch name is empty!') +} +process.stdout.write(`dt-${branch}`) + +process.exit(0) diff --git a/dev/k8s-get-deploy-name/package-lock.json b/dev/k8s-get-deploy-name/package-lock.json new file mode 100644 index 00000000000..e492a4cd38c --- /dev/null +++ b/dev/k8s-get-deploy-name/package-lock.json @@ -0,0 +1,303 @@ +{ + "name": "k8s-get-deploy-name", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "k8s-get-deploy-name", + "dependencies": { + "slugify": "1.6.6", + "yargs": "17.7.2" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/slugify": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", + "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "engines": { + "node": ">=12" + } + } + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" + }, + "slugify": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", + "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==" + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" + }, + "yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "requires": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + } + }, + "yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" + } + } +} diff --git a/dev/k8s-get-deploy-name/package.json b/dev/k8s-get-deploy-name/package.json new file mode 100644 index 00000000000..849f5d9b8d0 --- /dev/null +++ b/dev/k8s-get-deploy-name/package.json @@ -0,0 +1,8 @@ +{ + "name": "k8s-get-deploy-name", + "type": "module", + "dependencies": { + "slugify": "1.6.6", + "yargs": "17.7.2" + } +} From 5a7be260dd6dfd9c484bc7c50ef991642fa8ad8e Mon Sep 17 00:00:00 2001 From: NGPixel Date: Wed, 15 Oct 2025 03:07:07 -0400 Subject: [PATCH 043/313] chore: add disableDailyDbRefresh flag to build workflow --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 15eaba48d1a..4c70456a73e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -426,7 +426,7 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}" }' + inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' wait-for-completion: true wait-for-completion-timeout: 30m wait-for-completion-interval: 30s From 93c1124c21267556625df760c68f35f6d4ae8139 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 15 Oct 2025 13:06:45 -0500 Subject: [PATCH 044/313] ci: add ruff to devcontainer (#9731) --- .devcontainer/devcontainer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6b0fd79bb37..bf285500840 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -23,7 +23,6 @@ "dbaeumer.vscode-eslint", "eamodio.gitlens", "editorconfig.editorconfig", - // Newer volar >=3.0.0 causes crashes in devcontainers "vue.volar@2.2.10", "mrmlnc.vscode-duplicate", "ms-azuretools.vscode-docker", @@ -35,7 +34,8 @@ "redhat.vscode-yaml", "spmeesseman.vscode-taskexplorer", "visualstudioexptteam.vscodeintellicode", - "ms-python.pylint" + "ms-python.pylint", + "charliermarsh.ruff" ], "settings": { "terminal.integrated.defaultProfile.linux": "zsh", From d5660ab8e953fec25dbb20025aba73b2e58f0609 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 15 Oct 2025 18:30:18 -0300 Subject: [PATCH 045/313] fix: unbreak EmptyAwareJSONField (#9732) * fix: specify default form_class correctly * style: ruff ruff --- ietf/utils/db.py | 63 ++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/ietf/utils/db.py b/ietf/utils/db.py index 865c9b999fa..49c89da13ac 100644 --- a/ietf/utils/db.py +++ b/ietf/utils/db.py @@ -1,33 +1,44 @@ -# Copyright The IETF Trust 2021, All Rights Reserved -# -*- coding: utf-8 -*- - -# Taken from/inspired by -# https://stackoverflow.com/questions/55147169/django-admin-jsonfield-default-empty-dict-wont-save-in-admin -# -# JSONField should recognize {}, (), and [] as valid, non-empty JSON -# values. However, the base Field class excludes them +# Copyright The IETF Trust 2021-2025, All Rights Reserved import jsonfield from django.db import models -from ietf.utils.fields import IETFJSONField as FormIETFJSONField, EmptyAwareJSONField as FormEmptyAwareJSONField +from ietf.utils.fields import ( + IETFJSONField as FormIETFJSONField, + EmptyAwareJSONField as FormEmptyAwareJSONField, +) class EmptyAwareJSONField(models.JSONField): - form_class = FormEmptyAwareJSONField + """JSONField that allows empty JSON values when model specifies empty=False + + Taken from/inspired by + https://stackoverflow.com/questions/55147169/django-admin-jsonfield-default-empty-dict-wont-save-in-admin + + JSONField should recognize {}, (), and [] as valid, non-empty JSON values. - def __init__(self, *args, empty_values=FormEmptyAwareJSONField.empty_values, accepted_empty_values=None, **kwargs): + If customizing the formfield, the field must accept the `empty_values` argument. + """ + + def __init__( + self, + *args, + empty_values=FormEmptyAwareJSONField.empty_values, + accepted_empty_values=None, + **kwargs, + ): if accepted_empty_values is None: accepted_empty_values = [] - self.empty_values = [x - for x in empty_values - if x not in accepted_empty_values] + self.empty_values = [x for x in empty_values if x not in accepted_empty_values] super().__init__(*args, **kwargs) def formfield(self, **kwargs): - if 'form_class' not in kwargs or issubclass(kwargs['form_class'], FormEmptyAwareJSONField): - kwargs.setdefault('empty_values', self.empty_values) - return super().formfield(**{**kwargs}) + defaults = { + "form_class": FormEmptyAwareJSONField, + "empty_values": self.empty_values, + } + defaults.update(kwargs) + return super().formfield(**defaults) class IETFJSONField(jsonfield.JSONField): # pragma: no cover @@ -36,15 +47,21 @@ class IETFJSONField(jsonfield.JSONField): # pragma: no cover # Remove this class when migrations are squashed and it is no longer referenced form_class = FormIETFJSONField - def __init__(self, *args, empty_values=FormIETFJSONField.empty_values, accepted_empty_values=None, **kwargs): + def __init__( + self, + *args, + empty_values=FormIETFJSONField.empty_values, + accepted_empty_values=None, + **kwargs, + ): if accepted_empty_values is None: accepted_empty_values = [] - self.empty_values = [x - for x in empty_values - if x not in accepted_empty_values] + self.empty_values = [x for x in empty_values if x not in accepted_empty_values] super().__init__(*args, **kwargs) def formfield(self, **kwargs): - if 'form_class' not in kwargs or issubclass(kwargs['form_class'], FormIETFJSONField): - kwargs.setdefault('empty_values', self.empty_values) + if "form_class" not in kwargs or issubclass( + kwargs["form_class"], FormIETFJSONField + ): + kwargs.setdefault("empty_values", self.empty_values) return super().formfield(**{**kwargs}) From 1d2d304fa5c99db6cd2a944328246ce900c73b3c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 16 Oct 2025 12:39:04 -0300 Subject: [PATCH 046/313] fix: improve proceedings caching/performance (#9733) * refactor: speed up get_attendance() * fix: avoid cache invalidation by later draft rev * fix: guard against empty value * feat: freeze cache key for final proceedings --- ietf/meeting/models.py | 28 +++++++++++++++++++++------- ietf/meeting/utils.py | 39 +++++++++++++++++++++++++++++++++++---- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index f3df23e9169..9e44df33b74 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -250,25 +250,39 @@ def get_attendance(self): # MeetingRegistration.attended started conflating badge-pickup and session attendance before IETF 114. # We've separated session attendance off to ietf.meeting.Attended, but need to report attendance at older # meetings correctly. - + # + # Looking up by registration and attendance records separately and joining in + # python is far faster than combining the Q objects in the query (~100x). + # Further optimization may be possible, but the queries are tricky... attended_per_meeting_registration = ( Q(registration__meeting=self) & ( Q(registration__attended=True) | Q(registration__checkedin=True) ) ) + attendees_by_reg = set( + Person.objects.filter(attended_per_meeting_registration).values_list( + "pk", flat=True + ) + ) + attended_per_meeting_attended = ( Q(attended__session__meeting=self) # Note that we are not filtering to plenary, wg, or rg sessions # as we do for nomcom eligibility - if picking up a badge (see above) # is good enough, just attending e.g. a training session is also good enough ) - attended = Person.objects.filter( - attended_per_meeting_registration | attended_per_meeting_attended - ).distinct() - - onsite = set(attended.filter(registration__meeting=self, registration__tickets__attendance_type__slug='onsite')) - remote = set(attended.filter(registration__meeting=self, registration__tickets__attendance_type__slug='remote')) + attendees_by_att = set( + Person.objects.filter(attended_per_meeting_attended).values_list( + "pk", flat=True + ) + ) + + attendees = Person.objects.filter( + pk__in=attendees_by_att | attendees_by_reg + ) + onsite = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='onsite')) + remote = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='remote')) remote.difference_update(onsite) return Attendance( diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index f6925269aad..feadb0c7fd4 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -1027,10 +1027,41 @@ def generate_proceedings_content(meeting, force_refresh=False): :force_refresh: true to force regeneration and cache refresh """ cache = caches["default"] - cache_version = Document.objects.filter(session__meeting__number=meeting.number).aggregate(Max('time'))["time__max"] - # Include proceedings_final in the bare_key so we'll always reflect that accurately, even at the cost of - # a recomputation in the view - bare_key = f"proceedings.{meeting.number}.{cache_version}.final={meeting.proceedings_final}" + key_components = [ + "proceedings", + str(meeting.number), + ] + if meeting.proceedings_final: + # Freeze the cache key once proceedings are finalized. Further changes will + # not be picked up until the cache expires or is refreshed by the + # proceedings_content_refresh_task() + key_components.append("final") + else: + # Build a cache key that changes when materials are modified. For all but drafts, + # use the last modification time of the document. Exclude drafts from this because + # revisions long after the meeting ends will otherwise show up as changes and + # incorrectly invalidate the cache. Instead, include an ordered list of the + # drafts linked to the meeting so adding or removing drafts will trigger a + # recalculation. The list is long but that doesn't matter because we hash it into + # a fixed-length key. + meeting_docs = Document.objects.filter(session__meeting__number=meeting.number) + last_materials_update = ( + meeting_docs.exclude(type_id="draft") + .filter(session__meeting__number=meeting.number) + .aggregate(Max("time"))["time__max"] + ) + draft_names = ( + meeting_docs + .filter(type_id="draft") + .order_by("name") + .values_list("name", flat=True) + ) + key_components += [ + last_materials_update.isoformat() if last_materials_update else "-", + ",".join(draft_names), + ] + + bare_key = ".".join(key_components) cache_key = sha384(bare_key.encode("utf8")).hexdigest() if not force_refresh: cached_content = cache.get(cache_key, None) From 2cfbaf90c3504a53135d61f9bf976bab3b388eb9 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 16 Oct 2025 14:28:13 -0300 Subject: [PATCH 047/313] ci: drop caching from build images step (#9738) --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4c70456a73e..7eac7b1c64f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -278,8 +278,6 @@ jobs: tags: | ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} - cache-from: type=gha - cache-to: type=gha,mode=max - name: Update CHANGELOG id: changelog From b0ec8c4b27d6225c6ffa6cac27ce554ec4a49a7c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Oct 2025 13:08:11 -0300 Subject: [PATCH 048/313] chore: remove unused variables (#9742) --- ietf/meeting/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 9e44df33b74..7d9e318aabc 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -956,8 +956,6 @@ class Meta: def __str__(self): return u"%s -> %s-%s" % (self.session, self.document.name, self.rev) -constraint_cache_uses = 0 -constraint_cache_initials = 0 class SessionQuerySet(models.QuerySet): def with_current_status(self): From 62f720ceaf951fba91b5a818473d798663dfbf1d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 21 Oct 2025 12:31:39 -0300 Subject: [PATCH 049/313] ci: imagePullPolicy for migration container (#9764) --- k8s/datatracker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 3d9e86a29d4..50a2c696878 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -115,6 +115,7 @@ spec: initContainers: - name: migration image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always env: - name: "CONTAINER_ROLE" value: "migrations" From a3a3d215ca4067e722ead94e886175adb589e235 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 23 Oct 2025 12:14:48 -0500 Subject: [PATCH 050/313] fix: don't limit from_contact for incoming liaison statements (#9773) --- ietf/liaisons/forms.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index ef5b29535e8..1747e55571e 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -495,14 +495,18 @@ def set_from_fields(self): self.fields['from_groups'].initial = qs # Note that the IAB chair currently doesn't get to work with incoming liaison statements - if not ( - has_role(self.user, "Secretariat") - or has_role(self.user, "Liaison Coordinator") - ): - self.fields["from_contact"].initial = ( - self.person.role_set.filter(group=qs[0]).first().email.formatted_email() - ) - self.fields["from_contact"].widget.attrs["disabled"] = True + + # Removing this block at the request of the IAB - as a workaround until the new liaison tool is + # create, anyone with access to the form can set any from_contact value + # + # if not ( + # has_role(self.user, "Secretariat") + # or has_role(self.user, "Liaison Coordinator") + # ): + # self.fields["from_contact"].initial = ( + # self.person.role_set.filter(group=qs[0]).first().email.formatted_email() + # ) + # self.fields["from_contact"].widget.attrs["disabled"] = True def set_to_fields(self): '''Set to_groups and to_contacts options and initial value based on user From 1243957f06da485e5cf4c04a8479d551817d4d78 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 23 Oct 2025 14:15:22 -0300 Subject: [PATCH 051/313] feat: unversioned proceedings cache (#9779) * feat: separate, unversioned proceedings cache * refactor: don't double-hash the cache key --- ietf/meeting/utils.py | 8 ++++---- ietf/settings.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index feadb0c7fd4..afcf7656f23 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -5,7 +5,6 @@ import jsonschema import os import requests -from hashlib import sha384 import pytz import subprocess @@ -1026,7 +1025,7 @@ def generate_proceedings_content(meeting, force_refresh=False): :meeting: meeting whose proceedings should be rendered :force_refresh: true to force regeneration and cache refresh """ - cache = caches["default"] + cache = caches["proceedings"] key_components = [ "proceedings", str(meeting.number), @@ -1061,8 +1060,9 @@ def generate_proceedings_content(meeting, force_refresh=False): ",".join(draft_names), ] - bare_key = ".".join(key_components) - cache_key = sha384(bare_key.encode("utf8")).hexdigest() + # Key is potentially long, but the "proceedings" cache hashes it to a fixed + # length. If that changes, hash it separately here first. + cache_key = ".".join(key_components) if not force_refresh: cached_content = cache.get(cache_key, None) if cached_content is not None: diff --git a/ietf/settings.py b/ietf/settings.py index 9a213c1a733..5e576430ed1 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1374,6 +1374,17 @@ def skip_unreadable_post(record): "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", "VERSION": __version__, "KEY_PREFIX": "ietf:dt", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "proceedings": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:proceedings", + # Key function is default except with sha384-encoded key "KEY_FUNCTION": lambda key, key_prefix, version: ( f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), @@ -1421,6 +1432,17 @@ def skip_unreadable_post(record): "VERSION": __version__, "KEY_PREFIX": "ietf:dt", }, + "proceedings": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + # "LOCATION": "127.0.0.1:11211", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:proceedings", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, "sessions": { "BACKEND": "django.core.cache.backends.locmem.LocMemCache", }, From 6412d1e24a9c499c39245bba58c2c31ec8110c0e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 23 Oct 2025 17:41:06 -0300 Subject: [PATCH 052/313] feat: blobstore-driven meeting materials (#9780) * feat: meeting materials blob resolver API (#9700) * refactor: exclude_deleted() for StoredObject queryset * chore: comment * feat: meeting materials blob resolver API * feat: materials blob retrieval API (#9728) * feat: materials blob retrieval API (WIP) * refactor: alphabetize ARTIFACT_STORAGE_NAMES * chore: limit buckets served * refactor: any-meeting option in _get_materials_doc() * feat: create missing blobs on retrieval * feat: render HTML from markdown via API (#9729) * chore: add comment * fix: allow bluesheets to be retrieved Normally not retrieved through /meeting/materials, but they're close enough in purpose that we might as well make them available. * fix: only update StoredObject.modified if changed * fix: preserve mtime when creating blob * refactor: better exception name * feat: render .md.html from .md blob * fix: explicit STATIC_IETF_ORG value in template Django's context_processors are not applied to render_string calls as we use them here, so settings are not available. * fix: typo * fix: decode utf-8 properly * feat: use filesystem to render .md.html * fix: copy/paste error in api_resolve_materials_name * refactor: get actual rev in _get_materials_doc (#9741) * fix: return filename, not full path * feat: precompute blob lookups for meeting materials (#9746) * feat: ResolvedMaterial model + migration * feat: method to populate ResolvedMaterial (WIP) * refactor: don't delete ResolvedMaterials Instead of deleting the ResolvedMaterials for a meeting, which might lose updates made during processing, update existing rows with any changes and warn if anything changed during the process. * fix: fix _get_materials_doc() Did not handle the possibility of multiple DocHistory objects with the same rev. * refactor: factor out material lookup helper * feat: resolve blobs via blobdb/fs for cache * chore: add resource * feat: admin for ResolvedMaterial * feat: cache-driven resolve materials API * fix: add all ResolvedMaterials; var names * fix: handle null case * feat: resolve_meeting_materials_task * feat: update resolver cache on material upload (#9759) * feat: robustness + date range for resolve materials task (#9760) * fix: limit types added to ResolvedMaterial * feat: resolve meeting materials in order by date * feat: add meetings_until param * fix: log&continue if resolving fails on a meeting * feat: log error message on parse errors * refactor: move ResolvedMaterial to blobdb app (#9762) * refactor: move ResolvedMaterial to blobdb app * fix: undo accidental removal * chore: fix lint (#9767) * fix: don't use DocHistory to find materials (#9771) * fix: don't use DocHistory to validate revs The DocHistory records are incomplete and, in particular, -00 revs are often missing. * Revert "refactor: get actual rev in _get_materials_doc (#9741)" This reverts commit 7fd15801 * chore: remove the on-demand resolver api * chore: fix lint * feat: populate materials buckets (#9777) * refactor: drop .txt from filename_with_rev() * feat: utilities to populate materials blobs * feat: store materials for a full meeting as blobs Plus a bunch of fixup from working with real data. (Based on meetings 71, 83, and 118, picked arbitrarily) * chore: update migration * feat: task to store materials in blobdb * refactor: reimplement api_retrieve_materials_blob * fix: update resolving task, fix bugs * Revert "refactor: drop .txt from filename_with_rev()" This reverts commit a849d0f92d4df54296a7062b6c3a05fb0977be93. * chore: fix lint --------- Co-authored-by: Robert Sparks --- ietf/api/urls.py | 3 + ietf/blobdb/admin.py | 11 +- .../migrations/0002_resolvedmaterial.py | 48 +++ ietf/blobdb/models.py | 20 + ietf/doc/models.py | 9 + ietf/doc/storage.py | 10 +- ietf/doc/storage_utils.py | 12 +- ietf/doc/views_material.py | 4 + ietf/meeting/resources.py | 14 +- ietf/meeting/tasks.py | 131 ++++++- ietf/meeting/utils.py | 355 +++++++++++++++++- ietf/meeting/views.py | 177 ++++++++- ietf/settings.py | 44 ++- ietf/templates/minimal.html | 4 +- 14 files changed, 798 insertions(+), 44 deletions(-) create mode 100644 ietf/blobdb/migrations/0002_resolvedmaterial.py diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 6f2efb3c1e5..04575b34cb9 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -49,6 +49,9 @@ url(r'^group/role-holder-addresses/$', api_views.role_holder_addresses), # Let IESG members set positions programmatically url(r'^iesg/position', views_ballot.api_set_position), + # Find the blob to store for a given materials document path + url(r'^meeting/(?:(?P(?:interim-)?[a-z0-9-]+)/)?materials/%(document)s(?P\.[A-Za-z0-9]+)?/resolve-cached/$' % settings.URL_REGEXPS, meeting_views.api_resolve_materials_name_cached), + url(r'^meeting/blob/(?P[a-z0-9-]+)/(?P[a-z][a-z0-9.-]+)$', meeting_views.api_retrieve_materials_blob), # Let Meetecho set session video URLs url(r'^meeting/session/video/url$', meeting_views.api_set_session_video_url), # Let Meetecho tell us the name of its recordings diff --git a/ietf/blobdb/admin.py b/ietf/blobdb/admin.py index f4cd002e070..3e1a2a311f7 100644 --- a/ietf/blobdb/admin.py +++ b/ietf/blobdb/admin.py @@ -3,7 +3,7 @@ from django.db.models.functions import Length from rangefilter.filters import DateRangeQuickSelectListFilterBuilder -from .models import Blob +from .models import Blob, ResolvedMaterial @admin.register(Blob) @@ -29,3 +29,12 @@ def get_queryset(self, request): def object_size(self, instance): """Get the size of the object""" return instance.object_size # annotation added in get_queryset() + + +@admin.register(ResolvedMaterial) +class ResolvedMaterialAdmin(admin.ModelAdmin): + model = ResolvedMaterial + list_display = ["name", "meeting_number", "bucket", "blob"] + list_filter = ["meeting_number", "bucket"] + search_fields = ["name", "blob"] + ordering = ["name"] diff --git a/ietf/blobdb/migrations/0002_resolvedmaterial.py b/ietf/blobdb/migrations/0002_resolvedmaterial.py new file mode 100644 index 00000000000..e0ab405b114 --- /dev/null +++ b/ietf/blobdb/migrations/0002_resolvedmaterial.py @@ -0,0 +1,48 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("blobdb", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="ResolvedMaterial", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(help_text="Name to resolve", max_length=300)), + ( + "meeting_number", + models.CharField( + help_text="Meeting material is related to", max_length=64 + ), + ), + ( + "bucket", + models.CharField(help_text="Resolved bucket name", max_length=255), + ), + ( + "blob", + models.CharField(help_text="Resolved blob name", max_length=300), + ), + ], + ), + migrations.AddConstraint( + model_name="resolvedmaterial", + constraint=models.UniqueConstraint( + fields=("name", "meeting_number"), name="unique_name_per_meeting" + ), + ), + ] diff --git a/ietf/blobdb/models.py b/ietf/blobdb/models.py index 8f423d9f6c4..fa7831f2032 100644 --- a/ietf/blobdb/models.py +++ b/ietf/blobdb/models.py @@ -96,3 +96,23 @@ def _emit_blob_change_event(self, using=None): ), using=using, ) + + +class ResolvedMaterial(models.Model): + # A Document name can be 255 characters; allow this name to be a bit longer + name = models.CharField(max_length=300, help_text="Name to resolve") + meeting_number = models.CharField( + max_length=64, help_text="Meeting material is related to" + ) + bucket = models.CharField(max_length=255, help_text="Resolved bucket name") + blob = models.CharField(max_length=300, help_text="Resolved blob name") + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["name", "meeting_number"], name="unique_name_per_meeting" + ) + ] + + def __str__(self): + return f"{self.name}@{self.meeting_number} -> {self.bucket}:{self.blob}" diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 25ee734cbe2..8bb79b64ed3 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -913,6 +913,7 @@ def role_for_doc(self): roles.append('Action Holder') return ', '.join(roles) +# N.B., at least a couple dozen documents exist that do not satisfy this validator validate_docname = RegexValidator( r'^[-a-z0-9]+$', "Provide a valid document name consisting of lowercase letters, numbers and hyphens.", @@ -1588,9 +1589,17 @@ class BofreqResponsibleDocEvent(DocEvent): """ Capture the responsible leadership (IAB and IESG members) for a BOF Request """ responsible = models.ManyToManyField('person.Person', blank=True) + +class StoredObjectQuerySet(models.QuerySet): + def exclude_deleted(self): + return self.filter(deleted__isnull=True) + + class StoredObject(models.Model): """Hold metadata about objects placed in object storage""" + objects = StoredObjectQuerySet.as_manager() + store = models.CharField(max_length=256) name = models.CharField(max_length=1024, null=False, blank=False) # N.B. the 1024 limit on name comes from S3 sha384 = models.CharField(max_length=96) diff --git a/ietf/doc/storage.py b/ietf/doc/storage.py index a234ef2d4fe..375620ccaf3 100644 --- a/ietf/doc/storage.py +++ b/ietf/doc/storage.py @@ -32,7 +32,7 @@ def __init__(self, file, name, mtime=None, content_type="", store=None, doc_name @classmethod def from_storedobject(cls, file, name, store): """Alternate constructor for objects that already exist in the StoredObject table""" - stored_object = StoredObject.objects.filter(store=store, name=name, deleted__isnull=True).first() + stored_object = StoredObject.objects.exclude_deleted().filter(store=store, name=name).first() if stored_object is None: raise FileNotFoundError(f"StoredObject for {store}:{name} does not exist or was deleted") file = cls(file, name, store, doc_name=stored_object.doc_name, doc_rev=stored_object.doc_rev) @@ -140,7 +140,11 @@ def _save_stored_object(self, name, content) -> StoredObject: ), ), ) - if not created: + if not created and ( + record.sha384 != content.custom_metadata["sha384"] + or record.len != int(content.custom_metadata["len"]) + or record.deleted is not None + ): record.sha384 = content.custom_metadata["sha384"] record.len = int(content.custom_metadata["len"]) record.modified = now @@ -160,7 +164,7 @@ def _delete_stored_object(self, name) -> Optional[StoredObject]: else: now = timezone.now() # Note that existing_record is a queryset that will have one matching object - existing_record.filter(deleted__isnull=True).update(deleted=now) + existing_record.exclude_deleted().update(deleted=now) return existing_record.first() def _save(self, name, content): diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index 510c98c4f55..81588c83ec3 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -12,6 +12,14 @@ from ietf.utils.log import log +class StorageUtilsError(Exception): + pass + + +class AlreadyExistsError(StorageUtilsError): + pass + + def _get_storage(kind: str) -> Storage: if kind in settings.ARTIFACT_STORAGE_NAMES: return storages[kind] @@ -70,7 +78,7 @@ def store_file( # debug.show('f"Asked to store {name} in {kind}: is_new={is_new}, allow_overwrite={allow_overwrite}"') if not allow_overwrite and not is_new: debug.show('f"Failed to save {kind}:{name} - name already exists in store"') - raise RuntimeError(f"Failed to save {kind}:{name} - name already exists in store") + raise AlreadyExistsError(f"Failed to save {kind}:{name} - name already exists in store") new_name = _get_storage(kind).save( name, StoredObjectFile( @@ -85,7 +93,7 @@ def store_file( if new_name != name: complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead." debug.show("complaint") - raise RuntimeError(complaint) + raise StorageUtilsError(complaint) except Exception as err: log(f"Blobstore Error: Failed to store file {kind}:{name}: {repr(err)}") if settings.SERVER_MODE == "development": diff --git a/ietf/doc/views_material.py b/ietf/doc/views_material.py index 6f8b8a8f122..eefac0ca61c 100644 --- a/ietf/doc/views_material.py +++ b/ietf/doc/views_material.py @@ -22,6 +22,7 @@ from ietf.doc.utils import add_state_change_event, check_common_doc_name_rules from ietf.group.models import Group from ietf.group.utils import can_manage_materials +from ietf.meeting.utils import resolve_uploaded_material from ietf.utils import log from ietf.utils.decorators import ignore_view_kwargs from ietf.utils.meetecho import MeetechoAPIError, SlidesManager @@ -179,6 +180,9 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None): "There was an error creating a hardlink at %s pointing to %s: %s" % (ftp_filepath, filepath, ex) ) + else: + for meeting in set([s.meeting for s in doc.session_set.all()]): + resolve_uploaded_material(meeting=meeting, doc=doc) if prev_rev != doc.rev: e = NewRevisionDocEvent(type="new_revision", doc=doc, rev=doc.rev) diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py index ede2b5b9939..88562a88feb 100644 --- a/ietf/meeting/resources.py +++ b/ietf/meeting/resources.py @@ -11,11 +11,15 @@ from ietf import api -from ietf.meeting.models import ( Meeting, ResourceAssociation, Constraint, Room, Schedule, Session, - TimeSlot, SchedTimeSessAssignment, SessionPresentation, FloorPlan, - UrlResource, ImportantDate, SlideSubmission, SchedulingEvent, - BusinessConstraint, ProceedingsMaterial, MeetingHost, Attended, - Registration, RegistrationTicket) +from ietf.meeting.models import (Meeting, ResourceAssociation, Constraint, Room, + Schedule, Session, + TimeSlot, SchedTimeSessAssignment, SessionPresentation, + FloorPlan, + UrlResource, ImportantDate, SlideSubmission, + SchedulingEvent, + BusinessConstraint, ProceedingsMaterial, MeetingHost, + Attended, + Registration, RegistrationTicket) from ietf.name.resources import MeetingTypeNameResource class MeetingResource(ModelResource): diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py index 784eb00d870..c361325f9ab 100644 --- a/ietf/meeting/tasks.py +++ b/ietf/meeting/tasks.py @@ -1,13 +1,20 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2025, All Rights Reserved # # Celery task definitions # +import datetime + from celery import shared_task +# from django.db.models import QuerySet from django.utils import timezone from ietf.utils import log from .models import Meeting -from .utils import generate_proceedings_content +from .utils import ( + generate_proceedings_content, + resolve_materials_for_one_meeting, + store_blobs_for_one_meeting, +) from .views import generate_agenda_data from .utils import fetch_attendance_from_meetings @@ -61,3 +68,123 @@ def fetch_meeting_attendance_task(): meeting_stats['processed'] ) ) + + +def _select_meetings( + meetings: list[str] | None = None, + meetings_since: str | None = None, + meetings_until: str | None = None +): # nyah + """Select meetings by number or date range""" + # IETF-1 = 1986-01-16 + EARLIEST_MEETING_DATE = datetime.datetime(1986, 1, 1) + meetings_since_dt: datetime.datetime | None = None + meetings_until_dt: datetime.datetime | None = None + + if meetings_since == "zero": + meetings_since_dt = EARLIEST_MEETING_DATE + elif meetings_since is not None: + try: + meetings_since_dt = datetime.datetime.fromisoformat(meetings_since) + except ValueError: + log.log( + "Failed to parse meetings_since='{meetings_since}' with fromisoformat" + ) + raise + + if meetings_until is not None: + try: + meetings_until_dt = datetime.datetime.fromisoformat(meetings_until) + except ValueError: + log.log( + "Failed to parse meetings_until='{meetings_until}' with fromisoformat" + ) + raise + if meetings_since_dt is None: + # if we only got meetings_until, start from the first meeting + meetings_since_dt = EARLIEST_MEETING_DATE + + if meetings is None: + if meetings_since_dt is None: + log.log("No meetings requested, doing nothing.") + return Meeting.objects.none() + meetings_qs = Meeting.objects.filter(date__gte=meetings_since_dt) + if meetings_until_dt is not None: + meetings_qs = meetings_qs.filter(date__lte=meetings_until_dt) + log.log( + "Selecting meetings between " + f"{meetings_since_dt} and {meetings_until_dt}" + ) + else: + log.log(f"Selecting meetings since {meetings_since_dt}") + else: + if meetings_since_dt is not None: + log.log( + "Ignoring meetings_since and meetings_until " + "because specific meetings were requested." + ) + meetings_qs = Meeting.objects.filter(number__in=meetings) + return meetings_qs + + +@shared_task +def resolve_meeting_materials_task( + *, # only allow kw arguments + meetings: list[str] | None=None, + meetings_since: str | None=None, + meetings_until: str | None=None +): + """Run materials resolver on meetings + + Can request a set of meetings by number by passing a list in the meetings arg, or + by range by passing an iso-format timestamps in meetings_since / meetings_until. + To select all meetings, set meetings_since="zero" and omit other parameters. + """ + meetings_qs = _select_meetings(meetings, meetings_since, meetings_until) + for meeting in meetings_qs.order_by("date"): + log.log( + f"Resolving materials for {meeting.type_id} " + f"meeting {meeting.number} ({meeting.date})..." + ) + mark = timezone.now() + try: + resolve_materials_for_one_meeting(meeting) + except Exception as err: + log.log( + "Exception raised while resolving materials for " + f"meeting {meeting.number}: {err}" + ) + else: + log.log(f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds.") + + +@shared_task +def store_meeting_materials_as_blobs_task( + *, # only allow kw arguments + meetings: list[str] | None = None, + meetings_since: str | None = None, + meetings_until: str | None = None +): + """Push meeting materials into the blob store + + Can request a set of meetings by number by passing a list in the meetings arg, or + by range by passing an iso-format timestamps in meetings_since / meetings_until. + To select all meetings, set meetings_since="zero" and omit other parameters. + """ + meetings_qs = _select_meetings(meetings, meetings_since, meetings_until) + for meeting in meetings_qs.order_by("date"): + log.log( + f"Creating blobs for materials for {meeting.type_id} " + f"meeting {meeting.number} ({meeting.date})..." + ) + mark = timezone.now() + try: + store_blobs_for_one_meeting(meeting) + except Exception as err: + log.log( + "Exception raised while creating blobs for " + f"meeting {meeting.number}: {err}" + ) + else: + log.log( + f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds.") diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index afcf7656f23..bdf3d3d3d3a 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -2,6 +2,9 @@ # -*- coding: utf-8 -*- import datetime import itertools +from contextlib import suppress +from dataclasses import dataclass + import jsonschema import os import requests @@ -26,16 +29,33 @@ import debug # pyflakes:ignore from ietf.dbtemplate.models import DBTemplate -from ietf.doc.storage_utils import store_bytes, store_str -from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot, - Constraint, SchedTimeSessAssignment, SessionPresentation, Attended, - Registration, Meeting, RegistrationTicket) -from ietf.doc.models import Document, State, NewRevisionDocEvent, StateDocEvent +from ietf.doc.storage_utils import store_bytes, store_str, AlreadyExistsError +from ietf.meeting.models import ( + Session, + SchedulingEvent, + TimeSlot, + Constraint, + SchedTimeSessAssignment, + SessionPresentation, + Attended, + Registration, + Meeting, + RegistrationTicket, +) +from ietf.blobdb.models import ResolvedMaterial +from ietf.doc.models import ( + Document, + State, + NewRevisionDocEvent, + StateDocEvent, + StoredObject, +) from ietf.doc.models import DocEvent from ietf.group.models import Group from ietf.group.utils import can_manage_materials from ietf.name.models import SessionStatusName, ConstraintName, DocTypeName from ietf.person.models import Person +from ietf.utils import markdown from ietf.utils.html import clean_html from ietf.utils.log import log from ietf.utils.timezone import date_today @@ -220,6 +240,7 @@ def save_bluesheet(request, session, file, encoding='utf-8'): save_error = handle_upload_file(file, filename, session.meeting, 'bluesheets', request=request, encoding=encoding) if not save_error: doc.save_with_history([e]) + resolve_uploaded_material(meeting=session.meeting, doc=doc) return save_error @@ -832,6 +853,330 @@ def write_doc_for_session(session, type_id, filename, contents): store_str(type_id, filename.name, contents) return None + +@dataclass +class BlobSpec: + bucket: str + name: str + + +def resolve_one_material( + doc: Document, rev: str | None, ext: str | None +) -> BlobSpec | None: + if doc.type_id is None: + log(f"Cannot resolve a doc with no type: {doc.name}") + return None + + # Get the Document's base name. It may or may not have an extension. + if rev is None: + basename = Path(doc.get_base_name()) + else: + basename = Path(f"{doc.name}-{int(rev):02d}") + + # If the document's file exists, the blob is _always_ named with this stem, + # even if it's different from the original. + blob_stem = Path(f"{doc.name}-{rev or doc.rev}") + + # If we have an extension, either from the URL or the Document's base name, look up + # the blob or file or return 404. N.b. the suffix check needs adjustment to handle + # a bare "." extension when we reach py3.14. + if ext or basename.suffix != "": + if ext: + blob_name = str(blob_stem.with_suffix(ext)) + else: + blob_name = str(blob_stem.with_suffix(basename.suffix)) + + # See if we have a stored object under that name + preferred_blob = ( + StoredObject.objects.exclude_deleted() + .filter(store=doc.type_id, name=blob_name) + .first() + ) + if preferred_blob is not None: + return BlobSpec( + bucket=preferred_blob.store, + name=preferred_blob.name, + ) + # No stored object, fall back to the file system. + filename = Path(doc.get_file_path()) / basename # use basename for file + if filename.is_file(): + return BlobSpec( + bucket=doc.type_id, + name=str(blob_stem.with_suffix(filename.suffix)), + ) + else: + return None + + # No extension has been specified so far, so look one up. + matching_stored_objects = ( + StoredObject.objects.exclude_deleted() + .filter( + store=doc.type_id, + name__startswith=f"{blob_stem}.", # anchor to end with trailing "." + ) + .order_by("name") + ) # orders by suffix + blob_ext_choices = { + Path(stored_obj.name).suffix: stored_obj + for stored_obj in matching_stored_objects + } + + # Short-circuit to return pdf if present + if ".pdf" in blob_ext_choices: + pdf_blob = blob_ext_choices[".pdf"] + return BlobSpec( + bucket=pdf_blob.store, + name=str(blob_stem.with_suffix(".pdf")), + ) + + # Now look for files + filename = Path(doc.get_file_path()) / basename + file_ext_choices = { + # Construct a map from suffix to full filename + fn.suffix: fn.name + for fn in sorted(filename.parent.glob(filename.stem + ".*")) + } + + # Short-circuit to return pdf if we have the file + if ".pdf" in file_ext_choices: + return BlobSpec( + bucket=doc.type_id, + name=str(blob_stem.with_suffix(".pdf")), + ) + + all_exts = set(blob_ext_choices.keys()).union(file_ext_choices.keys()) + if len(all_exts) > 0: + preferred_ext = sorted(all_exts)[0] + if preferred_ext in blob_ext_choices: + preferred_blob = blob_ext_choices[preferred_ext] + return BlobSpec( + bucket=preferred_blob.store, + name=preferred_blob.name, + ) + else: + return BlobSpec( + bucket=doc.type_id, + name=str(blob_stem.with_suffix(preferred_ext)), + ) + + return None + + +def resolve_materials_for_one_meeting(meeting: Meeting): + start_time = timezone.now() + meeting_documents = ( + Document.objects.filter( + type_id__in=settings.MATERIALS_TYPES_SERVED_BY_WORKER + ).filter( + Q(session__meeting=meeting) | Q(proceedingsmaterial__meeting=meeting) + ) + ).distinct() + + resolved = [] + for doc in meeting_documents: + # request by doc name with no rev + blob = resolve_one_material(doc, rev=None, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=doc.name, + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + # request by doc name + rev + blob = resolve_one_material(doc, rev=doc.rev, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=f"{doc.name}-{doc.rev:02}", + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + # for other revisions, only need request by doc name + rev + other_revisions = doc.revisions_by_newrevisionevent() + other_revisions.remove(doc.rev) + for rev in other_revisions: + blob = resolve_one_material(doc, rev=rev, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=f"{doc.name}-{rev:02}", + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + ResolvedMaterial.objects.bulk_create( + resolved, + update_conflicts=True, + unique_fields=["name", "meeting_number"], + update_fields=["bucket", "blob"], + ) + # Warn if any files were updated during the above process + last_update = meeting_documents.aggregate(Max("time"))["time__max"] + if last_update and last_update > start_time: + log( + f"Warning: materials for meeting {meeting.number} " + "changed during ResolvedMaterial update" + ) + +def resolve_uploaded_material(meeting: Meeting, doc: Document): + resolved = [] + blob = resolve_one_material(doc, rev=None, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=doc.name, + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + # request by doc name + rev + blob = resolve_one_material(doc, rev=doc.rev, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=f"{doc.name}-{doc.rev:02}", + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + ResolvedMaterial.objects.bulk_create( + resolved, + update_conflicts=True, + unique_fields=["name", "meeting_number"], + update_fields=["bucket", "blob"], + ) + + +def store_blob_for_one_material_file(doc: Document, rev: str, filepath: Path): + if not settings.ENABLE_BLOBSTORAGE: + raise RuntimeError("Cannot store blobs: ENABLE_BLOBSTORAGE is False") + + bucket = doc.type_id + if bucket not in settings.MATERIALS_TYPES_SERVED_BY_WORKER: + raise ValueError(f"Bucket {bucket} not found for doc {doc.name}.") + blob_stem = f"{doc.name}-{rev}" + suffix = filepath.suffix # includes leading "." + + # Store the file + try: + file_bytes = filepath.read_bytes() + except Exception as err: + log(f"Failed to read {filepath}: {err}") + raise + with suppress(AlreadyExistsError): + store_bytes( + kind=bucket, + name= blob_stem + suffix, + content=file_bytes, + mtime=datetime.datetime.fromtimestamp( + filepath.stat().st_mtime, + tz=datetime.UTC, + ), + allow_overwrite=False, + doc_name=doc.name, + doc_rev=rev, + ) + + # Special case: pre-render markdown into HTML as .md.html + if suffix == ".md": + try: + markdown_source = file_bytes.decode("utf-8") + except UnicodeDecodeError as err: + log(f"Unable to decode {filepath} as UTF-8, treating as latin-1: {err}") + markdown_source = file_bytes.decode("latin-1") + # render the markdown + try: + html = render_to_string( + "minimal.html", + { + "content": markdown.markdown(markdown_source), + "title": blob_stem, + "static_ietf_org": settings.STATIC_IETF_ORG, + }, + ) + except Exception as err: + log(f"Failed to render markdown for {filepath}: {err}") + else: + # Don't overwrite, but don't fail if the blob exists + with suppress(AlreadyExistsError): + store_str( + kind=bucket, + name=blob_stem + ".md.html", + content=html, + allow_overwrite=False, + doc_name=doc.name, + doc_rev=rev, + content_type="text/html;charset=utf-8", + ) + + +def store_blobs_for_one_material_doc(doc: Document): + """Ensure that all files related to a materials Document are in the blob store""" + if doc.type_id not in settings.MATERIALS_TYPES_SERVED_BY_WORKER: + log(f"This method does not handle docs of type {doc.name}") + return + + # Store files for current Document / rev + file_path = Path(doc.get_file_path()) + base_name = Path(doc.get_base_name()) + # .stem would remove directories, so use .with_suffix("") + base_name_stem = str(base_name.with_suffix("")) + if base_name_stem.endswith(".") and base_name.suffix == "": + # In Python 3.14, a trailing "." is a valid suffix, but in prior versions + # it is left as part of the stem. The suffix check ensures that either way, + # only a single "." will be removed. + base_name_stem = base_name_stem[:-1] + # Add any we find without the rev + for file_to_store in file_path.glob(base_name_stem + ".*"): + if not (file_to_store.is_file()): + continue + try: + store_blob_for_one_material_file(doc, doc.rev, file_to_store) + except Exception as err: + log( + f"Failed to store blob for {doc} rev {doc.rev} " + f"from {file_to_store}: {err}" + ) + + # Get other revisions + for rev in doc.revisions_by_newrevisionevent(): + if rev == doc.rev: + continue # already handled this + + # Add some that have the rev + for file_to_store in file_path.glob(doc.name + f"-{rev}.*"): + if not file_to_store.is_file(): + continue + try: + store_blob_for_one_material_file(doc, rev, file_to_store) + except Exception as err: + log( + f"Failed to store blob for {doc} rev {rev} " + f"from {file_to_store}: {err}" + ) + + +def store_blobs_for_one_meeting(meeting: Meeting): + meeting_documents = ( + Document.objects.filter( + type_id__in=settings.MATERIALS_TYPES_SERVED_BY_WORKER + ).filter( + Q(session__meeting=meeting) | Q(proceedingsmaterial__meeting=meeting) + ) + ).distinct() + + for doc in meeting_documents: + store_blobs_for_one_material_doc(doc) + + def create_recording(session, url, title=None, user=None): ''' Creates the Document type=recording, setting external_url and creating diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index fcc9312609f..cf6fed596b0 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -9,6 +9,7 @@ import json import math import os + import pytz import re import tarfile @@ -27,10 +28,12 @@ from django import forms from django.core.cache import caches +from django.core.files.storage import storages from django.shortcuts import render, redirect, get_object_or_404 from django.http import (HttpResponse, HttpResponseRedirect, HttpResponseForbidden, HttpResponseNotFound, Http404, HttpResponseBadRequest, - JsonResponse, HttpResponseGone, HttpResponseNotAllowed) + JsonResponse, HttpResponseGone, HttpResponseNotAllowed, + FileResponse) from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required @@ -48,18 +51,25 @@ from django.views.decorators.cache import cache_page from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt from django.views.generic import RedirectView +from rest_framework.status import HTTP_404_NOT_FOUND import debug # pyflakes:ignore from ietf.doc.fields import SearchableDocumentsField from ietf.doc.models import Document, State, DocEvent, NewRevisionDocEvent -from ietf.doc.storage_utils import remove_from_storage, retrieve_bytes, store_file +from ietf.doc.storage_utils import ( + remove_from_storage, + retrieve_bytes, + store_file, +) from ietf.group.models import Group from ietf.group.utils import can_manage_session_materials, can_manage_some_groups, can_manage_group from ietf.person.models import Person, User from ietf.ietfauth.utils import role_required, has_role, user_is_person from ietf.mailtrigger.utils import gather_address_lists -from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, SessionPresentation, TimeSlot, SlideSubmission, Attended +from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, \ + SessionPresentation, TimeSlot, SlideSubmission, Attended +from ..blobdb.models import ResolvedMaterial from ietf.meeting.models import ImportantDate, SessionStatusName, SchedulingEvent, SchedTimeSessAssignment, Room, TimeSlotTypeName from ietf.meeting.models import Registration from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm, ImportMinutesForm, @@ -83,7 +93,8 @@ finalize, generate_proceedings_content, organize_proceedings_sessions, - sort_accept_tuple, + resolve_uploaded_material, + sort_accept_tuple, store_blobs_for_one_material_doc, ) from ietf.meeting.utils import add_event_info_to_session_qs from ietf.meeting.utils import session_time_for_sorting @@ -120,6 +131,8 @@ from icalendar import Calendar, Event from ietf.doc.templatetags.ietf_filters import absurl +from ..api.ietf_utils import requires_api_token +from ..blobdb.storage import BlobdbStorage, BlobFile request_summary_exclude_group_types = ['team'] @@ -245,21 +258,32 @@ def current_materials(request): raise Http404('No such meeting') -def _get_materials_doc(meeting, name): +def _get_materials_doc(name, meeting=None): """Get meeting materials document named by name - Raises Document.DoesNotExist if a match cannot be found. + Raises Document.DoesNotExist if a match cannot be found. If meeting is None, + matches a name that is associated with _any_ meeting. """ + + def _matches_meeting(doc, meeting=None): + if meeting is None: + return doc.get_related_meeting() is not None + return doc.get_related_meeting() == meeting + # try an exact match first doc = Document.objects.filter(name=name).first() - if doc is not None and doc.get_related_meeting() == meeting: + if doc is not None and _matches_meeting(doc, meeting): return doc, None + # try parsing a rev number if "-" in name: docname, rev = name.rsplit("-", 1) if len(rev) == 2 and rev.isdigit(): doc = Document.objects.get(name=docname) # may raise Document.DoesNotExist - if doc.get_related_meeting() == meeting and rev in doc.revisions_by_newrevisionevent(): + if ( + _matches_meeting(doc, meeting) + and rev in doc.revisions_by_newrevisionevent() + ): return doc, rev # give up raise Document.DoesNotExist @@ -277,7 +301,7 @@ def materials_document(request, document, num=None, ext=None): meeting = get_meeting(num, type_in=["ietf", "interim"]) num = meeting.number try: - doc, rev = _get_materials_doc(meeting=meeting, name=document) + doc, rev = _get_materials_doc(name=document, meeting=meeting) except Document.DoesNotExist: raise Http404("No such document for meeting %s" % num) @@ -320,6 +344,7 @@ def materials_document(request, document, num=None, ext=None): { "content": markdown.markdown(bytes.decode(encoding=chset)), "title": filename.name, + "static_ietf_org": settings.STATIC_IETF_ORG, }, ) content_type = content_type.replace("plain", "html", 1) @@ -334,6 +359,133 @@ def materials_document(request, document, num=None, ext=None): return HttpResponseRedirect(redirect_to=doc.get_href(meeting=meeting)) +@requires_api_token("ietf.meeting.views.api_resolve_materials_name") +def api_resolve_materials_name_cached(request, document, num=None, ext=None): + """Resolve materials name into document to a blob spec + + Returns the bucket/name of a blob in the blob store that corresponds to the named + document. Handles resolution of revision if it is not specified and determines the + best extension if one is not provided. Response is JSON. + + As of 2025-10-10 we do not have blobs for all materials documents or for every + format of every document. This API still returns the bucket/name as if the blob + exists. Another API will allow the caller to obtain the file contents using that + name if it cannot be retrieved from the blob store. + """ + + def _error_response(status: int, detail: str): + return JsonResponse( + { + "status": status, + "title": "Error", + "detail": detail, + }, + status=status, + ) + + def _response(bucket: str, name: str): + return JsonResponse( + { + "bucket": bucket, + "name": name, + } + ) + + try: + resolved = ResolvedMaterial.objects.get( + meeting_number=num, name=document + ) + except ResolvedMaterial.DoesNotExist: + return _error_response( + HTTP_404_NOT_FOUND, f"No suitable file for {document} for meeting {num}" + ) + return _response(bucket=resolved.bucket, name=resolved.blob) + + +@requires_api_token +def api_retrieve_materials_blob(request, bucket, name): + """Retrieve contents of a meeting materials blob + + This is intended as a fallback if the web worker cannot retrieve a blob from + the blobstore itself. The most likely cause is retrieving an old materials document + that has not been backfilled. + + If a blob is requested that does not exist, this checks for it on the filesystem + and if found, adds it to the blobstore, creates a StoredObject record, and returns + the contents as it would have done if the blob was already present. + + As a special case, if a requested file with extension `.md.html` does not exist + but a file with the same name but extension `.md` does, `.md` file will be rendered + from markdown to html and returned / stored. + """ + DEFAULT_CONTENT_TYPES = { + ".html": "text/html;charset=utf-8", + ".md": "text/markdown;charset=utf-8", + ".pdf": "application/pdf", + ".txt": "text/plain;charset=utf-8", + } + + def _default_content_type(blob_name: str): + return DEFAULT_CONTENT_TYPES.get(Path(name).suffix, "application/octet-stream") + + if not ( + settings.ENABLE_BLOBSTORAGE + and bucket in settings.MATERIALS_TYPES_SERVED_BY_WORKER + ): + return HttpResponseNotFound(f"Bucket {bucket} not found.") + storage = storages[bucket] # if not configured, a server error will result + assert isinstance(storage, BlobdbStorage) + try: + blob = storage.open(name, "rb") + except FileNotFoundError: + pass + else: + # found the blob - return it + assert isinstance(blob, BlobFile) + return FileResponse( + blob, + filename=name, + content_type=blob.content_type or _default_content_type(name), + ) + + # Did not find the blob. Create it if we can + name_as_path = Path(name) + if name_as_path.suffixes == [".md", ".html"]: + # special case: .md.html means we want to create the .md and the .md.html + # will come along as a bonus + name_to_store = name_as_path.stem # removes the .html + else: + name_to_store = name + + # See if we have a meeting-related document that matches the requested bucket and + # name. + try: + doc, rev = _get_materials_doc(Path(name_to_store).stem) + if doc.type_id != bucket: + raise Document.DoesNotExist + except Document.DoesNotExist: + return HttpResponseNotFound( + f"Document corresponding to {bucket}:{name} not found." + ) + else: + # create all missing blobs for the doc while we're at it + store_blobs_for_one_material_doc(doc) + + # If we can make the blob at all, it now exists, so return it or a 404 + try: + blob = storage.open(name, "rb") + except FileNotFoundError: + return HttpResponseNotFound(f"Object {bucket}:{name} not found.") + else: + # found the blob - return it + assert isinstance(blob, BlobFile) + return FileResponse( + blob, + filename=name, + content_type=blob.content_type or _default_content_type(name), + ) + + @login_required def materials_editable_groups(request, num=None): meeting = get_meeting(num) @@ -2949,6 +3101,7 @@ def upload_session_minutes(request, session_id, num): form.add_error(None, str(err)) else: # no exception -- success! + resolve_uploaded_material(meeting=session.meeting, doc=session.minutes()) messages.success(request, f'Successfully uploaded minutes as revision {session.minutes().rev}.') return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) else: @@ -3008,6 +3161,7 @@ def upload_session_narrativeminutes(request, session_id, num): form.add_error(None, str(err)) else: # no exception -- success! + resolve_uploaded_material(meeting=session.meeting, doc=session.narrative_minutes()) messages.success(request, f'Successfully uploaded narrative minutes as revision {session.narrative_minutes().rev}.') return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) else: @@ -3154,6 +3308,7 @@ def upload_session_agenda(request, session_id, num): form.add_error(None, save_error) else: doc.save_with_history([e]) + resolve_uploaded_material(meeting=session.meeting, doc=doc) messages.success(request, f'Successfully uploaded agenda as revision {doc.rev}.') return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym) else: @@ -3337,6 +3492,7 @@ def upload_session_slides(request, session_id, num, name=None): else: doc.save_with_history([e]) post_process(doc) + resolve_uploaded_material(meeting=session.meeting, doc=doc) # Send MeetEcho updates even if we had a problem saving - that will keep it in sync with the # SessionPresentation, which was already saved regardless of problems saving the file. @@ -4737,6 +4893,7 @@ def err(code, text): write_doc_for_session(session, 'chatlog', filename, json.dumps(apidata['chatlog'])) e = NewRevisionDocEvent.objects.create(doc=doc, rev=doc.rev, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev) doc.save_with_history([e]) + resolve_uploaded_material(meeting=session.meeting, doc=doc) return HttpResponse( "Done", status=200, @@ -4785,6 +4942,7 @@ def err(code, text): write_doc_for_session(session, 'polls', filename, json.dumps(apidata['polls'])) e = NewRevisionDocEvent.objects.create(doc=doc, rev=doc.rev, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev) doc.save_with_history([e]) + resolve_uploaded_material(meeting=session.meeting, doc=doc) return HttpResponse( "Done", status=200, @@ -5167,6 +5325,7 @@ def approve_proposed_slides(request, slidesubmission_id, num): doc.store_bytes(target_filename, retrieve_bytes("staging", submission.filename)) remove_from_storage("staging", submission.filename) post_process(doc) + resolve_uploaded_material(meeting=submission.session.meeting, doc=doc) DocEvent.objects.create(type="approved_slides", doc=doc, rev=doc.rev, by=request.user.person, desc="Slides approved") # update meetecho slide info if configured diff --git a/ietf/settings.py b/ietf/settings.py index 5e576430ed1..eb5f9d2161e 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -786,29 +786,29 @@ def skip_unreadable_post(record): # Storages for artifacts stored as blobs ARTIFACT_STORAGE_NAMES: list[str] = [ - "bofreq", - "charter", - "conflrev", "active-draft", - "draft", - "slides", - "minutes", "agenda", + "bibxml-ids", "bluesheets", - "procmaterials", - "narrativeminutes", - "statement", - "statchg", - "liai-att", + "bofreq", + "charter", "chatlog", - "polls", - "staging", - "bibxml-ids", - "indexes", + "conflrev", + "draft", "floorplan", + "indexes", + "liai-att", "meetinghostlogo", + "minutes", + "narrativeminutes", "photo", + "polls", + "procmaterials", "review", + "slides", + "staging", + "statchg", + "statement", ] for storagename in ARTIFACT_STORAGE_NAMES: STORAGES[storagename] = { @@ -816,6 +816,20 @@ def skip_unreadable_post(record): "OPTIONS": {"bucket_name": storagename}, } +# Buckets / doc types of meeting materials the CF worker is allowed to serve. This +# differs from the list in Session.meeting_related() by the omission of "recording" +MATERIALS_TYPES_SERVED_BY_WORKER = [ + "agenda", + "bluesheets", + "chatlog", + "minutes", + "narrativeminutes", + "polls", + "procmaterials", + "slides", +] + + # Override this in settings_local.py if needed # *_PATH variables ends with a slash/ . diff --git a/ietf/templates/minimal.html b/ietf/templates/minimal.html index 87f661f501f..15c432505ef 100644 --- a/ietf/templates/minimal.html +++ b/ietf/templates/minimal.html @@ -9,8 +9,8 @@ {{ title }} - - + + {# load this in the head, to prevent flickering #} From af0bcc743f6e449f93e0c7a7e4f2e2eec3ec76ae Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 23 Oct 2025 17:14:39 -0400 Subject: [PATCH 053/313] docs: Update PostgreSQL version badge in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e1b7e1a454..dfaf8710520 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Python Version](https://img.shields.io/badge/python-3.9-blue?logo=python&logoColor=white)](#prerequisites) [![Django Version](https://img.shields.io/badge/django-4.x-51be95?logo=django&logoColor=white)](#prerequisites) [![Node Version](https://img.shields.io/badge/node.js-16.x-green?logo=node.js&logoColor=white)](#prerequisites) -[![MariaDB Version](https://img.shields.io/badge/postgres-16-blue?logo=postgresql&logoColor=white)](#prerequisites) +[![MariaDB Version](https://img.shields.io/badge/postgres-17-blue?logo=postgresql&logoColor=white)](#prerequisites) ##### The day-to-day front-end to the IETF database for people who work on IETF standards. From f9dea7df9d562ba818cf9224c1594f0e0983cdbe Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 23 Oct 2025 17:24:58 -0400 Subject: [PATCH 054/313] docs: Update Python version badge to 3.12 in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dfaf8710520..baffc311e7e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![Release](https://img.shields.io/github/release/ietf-tools/datatracker.svg?style=flat&maxAge=300)](https://github.com/ietf-tools/datatracker/releases) [![License](https://img.shields.io/github/license/ietf-tools/datatracker)](https://github.com/ietf-tools/datatracker/blob/main/LICENSE) [![Code Coverage](https://codecov.io/gh/ietf-tools/datatracker/branch/feat/bs5/graph/badge.svg?token=V4DXB0Q28C)](https://codecov.io/gh/ietf-tools/datatracker) -[![Python Version](https://img.shields.io/badge/python-3.9-blue?logo=python&logoColor=white)](#prerequisites) +[![Python Version](https://img.shields.io/badge/python-3.12-blue?logo=python&logoColor=white)](#prerequisites) [![Django Version](https://img.shields.io/badge/django-4.x-51be95?logo=django&logoColor=white)](#prerequisites) [![Node Version](https://img.shields.io/badge/node.js-16.x-green?logo=node.js&logoColor=white)](#prerequisites) [![MariaDB Version](https://img.shields.io/badge/postgres-17-blue?logo=postgresql&logoColor=white)](#prerequisites) From e0691c17121d2324d812bc68c3943d963d1c5d4d Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 23 Oct 2025 17:30:50 -0400 Subject: [PATCH 055/313] ci: remove assets rsync sync job from dev-assets-sync-nightly workflow Removed the nightly sync job for assets in the workflow. --- .github/workflows/dev-assets-sync-nightly.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index 19933bddfdd..4cfbf6365b4 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -47,17 +47,3 @@ jobs: file: dev/shared-assets-sync/Dockerfile push: true tags: ghcr.io/ietf-tools/datatracker-rsync-assets:latest - - sync: - name: Run assets rsync - if: ${{ always() }} - runs-on: [self-hosted, dev-server] - needs: [build] - steps: - - name: Run rsync - env: - DEBIAN_FRONTEND: noninteractive - run: | - docker pull ghcr.io/ietf-tools/datatracker-rsync-assets:latest - docker run --rm -v dt-assets:/assets ghcr.io/ietf-tools/datatracker-rsync-assets:latest - docker image prune -a -f From 354d83d2fa22f817384a792bcbdef9757771f70a Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 23 Oct 2025 17:34:00 -0400 Subject: [PATCH 056/313] ci: remove sandbox-refresh workflow --- .github/workflows/sandbox-refresh.yml | 35 --------------------------- 1 file changed, 35 deletions(-) delete mode 100644 .github/workflows/sandbox-refresh.yml diff --git a/.github/workflows/sandbox-refresh.yml b/.github/workflows/sandbox-refresh.yml deleted file mode 100644 index 3ddb119e4fa..00000000000 --- a/.github/workflows/sandbox-refresh.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Sandbox Refresh - -on: - # Run every night - schedule: - - cron: '0 9 * * *' - - workflow_dispatch: - -jobs: - main: - name: Refresh DBs - runs-on: [self-hosted, dev-server] - permissions: - contents: read - - steps: - - uses: actions/checkout@v4 - - - name: Refresh DBs - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Install Deploy to Container CLI dependencies..." - cd dev/deploy-to-container - npm ci - cd ../.. - echo "Start Refresh..." - node ./dev/deploy-to-container/refresh.js - - - name: Cleanup old docker resources - env: - DEBIAN_FRONTEND: noninteractive - run: | - docker image prune -a -f From 4e6168607cb49abc9341b27049f458bc9363297a Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 23 Oct 2025 20:43:04 -0300 Subject: [PATCH 057/313] ci: proceedings cache cfg for prod/tests (#9784) --- ietf/settings_testcrawl.py | 4 +++- k8s/settings_local.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ietf/settings_testcrawl.py b/ietf/settings_testcrawl.py index a1b5ce8946d..40744a228df 100644 --- a/ietf/settings_testcrawl.py +++ b/ietf/settings_testcrawl.py @@ -27,9 +27,11 @@ 'MAX_ENTRIES': 10000, }, }, + 'proceedings': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, 'sessions': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - # No version-specific VERSION setting. }, 'htmlized': { 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', diff --git a/k8s/settings_local.py b/k8s/settings_local.py index c09bd70c869..f8ffacc83fc 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -301,6 +301,17 @@ def _multiline_to_list(s): "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", "VERSION": __version__, "KEY_PREFIX": "ietf:dt", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "proceedings": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:proceedings", + # Key function is default except with sha384-encoded key "KEY_FUNCTION": lambda key, key_prefix, version: ( f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), From 6db7d4afbe2b876192d0aa4a63a0bbe98a3806be Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Oct 2025 20:06:53 -0300 Subject: [PATCH 058/313] fix: don't trust libmagic charset recognition (#9815) --- ietf/meeting/views.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index cf6fed596b0..d6b5a1c0db5 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -329,7 +329,7 @@ def materials_document(request, document, num=None, ext=None): old_proceedings_format = meeting.number.isdigit() and int(meeting.number) <= 96 if settings.MEETING_MATERIALS_SERVE_LOCALLY or old_proceedings_format: bytes = filename.read_bytes() - mtype, chset = get_mime_type(bytes) + mtype, chset = get_mime_type(bytes) # chset does not consider entire file! content_type = "%s; charset=%s" % (mtype, chset) if filename.suffix == ".md" and mtype == "text/plain": @@ -339,15 +339,24 @@ def materials_document(request, document, num=None, ext=None): content_type = content_type.replace("plain", "markdown", 1) break elif atype[0] == "text/html": + # Render markdown, allowing that charset may be inaccurate. + try: + md_src = bytes.decode( + "utf-8" if chset in ["ascii", "us-ascii"] else chset + ) + except UnicodeDecodeError: + # latin-1, aka iso8859-1, accepts all 8-bit code points + md_src = bytes.decode("latin-1") + content = markdown.markdown(md_src) # a string bytes = render_to_string( "minimal.html", { - "content": markdown.markdown(bytes.decode(encoding=chset)), + "content": content, "title": filename.name, "static_ietf_org": settings.STATIC_IETF_ORG, }, - ) - content_type = content_type.replace("plain", "html", 1) + ).encode("utf-8") + content_type = "text/html; charset=utf-8" break elif atype[0] == "text/plain": break From 3e34efe74950d7f237171e9ea5cedc24d8d08615 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Oct 2025 20:09:27 -0300 Subject: [PATCH 059/313] chore: update names fixture (#9807) * chore(dev): update names fixture * chore(dev): update names fixture again --- ietf/name/fixtures/names.json | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 58deb01f0ca..64e26e503ad 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -650,7 +650,7 @@ }, { "fields": { - "desc": "4.2.1. Call for Adoption by WG Issued\r\n\r\n The \"Call for Adoption by WG Issued\" state should be used to indicate when an I-D is being considered for adoption by an IETF WG. An I-D that is in this state is actively being considered for adoption and has not yet achieved consensus, preference, or selection in the WG.\r\n\r\n This state may be used to describe an I-D that someone has asked a WG to consider for adoption, if the WG Chair has agreed with the request. This state may also be used to identify an I-D that a WG Chair asked an author to write specifically for consideration as a candidate WG item [WGDTSPEC], and/or an I-D that is listed as a 'candidate draft' in the WG's charter.\r\n\r\n Under normal conditions, it should not be possible for an I-D to be in the \"Call for Adoption by WG Issued\" state in more than one working group at the same time. This said, it is not uncommon for authors to \"shop\" their I-Ds to more than one WG at a time, with the hope of getting their documents adopted somewhere.\r\n\r\n After this state is implemented in the Datatracker, an I-D that is in the \"Call for Adoption by WG Issued\" state will not be able to be \"shopped\" to any other WG without the consent of the WG Chairs and the responsible ADs impacted by the shopping.\r\n\r\n Note that Figure 1 includes an arc leading from this state to outside of the WG state machine. This illustrates that some I-Ds that are considered do not get adopted as WG drafts. An I-D that is not adopted as a WG draft will transition out of the WG state machine and revert back to having no stream-specific state; however, the status change history log of the I-D will record that the I-D was previously in the \"Call for Adoption by WG Issued\" state.", + "desc": "A call for adoption of the individual submission document has been issued by the Working Group (WG) chairs. This call is still running but the WG has not yet reached consensus for adoption.", "name": "Call For Adoption By WG Issued", "next_states": [ 36, @@ -666,7 +666,7 @@ }, { "fields": { - "desc": "4.2.2. Adopted by a WG\r\n\r\n The \"Adopted by a WG\" state describes an individual submission I-D that an IETF WG has agreed to adopt as one of its WG drafts.\r\n\r\n WG Chairs who use this state will be able to clearly indicate when their WGs adopt individual submission I-Ds. This will facilitate the Datatracker's ability to correctly capture \"Replaces\" information for WG drafts and correct \"Replaced by\" information for individual submission I-Ds that have been replaced by WG drafts.\r\n\r\n This state is needed because the Datatracker uses the filename of an I-D as a key to search its database for status information about the I-D, and because the filename of a WG I-D is supposed to be different from the filename of an individual submission I-D. The filename of an individual submission I-D will typically be formatted as 'draft-author-wgname-topic-nn'.\r\n\r\n The filename of a WG document is supposed to be formatted as 'draft- ietf-wgname-topic-nn'.\r\n\r\n An individual I-D that is adopted by a WG may take weeks or months to be resubmitted by the author as a new (version-00) WG draft. If the \"Adopted by a WG\" state is not used, the Datatracker has no way to determine that an I-D has been adopted until a new version of the I-D is submitted to the WG by the author and until the I-D is approved for posting by a WG Chair.", + "desc": "The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted.", "name": "Adopted by a WG", "next_states": [ 38 @@ -681,7 +681,7 @@ }, { "fields": { - "desc": "4.2.3. Adopted for WG Info Only\r\n\r\n The \"Adopted for WG Info Only\" state describes a document that contains useful information for the WG that adopted it, but the document is not intended to be published as an RFC. The WG will not actively develop the contents of the I-D or progress it for publication as an RFC. The only purpose of the I-D is to provide information for internal use by the WG.", + "desc": "The document is adopted by the Working Group (WG) for its internal use. The WG has decided that it will not pursue publication of it as an RFC.", "name": "Adopted for WG Info Only", "next_states": [], "order": 3, @@ -694,7 +694,7 @@ }, { "fields": { - "desc": "4.2.4. WG Document\r\n\r\n The \"WG Document\" state describes an I-D that has been adopted by an IETF WG and is being actively developed.\r\n\r\n A WG Chair may transition an I-D into the \"WG Document\" state at any time as long as the I-D is not being considered or developed in any other WG.\r\n\r\n Alternatively, WG Chairs may rely upon new functionality to be added to the Datatracker to automatically move version-00 drafts into the \"WG Document\" state as described in Section 4.1.\r\n\r\n Under normal conditions, it should not be possible for an I-D to be in the \"WG Document\" state in more than one WG at a time. This said, I-Ds may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs.", + "desc": "The document has been adopted by the Working Group (WG) and is under development. A document can only be adopted by one WG at a time. However, a document may be transferred between WGs.", "name": "WG Document", "next_states": [ 39, @@ -712,7 +712,7 @@ }, { "fields": { - "desc": "4.2.5. Parked WG Document\r\n\r\n A \"Parked WG Document\" is an I-D that has lost its author or editor, is waiting for another document to be written or for a review to be completed, or cannot be progressed by the working group for some other reason.\r\n\r\n Some of the annotation tags described in Section 4.3 may be used in conjunction with this state to indicate why an I-D has been parked, and/or what may need to happen for the I-D to be un-parked.\r\n\r\n Parking a WG draft will not prevent it from expiring; however, this state can be used to indicate why the I-D has stopped progressing in the WG.\r\n\r\n A \"Parked WG Document\" that is not expired may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs.", + "desc": "The Working Group (WG) document is in a temporary state where it will not be actively developed. The reason for the pause is explained via a datatracker comments section.", "name": "Parked WG Document", "next_states": [ 38 @@ -727,7 +727,7 @@ }, { "fields": { - "desc": "4.2.6. Dead WG Document\r\n\r\n A \"Dead WG Document\" is an I-D that has been abandoned. Note that 'Dead' is not always a final state for a WG I-D. If consensus is subsequently achieved, a \"Dead WG Document\" may be resurrected. A \"Dead WG Document\" that is not resurrected will eventually expire.\r\n\r\n Note that an I-D that is declared to be \"Dead\" in one WG and that is not expired may be transferred to a non-dead state in another WG with the consent of the WG Chairs and the responsible ADs.", + "desc": "The Working Group (WG) document has been abandoned by the WG. No further development is planned in this WG. A decision to resume work on this document and move it out of this state is possible.", "name": "Dead WG Document", "next_states": [ 38 @@ -742,7 +742,7 @@ }, { "fields": { - "desc": "4.2.7. In WG Last Call\r\n\r\n A document \"In WG Last Call\" is an I-D for which a WG Last Call (WGLC) has been issued and is in progress.\r\n\r\n Note that conducting a WGLC is an optional part of the IETF WG process, per Section 7.4 of RFC 2418 [RFC2418].\r\n\r\n If a WG Chair decides to conduct a WGLC on an I-D, the \"In WG Last Call\" state can be used to track the progress of the WGLC. The Chair may configure the Datatracker to send a WGLC message to one or more mailing lists when the Chair moves the I-D into this state. The WG Chair may also be able to select a different set of mailing lists for a different document undergoing a WGLC; some documents may deserve coordination with other WGs.\r\n\r\n A WG I-D in this state should remain \"In WG Last Call\" until the WG Chair moves it to another state. The WG Chair may configure the Datatracker to send an e-mail after a specified period of time to remind or 'nudge' the Chair to conclude the WGLC and to determine the next state for the document.\r\n\r\n It is possible for one WGLC to lead into another WGLC for the same document. For example, an I-D that completed a WGLC as an \"Informational\" document may need another WGLC if a decision is taken to convert the I-D into a Standards Track document.", + "desc": "The Working Group (WG) document is currently subject to an active WG Last Call (WGLC) review per Section 7.4 of RFC2418.", "name": "In WG Last Call", "next_states": [ 38, @@ -759,7 +759,7 @@ }, { "fields": { - "desc": "4.2.8. Waiting for WG Chair Go-Ahead\r\n\r\n A WG Chair may wish to place an I-D that receives a lot of comments during a WGLC into the \"Waiting for WG Chair Go-Ahead\" state. This state describes an I-D that has undergone a WGLC; however, the Chair is not yet ready to call consensus on the document.\r\n\r\n If comments from the WGLC need to be responded to, or a revision to the I-D is needed, the Chair may place an I-D into this state until all of the WGLC comments are adequately addressed and the (possibly revised) document is in the I-D repository.", + "desc": "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed", "name": "Waiting for WG Chair Go-Ahead", "next_states": [ 41, @@ -775,7 +775,7 @@ }, { "fields": { - "desc": "4.2.9. WG Consensus: Waiting for Writeup\r\n\r\n A document in the \"WG Consensus: Waiting for Writeup\" state has essentially completed its development within the working group, and is nearly ready to be sent to the IESG for publication. The last thing to be done is the preparation of a protocol writeup by a Document Shepherd. The IESG requires that a document shepherd writeup be completed before publication of the I-D is requested. The IETF document shepherding process and the role of a WG Document Shepherd is described in RFC 4858 [RFC4858]\r\n\r\n A WG Chair may call consensus on an I-D without a formal WGLC and transition an I-D that was in the \"WG Document\" state directly into this state.\r\n\r\n The name of this state includes the words \"Waiting for Writeup\" because a good document shepherd writeup takes time to prepare.", + "desc": "The Working Group (WG) document has consensus to proceed to publication. However, the document is waiting for a document shepherd write-up per RFC4858.", "name": "WG Consensus: Waiting for Write-Up", "next_states": [ 44 @@ -790,7 +790,7 @@ }, { "fields": { - "desc": "4.2.10. Submitted to IESG for Publication\r\n\r\n This state describes a WG document that has been submitted to the IESG for publication and that has not been sent back to the working group for revision.\r\n\r\n An I-D in this state may be under review by the IESG, it may have been approved and be in the RFC Editor's queue, or it may have been published as an RFC. Other possibilities exist too. The document may be \"Dead\" (in the IESG state machine) or in a \"Do Not Publish\" state.", + "desc": "The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", "name": "Submitted to IESG for Publication", "next_states": [ 38 @@ -2020,7 +2020,7 @@ }, { "fields": { - "desc": "The document has been marked as a candidate for WG adoption by the WG Chair. This state can be used before a call for adoption is issued (and the document is put in the \"Call For Adoption By WG Issued\" state), to indicate that the document is in the queue for a call for adoption, even if none has been issued yet.", + "desc": "The individual submission document has been marked by the Working Group (WG) chairs as a candidate for adoption by the WG, but no adoption call has been started.", "name": "Candidate for WG Adoption", "next_states": [ 35 @@ -2152,7 +2152,7 @@ }, { "fields": { - "desc": "In some areas, it can be desirable to wait for multiple interoperable implementations before progressing a draft to be an RFC, and in some WGs this is required. This state should be entered after WG Last Call has completed.", + "desc": "The progression of this Working Group (WG) document towards publication is paused as it awaits implementation. The process governing the approach to implementations is WG-specific.", "name": "Waiting for Implementation", "next_states": [], "order": 8, @@ -2165,7 +2165,7 @@ }, { "fields": { - "desc": "Held by WG, see document history for details.", + "desc": "Held by Working Group (WG) chairs for administrative reasons. See document history for details.", "name": "Held by WG", "next_states": [], "order": 9, @@ -4473,6 +4473,7 @@ ], "session_purposes": [ "coding", + "open_meeting", "presentation", "social", "tutorial" @@ -5535,7 +5536,6 @@ ], "desc": "Recipients for a message when a new incoming liaison statement is posted", "to": [ - "liaison_from_contact", "liaison_to_contacts" ] }, From 145b9f76c19030b67628432b5f811a1c3c55c749 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 28 Oct 2025 20:11:52 -0300 Subject: [PATCH 060/313] chore(dev): bump dev blobdb to pg17 (#9806) --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8c6e0ea4866..2440faf1214 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -116,7 +116,7 @@ services: - "minio-data:/data" blobdb: - image: postgres:16 + image: postgres:17 restart: unless-stopped environment: POSTGRES_DB: blob From cbb0e2e3db4cc9e591b4397b7bc6cdebb51cfc8c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 29 Oct 2025 11:18:47 -0300 Subject: [PATCH 061/313] feat: logs in api_retrieve_materials_blob() (#9818) --- ietf/meeting/views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index d6b5a1c0db5..69635d62194 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -451,6 +451,7 @@ def _default_content_type(blob_name: str): else: # found the blob - return it assert isinstance(blob, BlobFile) + log(f"Materials blob: directly returning {bucket}:{name}") return FileResponse( blob, filename=name, @@ -473,17 +474,20 @@ def _default_content_type(blob_name: str): if doc.type_id != bucket: raise Document.DoesNotExist except Document.DoesNotExist: + log(f"Materials blob: no doc for {bucket}:{name}") return HttpResponseNotFound( f"Document corresponding to {bucket}:{name} not found." ) else: # create all missing blobs for the doc while we're at it + log(f"Materials blob: storing blobs for {doc.name}-{doc.rev}") store_blobs_for_one_material_doc(doc) # If we can make the blob at all, it now exists, so return it or a 404 try: blob = storage.open(name, "rb") except FileNotFoundError: + log(f"Materials blob: no blob for {bucket}:{name}") return HttpResponseNotFound(f"Object {bucket}:{name} not found.") else: # found the blob - return it From c47fe34b0e409f4811e2f96fc45ec87bc1b7931f Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 3 Nov 2025 09:05:30 -0500 Subject: [PATCH 062/313] fix: include punctuation when tablesorting (#9855) --- ietf/static/js/list.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/static/js/list.js b/ietf/static/js/list.js index 756a75001a8..c03368cd728 100644 --- a/ietf/static/js/list.js +++ b/ietf/static/js/list.js @@ -16,7 +16,7 @@ function text_sort(a, b, options) { // sort by text content return prep(a, options).localeCompare(prep(b, options), "en", { sensitivity: "base", - ignorePunctuation: true, + ignorePunctuation: false, numeric: true }); } From 87c3a9db06b784d2cf1484a547171a9783e50fdc Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Mon, 3 Nov 2025 09:08:53 -0500 Subject: [PATCH 063/313] feat(agenda): Show calendar links to all the events (#9843) * feat(agenda): Show calendar links to all the events * test: Update playwright tests --- client/agenda/AgendaScheduleList.vue | 20 ++++++++++---------- playwright/tests/meeting/agenda.spec.js | 7 ++++++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue index fc8b5fd30f7..bbe5dfee8b7 100644 --- a/client/agenda/AgendaScheduleList.vue +++ b/client/agenda/AgendaScheduleList.vue @@ -398,16 +398,6 @@ const meetingEvents = computed(() => { color: 'teal' }) } - // -> Calendar item - if (item.links.calendar) { - links.push({ - id: `lnk-${item.id}-calendar`, - label: 'Calendar (.ics) entry for this session', - icon: 'calendar-check', - href: item.links.calendar, - color: 'pink' - }) - } } else { // -> Post event if (meetingNumberInt >= 60) { @@ -484,6 +474,16 @@ const meetingEvents = computed(() => { } } } + // Add Calendar item for all events that has a calendar link + if (item.adjustedEnd > current && item.links.calendar) { + links.push({ + id: `lnk-${item.id}-calendar`, + label: 'Calendar (.ics) entry for this session', + icon: 'calendar-check', + href: item.links.calendar, + color: 'pink' + }) + } // Event icon let icon = null diff --git a/playwright/tests/meeting/agenda.spec.js b/playwright/tests/meeting/agenda.spec.js index 412a3fe9b8c..2248027a38a 100644 --- a/playwright/tests/meeting/agenda.spec.js +++ b/playwright/tests/meeting/agenda.spec.js @@ -1219,7 +1219,12 @@ test.describe('future - desktop', () => { await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar > i.bi`)).toBeVisible() } } else { - await expect(eventButtons).toHaveCount(0) + if (event.links.calendar) { + await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar`)).toHaveAttribute('href', event.links.calendar) + await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar > i.bi`)).toBeVisible() + } else { + await expect(eventButtons).toHaveCount(0) + } } } } From 8da45cb8488345a1f449e6fc7442098cff81e3ff Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 3 Nov 2025 09:10:59 -0500 Subject: [PATCH 064/313] feat: optionally hide room-only schedule diffs (#9861) * feat: optionally hide room-only schedule diffs * test: update test --- ietf/meeting/tests_views.py | 194 +++++++++++++++++++++++++----------- ietf/meeting/views.py | 13 +++ 2 files changed, 151 insertions(+), 56 deletions(-) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index b1bbc629071..50960b51439 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -49,7 +49,11 @@ from ietf.meeting.helpers import send_interim_minutes_reminder, populate_important_dates, update_important_dates from ietf.meeting.models import Session, TimeSlot, Meeting, SchedTimeSessAssignment, Schedule, SessionPresentation, SlideSubmission, SchedulingEvent, Room, Constraint, ConstraintName from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting, make_interim_test_data -from ietf.meeting.utils import condition_slide_order, generate_proceedings_content +from ietf.meeting.utils import ( + condition_slide_order, + generate_proceedings_content, + diff_meeting_schedules, +) from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting from ietf.meeting.utils import create_recording, delete_recording, get_next_sequence, bluesheet_data from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule @@ -4765,73 +4769,151 @@ def test_list_schedules(self): self.assertTrue(r.status_code, 200) def test_diff_schedules(self): - meeting = make_meeting_test_data() - - url = urlreverse('ietf.meeting.views.diff_schedules',kwargs={'num':meeting.number}) - login_testing_unauthorized(self,"secretary", url) - r = self.client.get(url) - self.assertTrue(r.status_code, 200) - - from_schedule = Schedule.objects.get(meeting=meeting, name="test-unofficial-schedule") - - session1 = Session.objects.filter(meeting=meeting, group__acronym='mars').first() - session2 = Session.objects.filter(meeting=meeting, group__acronym='ames').first() - session3 = SessionFactory(meeting=meeting, group=Group.objects.get(acronym='mars'), - attendees=10, requested_duration=datetime.timedelta(minutes=70), - add_to_schedule=False) - SchedulingEvent.objects.create(session=session3, status_id='schedw', by=Person.objects.first()) - - slot2 = TimeSlot.objects.filter(meeting=meeting, type='regular').order_by('-time').first() - slot3 = TimeSlot.objects.create( - meeting=meeting, type_id='regular', location=slot2.location, - duration=datetime.timedelta(minutes=60), - time=slot2.time + datetime.timedelta(minutes=60), + # Create meeting and some time slots + meeting = MeetingFactory(type_id="ietf", populate_schedule=False) + rooms = RoomFactory.create_batch(2, meeting=meeting) + # first index is room, second is time + timeslots = [ + [ + TimeSlotFactory( + location=room, + meeting=meeting, + time=datetime.datetime.combine( + meeting.date, datetime.time(9, 0, tzinfo=datetime.UTC) + ) + ), + TimeSlotFactory( + location=room, + meeting=meeting, + time=datetime.datetime.combine( + meeting.date, datetime.time(10, 0, tzinfo=datetime.UTC) + ) + ), + TimeSlotFactory( + location=room, + meeting=meeting, + time=datetime.datetime.combine( + meeting.date, datetime.time(11, 0, tzinfo=datetime.UTC) + ) + ), + ] + for room in rooms + ] + sessions = SessionFactory.create_batch( + 5, meeting=meeting, add_to_schedule=False ) - # copy - new_url = urlreverse("ietf.meeting.views.new_meeting_schedule", kwargs=dict(num=meeting.number, owner=from_schedule.owner_email(), name=from_schedule.name)) - r = self.client.post(new_url, { - 'name': "newtest", - 'public': "on", - }) - self.assertNoFormPostErrors(r) + from_schedule = ScheduleFactory(meeting=meeting) + to_schedule = ScheduleFactory(meeting=meeting) - to_schedule = Schedule.objects.get(meeting=meeting, name='newtest') + # sessions[0]: not scheduled in from_schedule, scheduled in to_schedule + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[0], + timeslot=timeslots[0][0], + ) + # sessions[1]: scheduled in from_schedule, not scheduled in to_schedule + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[1], + timeslot=timeslots[0][0], + ) + # sessions[2]: moves rooms, not time + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[2], + timeslot=timeslots[0][1], + ) + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[2], + timeslot=timeslots[1][1], + ) + # sessions[3]: moves time, not room + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[3], + timeslot=timeslots[1][1], + ) + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[3], + timeslot=timeslots[1][2], + ) + # sessions[4]: moves room and time + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[4], + timeslot=timeslots[1][0], + ) + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[4], + timeslot=timeslots[0][2], + ) - # make some changes + # Check the raw diffs + raw_diffs = diff_meeting_schedules(from_schedule, to_schedule) + self.assertCountEqual( + raw_diffs, + [ + { + "change": "schedule", + "session": sessions[0].pk, + "to": timeslots[0][0].pk, + }, + { + "change": "unschedule", + "session": sessions[1].pk, + "from": timeslots[0][0].pk, + }, + { + "change": "move", + "session": sessions[2].pk, + "from": timeslots[0][1].pk, + "to": timeslots[1][1].pk, + }, + { + "change": "move", + "session": sessions[3].pk, + "from": timeslots[1][1].pk, + "to": timeslots[1][2].pk, + }, + { + "change": "move", + "session": sessions[4].pk, + "from": timeslots[1][0].pk, + "to": timeslots[0][2].pk, + }, + ] + ) - edit_url = urlreverse("ietf.meeting.views.edit_meeting_schedule", kwargs=dict(num=meeting.number, owner=to_schedule.owner_email(), name=to_schedule.name)) + # Check the view + url = urlreverse("ietf.meeting.views.diff_schedules", + kwargs={"num": meeting.number}) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertTrue(r.status_code, 200) - # schedule session - r = self.client.post(edit_url, { - 'action': 'assign', - 'timeslot': slot3.pk, - 'session': session3.pk, - }) - self.assertEqual(json.loads(r.content)['success'], True) - # unschedule session - r = self.client.post(edit_url, { - 'action': 'unassign', - 'session': session1.pk, - }) - self.assertEqual(json.loads(r.content)['success'], True) - # move session - r = self.client.post(edit_url, { - 'action': 'assign', - 'timeslot': slot2.pk, - 'session': session2.pk, + # with show room changes disabled - does not show sessions[2] because it did + # not change time + r = self.client.get(url, { + "from_schedule": from_schedule.name, + "to_schedule": to_schedule.name, }) - self.assertEqual(json.loads(r.content)['success'], True) + self.assertTrue(r.status_code, 200) + q = PyQuery(r.content) + self.assertEqual(len(q(".schedule-diffs tr")), 4 + 1) - # now get differences + # with show room changes enabled - shows all changes r = self.client.get(url, { - 'from_schedule': from_schedule.name, - 'to_schedule': to_schedule.name, + "from_schedule": from_schedule.name, + "to_schedule": to_schedule.name, + "show_room_changes": "on", }) self.assertTrue(r.status_code, 200) - q = PyQuery(r.content) - self.assertEqual(len(q(".schedule-diffs tr")), 3+1) + self.assertEqual(len(q(".schedule-diffs tr")), 5 + 1) def test_delete_schedule(self): url = urlreverse('ietf.meeting.views.delete_schedule', diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 69635d62194..b0c46cb05ad 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -1675,6 +1675,11 @@ def list_schedules(request, num): class DiffSchedulesForm(forms.Form): from_schedule = forms.ChoiceField() to_schedule = forms.ChoiceField() + show_room_changes = forms.BooleanField( + initial=False, + required=False, + help_text="Include changes to room without a date or time change", + ) def __init__(self, meeting, user, *args, **kwargs): super().__init__(*args, **kwargs) @@ -1707,6 +1712,14 @@ def diff_schedules(request, num): raw_diffs = diff_meeting_schedules(from_schedule, to_schedule) diffs = prefetch_schedule_diff_objects(raw_diffs) + if not form.cleaned_data["show_room_changes"]: + # filter out room-only changes + diffs = [ + d + for d in diffs + if (d["change"] != "move") or (d["from"].time != d["to"].time) + ] + for d in diffs: s = d['session'] s.session_label = s.short_name From 9546e15224df7d8d9f385a8f670cd27012d7aee5 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 3 Nov 2025 09:11:32 -0500 Subject: [PATCH 065/313] fix: no autoescape for bluesheet template (#9858) --- ietf/templates/meeting/bluesheet.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ietf/templates/meeting/bluesheet.txt b/ietf/templates/meeting/bluesheet.txt index dd3bf36ac7c..5b3960f3aa7 100644 --- a/ietf/templates/meeting/bluesheet.txt +++ b/ietf/templates/meeting/bluesheet.txt @@ -1,7 +1,8 @@ -Bluesheet for {{session}} +{% autoescape off %}Bluesheet for {{session}} ======================================================================== {{ data|length }} attendees. {% for item in data %} {{ item.name }} {{ item.affiliation }}{% endfor %} +{% endautoescape %} From 7b4035d7fcd1130cdf8e08b3aa54efda35087a8a Mon Sep 17 00:00:00 2001 From: Tero Kivinen Date: Mon, 3 Nov 2025 18:16:33 +0200 Subject: [PATCH 066/313] fix: Change add period button to save new period. (#9847) --- ietf/templates/group/change_reviewer_settings.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/templates/group/change_reviewer_settings.html b/ietf/templates/group/change_reviewer_settings.html index 9ecec5633cf..75451fdd750 100644 --- a/ietf/templates/group/change_reviewer_settings.html +++ b/ietf/templates/group/change_reviewer_settings.html @@ -89,7 +89,7 @@

    Unavailable periods

    + value="add_period">Save new period

    History of settings

    From 1ba63977c00121572048c506289f88d41ce67291 Mon Sep 17 00:00:00 2001 From: Matthew Holloway Date: Tue, 4 Nov 2025 06:26:25 +1300 Subject: [PATCH 067/313] fix: ask google not to index noscript content (#9844) --- ietf/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/templates/base.html b/ietf/templates/base.html index aa449555276..d8ff85f86e9 100644 --- a/ietf/templates/base.html +++ b/ietf/templates/base.html @@ -96,7 +96,7 @@ {% endif %}
    -

    +

    + The not-prepped XML + is the RFC XML v3 source for an RFC at the moment in the publication process + just before the prep tool was used to expand default + values, generate section numbers, resolve cross-references, and embed + boilerplate. +

    + It is useful for authors who want to begin a new draft based on + the RFC's text, such as when creating a bis-draft, and for tools that process + author-facing RFC XML. +

    +

    + + + Download not-prepped XML for RFC {{ rfc.rfc_number }} + +

    +{% endblock %} From 20480d6242254693b3dbcbf9b73380b5b3c838cb Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 16 Apr 2026 13:59:23 -0500 Subject: [PATCH 221/313] fix: force notprepped downloads (#10719) --- ietf/doc/{tests_unprepped.py => tests_notprepped.py} | 10 +++++++--- ietf/doc/views_doc.py | 5 +++-- 2 files changed, 10 insertions(+), 5 deletions(-) rename ietf/doc/{tests_unprepped.py => tests_notprepped.py} (92%) diff --git a/ietf/doc/tests_unprepped.py b/ietf/doc/tests_notprepped.py similarity index 92% rename from ietf/doc/tests_unprepped.py rename to ietf/doc/tests_notprepped.py index f88af8e81a6..f417aa79310 100644 --- a/ietf/doc/tests_unprepped.py +++ b/ietf/doc/tests_notprepped.py @@ -12,7 +12,7 @@ from ietf.utils.test_utils import TestCase -class UnpreppedRfcXmlTests(TestCase): +class NotpreppedRfcXmlTests(TestCase): def test_editor_source_button_visibility(self): pre_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC - 1) first_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC) @@ -72,13 +72,17 @@ def test_rfcxml_notprepped(self): r = self.client.get(url) self.assertEqual(r.status_code, 404) - # 200 with correct content-type and body when object is fully stored + # 200 with correct content-type, attachment disposition, and body when object is fully stored xml_content = b"test" store_bytes("rfc", stored_name, xml_content, allow_overwrite=True) r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertEqual(r["Content-Type"], "application/xml") - self.assertEqual(r.content, xml_content) + self.assertEqual( + r["Content-Disposition"], + f'attachment; filename="rfc{number}.notprepped.xml"', + ) + self.assertEqual(b"".join(r.streaming_content), xml_content) def test_rfcxml_notprepped_wrapper(self): number = settings.FIRST_V3_RFC diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index a23185333ed..5b57a620743 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -43,9 +43,10 @@ from celery.result import AsyncResult from django.core.cache import caches +from django.core.files.base import ContentFile from django.core.exceptions import PermissionDenied from django.db.models import Max -from django.http import HttpResponse, Http404, HttpResponseBadRequest, JsonResponse +from django.http import FileResponse, HttpResponse, Http404, HttpResponseBadRequest, JsonResponse from django.shortcuts import render, get_object_or_404, redirect from django.template.loader import render_to_string from django.urls import reverse as urlreverse @@ -2372,7 +2373,7 @@ def rfcxml_notprepped(request, number): bytes = retrieve_bytes("rfc", name) except FileNotFoundError: raise Http404 - return HttpResponse(bytes, content_type="application/xml") + return FileResponse(ContentFile(bytes, name=f"rfc{number}.notprepped.xml"), as_attachment=True) def rfcxml_notprepped_wrapper(request, number): From 9cecc36bc7e42ecc5cd196d96f4bd0eaf03b5e69 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Apr 2026 00:25:02 -0300 Subject: [PATCH 222/313] feat: rebuild_searchindex task (#10723) * refactor: DRY * chore: typesense docker container (commented out) * feat: batched RFC search index import * feat: rebuild_searchindex task * feat: logging / error reporting * refactor: _task suffix for task name * test: tests for searchindex utils + tasks * fix: only create collection if dropped * fix: typing / lint --- docker-compose.yml | 12 ++ ietf/doc/tasks.py | 11 ++ ietf/doc/tests_tasks.py | 43 ++++++ ietf/utils/searchindex.py | 239 ++++++++++++++++++++++++++++++-- ietf/utils/tests_searchindex.py | 152 +++++++++++++++----- 5 files changed, 410 insertions(+), 47 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4c3f2f6b8ec..073d04b896a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -132,6 +132,18 @@ services: volumes: - blobdb-data:/var/lib/postgresql/data +# typesense: +# image: typesense/typesense:30.1 +# restart: on-failure +# ports: +# - "8108:8108" +# volumes: +# - ./typesense-data:/data +# command: +# - '--data-dir=/data' +# - '--api-key=typesense-api-key' +# - '--enable-cors' + # Celery Beat is a periodic task runner. It is not normally needed for development, # but can be enabled by uncommenting the following. # diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 19edb390149..273242e35f5 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -209,3 +209,14 @@ def update_rfc_searchindex_task(self, rfc_number: int): countdown=searchindex_settings["TASK_RETRY_DELAY"], max_retries=searchindex_settings["TASK_MAX_RETRIES"], ) + + +@shared_task +def rebuild_searchindex_task(*, batchsize=40, drop_collection=False): + if drop_collection: + searchindex.delete_collection() + searchindex.create_collection() + searchindex.update_or_create_rfc_entries( + Document.objects.filter(type_id="rfc").order_by("-rfc_number"), + batchsize=batchsize, + ) diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 728d21f131f..2e2d65463fe 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -24,6 +24,7 @@ generate_idnits2_rfc_status_task, investigate_fragment_task, notify_expirations_task, + rebuild_searchindex_task, update_rfc_searchindex_task, ) @@ -144,6 +145,48 @@ def test_update_rfc_searchindex_task( with self.assertRaises(Retry): update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + @mock.patch("ietf.doc.tasks.searchindex.update_or_create_rfc_entries") + @mock.patch("ietf.doc.tasks.searchindex.create_collection") + @mock.patch("ietf.doc.tasks.searchindex.delete_collection") + def test_rebuild_searchindex_task(self, mock_delete, mock_create, mock_update): + rfcs = WgRfcFactory.create_batch(10) + rebuild_searchindex_task() + self.assertFalse(mock_delete.called) + self.assertFalse(mock_create.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + + mock_delete.reset_mock() + mock_create.reset_mock() + mock_update.reset_mock() + rebuild_searchindex_task(drop_collection=True) + self.assertTrue(mock_delete.called) + self.assertTrue(mock_create.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + + mock_delete.reset_mock() + mock_create.reset_mock() + mock_update.reset_mock() + rebuild_searchindex_task(drop_collection=True, batchsize=3) + self.assertTrue(mock_delete.called) + self.assertTrue(mock_create.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + self.assertEqual(mock_update.call_args.kwargs["batchsize"], 3) + class Idnits2SupportTests(TestCase): settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [ diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py index e4427b88b53..a47e6d2f120 100644 --- a/ietf/utils/searchindex.py +++ b/ietf/utils/searchindex.py @@ -2,12 +2,15 @@ """Search indexing utilities""" import re +from itertools import batched from math import floor +from typing import Iterable import httpx # just for exceptions import typesense import typesense.exceptions from django.conf import settings +from typesense.types.document import DocumentSchema from ietf.doc.models import Document, StoredObject from ietf.doc.storage_utils import retrieve_str @@ -42,6 +45,24 @@ def enabled(): return _settings["TYPESENSE_API_URL"] != "" +def get_typesense_client() -> typesense.Client: + _settings = get_settings() + client = typesense.Client( + { + "api_key": _settings["TYPESENSE_API_KEY"], + "nodes": [_settings["TYPESENSE_API_URL"]], + } + ) + return client + + +def get_collection_name() -> str: + _settings = get_settings() + collection_name = _settings["TYPESENSE_COLLECTION_NAME"] + assert isinstance(collection_name, str) + return collection_name + + def _sanitize_text(content): """Sanitize content or abstract text for search""" # REs (with approximate names) @@ -62,7 +83,7 @@ def _sanitize_text(content): return content.strip() -def update_or_create_rfc_entry(rfc: Document): +def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: assert rfc.type_id == "rfc" assert rfc.rfc_number is not None @@ -75,8 +96,8 @@ def update_or_create_rfc_entry(rfc: Document): f"Indexing as {subseries[0].name}" ) subseries = subseries[0] if len(subseries) > 0 else None - obsoleted_by = rfc.relations_that("obs") - updated_by = rfc.relations_that("updates") + obsoleted_by = rfc.related_that("obs") + updated_by = rfc.related_that("updates") stored_txt = ( StoredObject.objects.exclude_deleted() @@ -91,8 +112,8 @@ def update_or_create_rfc_entry(rfc: Document): except Exception as err: log(f"Unable to retrieve {stored_txt} from storage: {err}") - ts_id = f"doc-{rfc.pk}" ts_document = { + "id": f"doc-{rfc.pk}", "rfcNumber": rfc.rfc_number, "rfc": str(rfc.rfc_number), "filename": rfc.name, @@ -143,13 +164,205 @@ def update_or_create_rfc_entry(rfc: Document): ts_document["adName"] = rfc.ad.name if content != "": ts_document["content"] = _sanitize_text(content) - _settings = get_settings() - client = typesense.Client( + return ts_document + + +def update_or_create_rfc_entry(rfc: Document): + """Update/create index entries for one RFC""" + ts_document = typesense_doc_from_rfc(rfc) + client = get_typesense_client() + client.collections[get_collection_name()].documents.upsert(ts_document) + + +def update_or_create_rfc_entries( + rfcs: Iterable[Document], batchsize: int | None = None +): + """Update/create index entries for RFCs in bulk + + If batchsize is set, computes index data in batches of batchsize and adds to the + index. Will make a total of (len(rfcs) // batchsize) + 1 API calls. + + N.b. that typesense has a server-side batch size that defaults to 40, which should + "almost never be changed from the default." This does not change that. Further, + the python client library's import_ method has a batch_size parameter that does + client-side batching. We don't use that, either. + """ + success_count = 0 + fail_count = 0 + client = get_typesense_client() + batches = [rfcs] if batchsize is None else batched(rfcs, batchsize) + for batch in batches: + tdoc_batch = [typesense_doc_from_rfc(rfc) for rfc in batch] + results = client.collections[get_collection_name()].documents.import_( + tdoc_batch, {"action": "upsert"} + ) + for tdoc, result in zip(tdoc_batch, results): + if result["success"]: + success_count += 1 + else: + fail_count += 1 + log(f"Failed to index RFC {tdoc['rfcNumber']}: {result['error']}") + log(f"Added {success_count} RFCs to the index, failed to add {fail_count}") + + +DOCS_SCHEMA = { + "enable_nested_fields": True, + "default_sorting_field": "ranking", + "fields": [ + # RFC number in integer form, for sorting asc/desc in search results + # Omit field for drafts { - "api_key": _settings["TYPESENSE_API_KEY"], - "nodes": [_settings["TYPESENSE_API_URL"]], - } - ) - client.collections[_settings["TYPESENSE_COLLECTION_NAME"]].documents.upsert( - {"id": ts_id} | ts_document - ) + "name": "rfcNumber", + "type": "int32", + "facet": False, + "optional": True, + "sort": True, + }, + # RFC number in string form, for direct matching with ranking + # Omit field for drafts + {"name": "rfc", "type": "string", "facet": False, "optional": True}, + # For drafts that correspond to an RFC, insert the RFC number + # Omit field for rfcs or if not relevant + {"name": "ref", "type": "string", "facet": False, "optional": True}, + # Filename of the document (without the extension, e.g. "rfc1234" + # or "draft-ietf-abc-def-02") + {"name": "filename", "type": "string", "facet": False, "infix": True}, + # Title of the draft / rfc + {"name": "title", "type": "string", "facet": False}, + # Abstract of the draft / rfc + {"name": "abstract", "type": "string", "facet": False}, + # A list of search keywords if relevant, set to empty array otherwise + {"name": "keywords", "type": "string[]", "facet": True}, + # Type of the document + # Accepted values: "draft" or "rfc" + {"name": "type", "type": "string", "facet": True}, + # State(s) of the document (e.g. "Published", "Adopted by a WG", etc.) + # Use the full name, not the slug + {"name": "state", "type": "string[]", "facet": True, "optional": True}, + # Status (Standard Level Name) + # Object with properties "slug" and "name" + # e.g.: { slug: "std", "name": "Internet Standard" } + {"name": "status", "type": "object", "facet": True, "optional": True}, + # The subseries it is part of. (e.g. "BCP") + # Omit otherwise. + { + "name": "subseries.acronym", + "type": "string", + "facet": True, + "optional": True, + }, + # The subseries number it is part of. (e.g. 123) + # Omit otherwise. + { + "name": "subseries.number", + "type": "int32", + "facet": True, + "sort": True, + "optional": True, + }, + # The total of RFCs in the subseries + # Omit if not part of a subseries + { + "name": "subseries.total", + "type": "int32", + "facet": False, + "sort": False, + "optional": True, + }, + # Date of the document, in unix epoch seconds (can be negative for < 1970) + {"name": "date", "type": "int64", "facet": False}, + # Expiration date of the document, in unix epoch seconds (can be negative + # for < 1970). Omit field for RFCs + {"name": "expires", "type": "int64", "facet": False, "optional": True}, + # Publication date of the RFC, in unix epoch seconds (can be negative + # for < 1970). Omit field for drafts + { + "name": "publicationDate", + "type": "int64", + "facet": True, + "optional": True, + }, + # Working Group + # Object with properties "acronym", "name" and "full" + # e.g.: + # { + # "acronym": "ntp", + # "name": "Network Time Protocols", + # "full": "ntp - Network Time Protocols", + # } + {"name": "group", "type": "object", "facet": True, "optional": True}, + # Area + # Object with properties "acronym", "name" and "full" + # e.g.: + # { + # "acronym": "mpls", + # "name": "Multiprotocol Label Switching", + # "full": "mpls - Multiprotocol Label Switching", + # } + {"name": "area", "type": "object", "facet": True, "optional": True}, + # Stream + # Object with properties "slug" and "name" + # e.g.: { slug: "ietf", "name": "IETF" } + {"name": "stream", "type": "object", "facet": True, "optional": True}, + # List of authors + # Array of objects with properties "name" and "affiliation" + # e.g.: + # [ + # {"name": "John Doe", "affiliation": "ACME Inc."}, + # {"name": "Ada Lovelace", "affiliation": "Babbage Corps."}, + # ] + {"name": "authors", "type": "object[]", "facet": True, "optional": True}, + # Area Director Name (e.g. "Leonardo DaVinci") + {"name": "adName", "type": "string", "facet": True, "optional": True}, + # Whether the document should be hidden by default in search results or not. + {"name": "flags.hiddenDefault", "type": "bool", "facet": True}, + # Whether the document is obsoleted by another document or not. + {"name": "flags.obsoleted", "type": "bool", "facet": True}, + # Whether the document is updated by another document or not. + {"name": "flags.updated", "type": "bool", "facet": True}, + # List of documents that obsolete this document. + # Array of strings. Use RFC number for RFCs. (e.g. ["123", "456"]) + # Omit if none. Must be provided if "flags.obsoleted" is set to True. + { + "name": "obsoletedBy", + "type": "string[]", + "facet": False, + "optional": True, + }, + # List of documents that update this document. + # Array of strings. Use RFC number for RFCs. (e.g. ["123", "456"]) + # Omit if none. Must be provided if "flags.updated" is set to True. + {"name": "updatedBy", "type": "string[]", "facet": False, "optional": True}, + # Sanitized content of the document. + # Make sure to remove newlines, double whitespaces, symbols and tags. + { + "name": "content", + "type": "string", + "facet": False, + "optional": True, + "store": False, + }, + # Ranking value to use when no explicit sorting is used during search + # Set to the RFC number for RFCs and the revision number for drafts + # This ensures newer RFCs get listed first in the default search results + # (without a query) + {"name": "ranking", "type": "int32", "facet": False}, + ], +} + + +def create_collection(): + collection_name = get_collection_name() + log(f"Creating '{collection_name}' collection") + client = get_typesense_client() + client.collections.create({"name": get_collection_name()} | DOCS_SCHEMA) + + +def delete_collection(): + collection_name = get_collection_name() + log(f"Deleting '{collection_name}' collection") + client = get_typesense_client() + try: + client.collections[collection_name].delete() + except typesense.exceptions.ObjectNotFound: + pass diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py index 8740716c85f..0bff96ec7dc 100644 --- a/ietf/utils/tests_searchindex.py +++ b/ietf/utils/tests_searchindex.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2026, All Rights Reserved from unittest import mock +import typesense.exceptions from django.conf import settings from django.test.utils import override_settings @@ -51,42 +52,29 @@ def test_sanitize_text(self): "TYPESENSE_COLLECTION_NAME": "frogs", } ) - @mock.patch("ietf.utils.searchindex.typesense.Client") - def test_update_or_create_rfc_entry(self, mock_ts_client_constructor): + def test_typesense_doc_from_rfc(self): not_rfc = WgDraftFactory() assert isinstance(not_rfc, Document) with self.assertRaises(AssertionError): - searchindex.update_or_create_rfc_entry(not_rfc) - self.assertFalse(mock_ts_client_constructor.called) + searchindex.typesense_doc_from_rfc(not_rfc) invalid_rfc = WgRfcFactory(name="rfc1000000", rfc_number=None) assert isinstance(invalid_rfc, Document) with self.assertRaises(AssertionError): - searchindex.update_or_create_rfc_entry(invalid_rfc) - self.assertFalse(mock_ts_client_constructor.called) + searchindex.typesense_doc_from_rfc(invalid_rfc) rfc = PublishedRfcDocEventFactory().doc assert isinstance(rfc, Document) - searchindex.update_or_create_rfc_entry(rfc) - self.assertTrue(mock_ts_client_constructor.called) - # walk the tree down to the method we expected to be called... - mock_upsert = mock_ts_client_constructor.return_value.collections[ - "frogs" - ].documents.upsert # matches value in override_settings above - self.assertTrue(mock_upsert.called) - upserted_dict = mock_upsert.call_args[0][0] + result = searchindex.typesense_doc_from_rfc(rfc) # Check a few values, not exhaustive - self.assertEqual(upserted_dict["id"], f"doc-{rfc.pk}") - self.assertEqual(upserted_dict["rfcNumber"], rfc.rfc_number) - self.assertEqual( - upserted_dict["abstract"], searchindex._sanitize_text(rfc.abstract) - ) - self.assertNotIn("adName", upserted_dict) - self.assertNotIn("content", upserted_dict) # no blob - self.assertNotIn("subseries", upserted_dict) + self.assertEqual(result["id"], f"doc-{rfc.pk}") + self.assertEqual(result["rfcNumber"], rfc.rfc_number) + self.assertEqual(result["abstract"], searchindex._sanitize_text(rfc.abstract)) + self.assertNotIn("adName", result) + self.assertNotIn("content", result) # no blob + self.assertNotIn("subseries", result) # repeat, this time with contents, an AD, and subseries docs - mock_upsert.reset_mock() store_str( kind="rfc", name=f"txt/{rfc.name}.txt", @@ -99,17 +87,15 @@ def test_update_or_create_rfc_entry(self, mock_ts_client_constructor): # (the typesense schema does not support this for real at the moment) BcpFactory(contains=[rfc], name="bcp1234") StdFactory(contains=[rfc], name="std1234") - searchindex.update_or_create_rfc_entry(rfc) - self.assertTrue(mock_upsert.called) - upserted_dict = mock_upsert.call_args[0][0] + result = searchindex.typesense_doc_from_rfc(rfc) # Check a few values, not exhaustive self.assertEqual( - upserted_dict["content"], + result["content"], searchindex._sanitize_text("The contents of this RFC"), ) - self.assertEqual(upserted_dict["adName"], "Alfred D. Rector") - self.assertIn("subseries", upserted_dict) - ss_dict = upserted_dict["subseries"] + self.assertEqual(result["adName"], "Alfred D. Rector") + self.assertIn("subseries", result) + ss_dict = result["subseries"] # We should get one of the two subseries docs, but neither is more correct # than the other... self.assertTrue( @@ -119,10 +105,108 @@ def test_update_or_create_rfc_entry(self, mock_ts_client_constructor): ) ) - # Finally, delete the contents blob and make sure things don't blow up - mock_upsert.reset_mock() + # Finally, delete the contents blob and make sure things don't blow up Blob.objects.get(bucket="rfc", name=f"txt/{rfc.name}.txt").delete() + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertNotIn("content", result) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense_doc_from_rfc") + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_update_or_create_rfc_entry( + self, mock_ts_client_constructor, mock_tdoc_from_rfc + ): + fake_tdoc = object() + mock_tdoc_from_rfc.return_value = fake_tdoc + rfc = WgRfcFactory() + assert isinstance(rfc, Document) searchindex.update_or_create_rfc_entry(rfc) + self.assertTrue(mock_ts_client_constructor.called) + # walk the tree down to the method we expected to be called... + mock_upsert = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.upsert self.assertTrue(mock_upsert.called) - upserted_dict = mock_upsert.call_args[0][0] - self.assertNotIn("content", upserted_dict) + self.assertEqual(mock_upsert.call_args, mock.call(fake_tdoc)) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense_doc_from_rfc") + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_update_or_create_rfc_entries( + self, mock_ts_client_constructor, mock_tdoc_from_rfc + ): + fake_tdoc = object() + mock_tdoc_from_rfc.return_value = fake_tdoc + rfc = WgRfcFactory() + assert isinstance(rfc, Document) + searchindex.update_or_create_rfc_entries([rfc] * 50) # list of docs... + self.assertEqual(mock_ts_client_constructor.call_count, 1) + # walk the tree down to the method we expected to be called... + mock_import_ = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.import_ + self.assertEqual(mock_import_.call_count, 1) + self.assertEqual( + mock_import_.call_args, mock.call([fake_tdoc] * 50, {"action": "upsert"}) + ) + + mock_import_.reset_mock() + searchindex.update_or_create_rfc_entries([rfc] * 50, batchsize=20) + self.assertEqual(mock_ts_client_constructor.call_count, 2) # one more + # walk the tree down to the method we expected to be called... + mock_import_ = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.import_ + self.assertEqual(mock_import_.call_count, 3) + self.assertEqual( + mock_import_.call_args_list, + [ + mock.call([fake_tdoc] * 20, {"action": "upsert"}), + mock.call([fake_tdoc] * 20, {"action": "upsert"}), + mock.call([fake_tdoc] * 10, {"action": "upsert"}), + ], + ) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_create_collection(self, mock_ts_client_constructor): + searchindex.create_collection() + self.assertEqual(mock_ts_client_constructor.call_count, 1) + mock_collections = mock_ts_client_constructor.return_value.collections + self.assertTrue(mock_collections.create.called) + self.assertEqual(mock_collections.create.call_args[0][0]["name"], "frogs") + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_delete_collection(self, mock_ts_client_constructor): + searchindex.delete_collection() + self.assertEqual(mock_ts_client_constructor.call_count, 1) + mock_collections = mock_ts_client_constructor.return_value.collections + self.assertTrue(mock_collections["frogs"].delete.called) + + mock_collections["frogs"].side_effect = typesense.exceptions.ObjectNotFound + searchindex.delete_collection() # should ignore the exception From c4cb8b91fc9434a3bb3419acfac2dd3b30cb4a6c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Apr 2026 07:33:51 -0300 Subject: [PATCH 223/313] fix: add pages to typesense schema (#10726) --- ietf/utils/searchindex.py | 4 ++++ ietf/utils/tests_searchindex.py | 1 + 2 files changed, 5 insertions(+) diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py index a47e6d2f120..87951abb609 100644 --- a/ietf/utils/searchindex.py +++ b/ietf/utils/searchindex.py @@ -86,6 +86,7 @@ def _sanitize_text(content): def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: assert rfc.type_id == "rfc" assert rfc.rfc_number is not None + assert rfc.pages is not None keywords: list[str] = rfc.keywords # help type checking @@ -119,6 +120,7 @@ def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: "filename": rfc.name, "title": rfc.title, "abstract": _sanitize_text(rfc.abstract), + "pages": rfc.pages, "keywords": keywords, "type": "rfc", "state": [state.name for state in rfc.states.all()], @@ -231,6 +233,8 @@ def update_or_create_rfc_entries( {"name": "title", "type": "string", "facet": False}, # Abstract of the draft / rfc {"name": "abstract", "type": "string", "facet": False}, + # Number of pages + {"name": "pages", "type": "int32", "facet": False}, # A list of search keywords if relevant, set to empty array otherwise {"name": "keywords", "type": "string[]", "facet": True}, # Type of the document diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py index 0bff96ec7dc..e9fbf520201 100644 --- a/ietf/utils/tests_searchindex.py +++ b/ietf/utils/tests_searchindex.py @@ -70,6 +70,7 @@ def test_typesense_doc_from_rfc(self): self.assertEqual(result["id"], f"doc-{rfc.pk}") self.assertEqual(result["rfcNumber"], rfc.rfc_number) self.assertEqual(result["abstract"], searchindex._sanitize_text(rfc.abstract)) + self.assertEqual(result["pages"], rfc.pages) self.assertNotIn("adName", result) self.assertNotIn("content", result) # no blob self.assertNotIn("subseries", result) From 629ffb13480201e25fc5d941cfcea9de123562f9 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 17 Apr 2026 15:04:23 -0300 Subject: [PATCH 224/313] fix: decode non-utf-8 blob content (#10729) * refactor: decode_document_content() utility method * fix: fall back to latin-1 in retrieve_str() * refactor: match structure with retrieve_bytes() * refactor: separate tests_text.py module * test: test_decode_document_content + ruff * fix: revert misguided refactor * test: assert to guarantee test is valid --- ietf/doc/models.py | 15 ++------- ietf/doc/storage_utils.py | 47 +++++++++++++------------- ietf/utils/tests.py | 19 ----------- ietf/utils/tests_text.py | 71 +++++++++++++++++++++++++++++++++++++++ ietf/utils/text.py | 18 ++++++++++ 5 files changed, 114 insertions(+), 56 deletions(-) create mode 100644 ietf/utils/tests_text.py diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 972f0a34e8e..cc79b73831e 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -52,6 +52,7 @@ from ietf.person.utils import get_active_balloters from ietf.utils import log from ietf.utils.decorators import memoize +from ietf.utils.text import decode_document_content from ietf.utils.validators import validate_no_control_chars from ietf.utils.mail import formataddr from ietf.utils.models import ForeignKey @@ -640,19 +641,7 @@ def text(self, size = -1): except IOError as e: log.log(f"Error reading text for {path}: {e}") return None - text = None - try: - text = raw.decode('utf-8') - except UnicodeDecodeError: - for back in range(1,4): - try: - text = raw[:-back].decode('utf-8') - break - except UnicodeDecodeError: - pass - if text is None: - text = raw.decode('latin-1') - return text + return decode_document_content(raw) def text_or_error(self): return self.text() or "Error; cannot read '%s'"%self.get_base_name() diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index ffdd4599beb..9c18bb8a8af 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -10,6 +10,7 @@ from django.core.files.storage import storages, Storage from ietf.utils.log import log +from ietf.utils.text import decode_document_content class StorageUtilsError(Exception): @@ -164,32 +165,30 @@ def store_str( def retrieve_bytes(kind: str, name: str) -> bytes: from ietf.doc.storage import maybe_log_timing - content = b"" - if settings.ENABLE_BLOBSTORAGE: - try: - store = _get_storage(kind) - with store.open(name) as f: - with maybe_log_timing( - hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, - "read", - bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", - name=name, - ): - content = f.read() - except Exception as err: - log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}") - raise + if not settings.ENABLE_BLOBSTORAGE: + return b"" + try: + store = _get_storage(kind) + with store.open(name) as f: + with maybe_log_timing( + hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, + "read", + bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", + name=name, + ): + content = f.read() + except Exception as err: + log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}") + raise return content def retrieve_str(kind: str, name: str) -> str: - content = "" - if settings.ENABLE_BLOBSTORAGE: - try: - content_bytes = retrieve_bytes(kind, name) - # TODO-BLOBSTORE: try to decode all the different ways doc.text() does - content = content_bytes.decode("utf-8") - except Exception as err: - log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") - raise + if not settings.ENABLE_BLOBSTORAGE: + return "" + try: + content = decode_document_content(retrieve_bytes(kind, name)) + except Exception as err: + log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") + raise return content diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 3288309095c..99c33f34b33 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -60,7 +60,6 @@ set_url_coverage, ) from ietf.utils.test_utils import TestCase, unicontent -from ietf.utils.text import parse_unicode from ietf.utils.timezone import timezone_not_near_midnight from ietf.utils.xmldraft import XMLDraft, InvalidMetadataError, capture_xml2rfc_output @@ -864,24 +863,6 @@ def test_assertion(self): assertion('False') settings.SERVER_MODE = 'test' -class TestRFC2047Strings(TestCase): - def test_parse_unicode(self): - names = ( - ('=?utf-8?b?4Yuz4YuK4Ym1IOGJoOGJgOGIiA==?=', 'ዳዊት በቀለ'), - ('=?utf-8?b?5Li9IOmDnA==?=', '丽 郜'), - ('=?utf-8?b?4KSV4KSu4KWN4KSs4KWL4KScIOCkoeCkvuCksA==?=', 'कम्बोज डार'), - ('=?utf-8?b?zpfPgc6szrrOu861zrnOsSDOm865z4zOvc+Ezrc=?=', 'Ηράκλεια Λιόντη'), - ('=?utf-8?b?15nXqdeo15DXnCDXqNeV15bXoNek15zXkw==?=', 'ישראל רוזנפלד'), - ('=?utf-8?b?5Li95Y2OIOeahw==?=', '丽华 皇'), - ('=?utf-8?b?77ul77qu766V77qzIO+tlu+7ru+vvu+6ju+7pw==?=', 'ﻥﺮﮕﺳ ﭖﻮﯾﺎﻧ'), - ('=?utf-8?b?77uh77uu77qz77uu76++IO+6su+7tO+7p++6jSDvurDvu6Pvuo7vu6jvr74=?=', 'ﻡﻮﺳﻮﯾ ﺲﻴﻧﺍ ﺰﻣﺎﻨﯾ'), - ('=?utf-8?b?ScOxaWdvIFNhbsOnIEliw6HDsWV6IGRlIGxhIFBlw7Fh?=', 'Iñigo Sanç Ibáñez de la Peña'), - ('Mart van Oostendorp', 'Mart van Oostendorp'), - ('', ''), - ) - for encoded_str, unicode in names: - self.assertEqual(unicode, parse_unicode(encoded_str)) - class TestAndroidSiteManifest(TestCase): def test_manifest(self): r = self.client.get(urlreverse('site.webmanifest')) diff --git a/ietf/utils/tests_text.py b/ietf/utils/tests_text.py new file mode 100644 index 00000000000..51aa2eff132 --- /dev/null +++ b/ietf/utils/tests_text.py @@ -0,0 +1,71 @@ +# Copyright The IETF Trust 2021-2026, All Rights Reserved +from ietf.utils.test_utils import TestCase +from ietf.utils.text import parse_unicode, decode_document_content + + +class TestDecoders(TestCase): + def test_parse_unicode(self): + names = ( + ("=?utf-8?b?4Yuz4YuK4Ym1IOGJoOGJgOGIiA==?=", "ዳዊት በቀለ"), + ("=?utf-8?b?5Li9IOmDnA==?=", "丽 郜"), + ("=?utf-8?b?4KSV4KSu4KWN4KSs4KWL4KScIOCkoeCkvuCksA==?=", "कम्बोज डार"), + ("=?utf-8?b?zpfPgc6szrrOu861zrnOsSDOm865z4zOvc+Ezrc=?=", "Ηράκλεια Λιόντη"), + ("=?utf-8?b?15nXqdeo15DXnCDXqNeV15bXoNek15zXkw==?=", "ישראל רוזנפלד"), + ("=?utf-8?b?5Li95Y2OIOeahw==?=", "丽华 皇"), + ("=?utf-8?b?77ul77qu766V77qzIO+tlu+7ru+vvu+6ju+7pw==?=", "ﻥﺮﮕﺳ ﭖﻮﯾﺎﻧ"), + ( + "=?utf-8?b?77uh77uu77qz77uu76++IO+6su+7tO+7p++6jSDvurDvu6Pvuo7vu6jvr74=?=", + "ﻡﻮﺳﻮﯾ ﺲﻴﻧﺍ ﺰﻣﺎﻨﯾ", + ), + ( + "=?utf-8?b?ScOxaWdvIFNhbsOnIEliw6HDsWV6IGRlIGxhIFBlw7Fh?=", + "Iñigo Sanç Ibáñez de la Peña", + ), + ("Mart van Oostendorp", "Mart van Oostendorp"), + ("", ""), + ) + for encoded_str, unicode in names: + self.assertEqual(unicode, parse_unicode(encoded_str)) + + def test_decode_document_content(self): + utf8_bytes = "𒀭𒊩𒌆𒄈𒋢".encode("utf-8") # ends with 4-byte character + latin1_bytes = "àéîøü".encode("latin-1") + other_bytes = "àéîøü".encode("macintosh") # different from its latin-1 encoding + assert other_bytes.decode("macintosh") != other_bytes.decode("latin-1"),\ + "test broken: other_bytes must decode differently as latin-1" + + # simplest case + self.assertEqual( + decode_document_content(utf8_bytes), + utf8_bytes.decode(), + ) + # losing 1-4 bytes from the end leave the last character incomplete; the + # decoder should decode all but that last character + self.assertEqual( + decode_document_content(utf8_bytes[:-1]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-2]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-3]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-4]), + utf8_bytes.decode()[:-1], + ) + + # latin-1 is also simple + self.assertEqual( + decode_document_content(latin1_bytes), + latin1_bytes.decode("latin-1"), + ) + + # other character sets are just treated as latin1 (bug? feature? you decide) + self.assertEqual( + decode_document_content(other_bytes), + other_bytes.decode("latin-1"), + ) diff --git a/ietf/utils/text.py b/ietf/utils/text.py index 590ec3fd30a..2763056e1a9 100644 --- a/ietf/utils/text.py +++ b/ietf/utils/text.py @@ -263,3 +263,21 @@ def parse_unicode(text): else: text = decoded_string return text + + +def decode_document_content(content: bytes) -> str: + """Decode document contents as utf-8 or latin1 + + Method was developed in DocumentInfo.text() where it gave acceptable results + for existing documents / RFCs. + """ + try: + return content.decode("utf-8") + except UnicodeDecodeError: + pass + for back in range(1, 4): + try: + return content[:-back].decode("utf-8") + except UnicodeDecodeError: + pass + return content.decode("latin-1") # everything is legal in latin-1 From 63a69945ab11b1c3b3ec490fb260073c90eed0bc Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 17 Apr 2026 16:24:18 -0500 Subject: [PATCH 225/313] test: Squash some transient test error vectors (#10730) * test: enforce queryset order assumed by test * test: match html escaping in test * test: search more specifically for tokens to avoid mis-reading them when they occur in faker data --- ietf/group/tests_review.py | 30 +++++++++++++------------- ietf/meeting/tests_session_requests.py | 2 +- ietf/meeting/tests_views.py | 7 +++--- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/ietf/group/tests_review.py b/ietf/group/tests_review.py index 89c755bb264..bb9b79a416a 100644 --- a/ietf/group/tests_review.py +++ b/ietf/group/tests_review.py @@ -888,10 +888,10 @@ def test_requests_history_filter_page(self): self.assertEqual(r.status_code, 200) self.assertContains(r, review_req.doc.name) self.assertContains(r, review_req2.doc.name) - self.assertContains(r, 'Assigned') - self.assertContains(r, 'Accepted') - self.assertContains(r, 'Completed') - self.assertContains(r, 'Ready') + self.assertContains(r, 'data-text="Assigned"') + self.assertContains(r, 'data-text="Accepted"') + self.assertContains(r, 'data-text="Completed"') + self.assertContains(r, 'data-text="Ready"') self.assertContains(r, escape(assignment.reviewer.person.name)) self.assertContains(r, escape(assignment2.reviewer.person.name)) @@ -907,10 +907,10 @@ def test_requests_history_filter_page(self): self.assertEqual(r.status_code, 200) self.assertContains(r, review_req.doc.name) self.assertNotContains(r, review_req2.doc.name) - self.assertContains(r, 'Assigned') - self.assertNotContains(r, 'Accepted') - self.assertNotContains(r, 'Completed') - self.assertNotContains(r, 'Ready') + self.assertContains(r, 'data-text="Assigned"') + self.assertNotContains(r, 'data-text="Accepted"') + self.assertNotContains(r, 'data-text="Completed"') + self.assertNotContains(r, 'data-text="Ready"') self.assertContains(r, escape(assignment.reviewer.person.name)) self.assertNotContains(r, escape(assignment2.reviewer.person.name)) @@ -926,10 +926,10 @@ def test_requests_history_filter_page(self): self.assertEqual(r.status_code, 200) self.assertNotContains(r, review_req.doc.name) self.assertContains(r, review_req2.doc.name) - self.assertNotContains(r, 'Assigned') - self.assertContains(r, 'Accepted') - self.assertContains(r, 'Completed') - self.assertContains(r, 'Ready') + self.assertNotContains(r, 'data-text="Assigned"') + self.assertContains(r, 'data-text="Accepted"') + self.assertContains(r, 'data-text="Completed"') + self.assertContains(r, 'data-text="Ready"') self.assertNotContains(r, escape(assignment.reviewer.person.name)) self.assertContains(r, escape(assignment2.reviewer.person.name)) @@ -940,9 +940,9 @@ def test_requests_history_filter_page(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertNotContains(r, review_req.doc.name) - self.assertNotContains(r, 'Assigned') - self.assertNotContains(r, 'Accepted') - self.assertNotContains(r, 'Completed') + self.assertNotContains(r, 'data-text="Assigned"') + self.assertNotContains(r, 'data-text="Accepted"') + self.assertNotContains(r, 'data-text="Completed"') def test_requests_history_invalid_filter_parameters(self): # First assignment as assigned diff --git a/ietf/meeting/tests_session_requests.py b/ietf/meeting/tests_session_requests.py index 0cb092d2f87..42dbee5f230 100644 --- a/ietf/meeting/tests_session_requests.py +++ b/ietf/meeting/tests_session_requests.py @@ -236,7 +236,7 @@ def test_edit(self): self.assertRedirects(r, redirect_url) # Check whether updates were stored in the database - sessions = Session.objects.filter(meeting=meeting, group=mars) + sessions = Session.objects.filter(meeting=meeting, group=mars).order_by("id") self.assertEqual(len(sessions), 2) session = sessions[0] self.assertFalse(session.constraints().filter(name='time_relation')) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 258ffe554c2..17988e50be4 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -33,6 +33,7 @@ from django.http import QueryDict, FileResponse from django.template import Context, Template from django.utils import timezone +from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.text import slugify @@ -9491,7 +9492,7 @@ def test_session_attendance(self): self.assertEqual(r.status_code, 200) self.assertContains(r, '3 attendees') for person in persons: - self.assertContains(r, person.plain_name()) + self.assertContains(r, escape(person.plain_name())) # Test for the "I was there" button. def _test_button(person, expected): @@ -9511,14 +9512,14 @@ def _test_button(person, expected): # attempt to POST anyway is ignored r = self.client.post(attendance_url) self.assertEqual(r.status_code, 200) - self.assertNotContains(r, persons[3].plain_name()) + self.assertNotContains(r, escape(persons[3].plain_name())) self.assertEqual(session.attended_set.count(), 3) # button is shown, and POST is accepted meeting.importantdate_set.update(name_id='revsub',date=date_today() + datetime.timedelta(days=20)) _test_button(persons[3], True) r = self.client.post(attendance_url) self.assertEqual(r.status_code, 200) - self.assertContains(r, persons[3].plain_name()) + self.assertContains(r, escape(persons[3].plain_name())) self.assertEqual(session.attended_set.count(), 4) # When the meeting is finalized, a bluesheet file is generated, From dc49dc8362812893cad560feecc55efcea1553dc Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 20 Apr 2026 14:29:41 -0300 Subject: [PATCH 226/313] chore: beat termination grace period -> 10 s (#10741) --- k8s/beat.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/beat.yaml b/k8s/beat.yaml index 9ab242681ce..b4291c7e31d 100644 --- a/k8s/beat.yaml +++ b/k8s/beat.yaml @@ -59,4 +59,4 @@ spec: name: files-cfgmap dnsPolicy: ClusterFirst restartPolicy: Always - terminationGracePeriodSeconds: 600 + terminationGracePeriodSeconds: 10 From 4d69329ef86054fa5bfb9da9acd0c966ab013d8f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 20 Apr 2026 23:59:36 -0300 Subject: [PATCH 227/313] chore: remove blobdb profiling logs (#10732) These are not useful any more, blobdb is fast --- ietf/doc/storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ietf/doc/storage.py b/ietf/doc/storage.py index 375620ccaf3..ee1e76c4fac 100644 --- a/ietf/doc/storage.py +++ b/ietf/doc/storage.py @@ -114,7 +114,6 @@ def _get_write_parameters(self, name, content=None): class StoredObjectBlobdbStorage(BlobdbStorage): - ietf_log_blob_timing = True warn_if_missing = True # TODO-BLOBSTORE make this configurable (or remove it) def _save_stored_object(self, name, content) -> StoredObject: From e8e4dd65c325199e7f1a1b5f254c0359bd8b144d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 27 Apr 2026 17:15:14 -0300 Subject: [PATCH 228/313] feat: sync blobs with replica via admin (#10756) * refactor: factor out queue_for_replication() * feat: BlobdbStorage.force_replication() * feat: force_replication() in storage_utils.py * feat: replicate_stored_objects_for_document * feat: admin action to replicate Document blobs * feat: admin action to replicate StoredObject blobs * feat: admin action to replicate Blobs * fix: use get_blobdb() helper * fix: drop references to R2 in the admin --- ietf/blobdb/admin.py | 20 +++++++++++++++++++- ietf/blobdb/models.py | 27 ++++---------------------- ietf/blobdb/storage.py | 12 +++++++++++- ietf/blobdb/utils.py | 32 +++++++++++++++++++++++++++++++ ietf/doc/admin.py | 40 +++++++++++++++++++++++++++++++++++++-- ietf/doc/storage_utils.py | 9 +++++++++ ietf/doc/utils.py | 25 +++++++++++++++++++++++- 7 files changed, 137 insertions(+), 28 deletions(-) create mode 100644 ietf/blobdb/utils.py diff --git a/ietf/blobdb/admin.py b/ietf/blobdb/admin.py index 3e1a2a311f7..44a30d1d7f2 100644 --- a/ietf/blobdb/admin.py +++ b/ietf/blobdb/admin.py @@ -1,9 +1,12 @@ -# Copyright The IETF Trust 2025, All Rights Reserved +# Copyright The IETF Trust 2025-2026, All Rights Reserved from django.contrib import admin +from django.db.models import QuerySet from django.db.models.functions import Length from rangefilter.filters import DateRangeQuickSelectListFilterBuilder +from .apps import get_blobdb from .models import Blob, ResolvedMaterial +from .utils import queue_for_replication @admin.register(Blob) @@ -17,6 +20,7 @@ class BlobAdmin(admin.ModelAdmin): ] search_fields = ["name"] list_display_links = ["name"] + actions = ["replicate_blob"] def get_queryset(self, request): return ( @@ -30,6 +34,20 @@ def object_size(self, instance): """Get the size of the object""" return instance.object_size # annotation added in get_queryset() + @admin.action(description="Replicate blobs") + def replicate_blob(self, request, queryset: QuerySet[Blob]): + blob_count = 0 + for blob in queryset.all(): + if isinstance(blob, Blob): + queue_for_replication( + bucket=blob.bucket, name=blob.name, using=get_blobdb() + ) + blob_count += 1 + self.message_user( + request, + f"Queued replication of a total of {blob_count} Blob(s)", + ) + @admin.register(ResolvedMaterial) class ResolvedMaterialAdmin(admin.ModelAdmin): diff --git a/ietf/blobdb/models.py b/ietf/blobdb/models.py index 27325ada5d3..6dbb615fa04 100644 --- a/ietf/blobdb/models.py +++ b/ietf/blobdb/models.py @@ -1,14 +1,11 @@ -# Copyright The IETF Trust 2025, All Rights Reserved -import json -from functools import partial +# Copyright The IETF Trust 2025-2026, All Rights Reserved from hashlib import sha384 from django.db import models, transaction from django.utils import timezone from .apps import get_blobdb -from .replication import replication_enabled -from .tasks import pybob_the_blob_replicator_task +from .utils import queue_for_replication class BlobQuerySet(models.QuerySet): @@ -81,24 +78,8 @@ def delete(self, **kwargs): self._emit_blob_change_event(using=db) return retval - def _emit_blob_change_event(self, using=None): - if not replication_enabled(self.bucket): - return - - # For now, fire a celery task we've arranged to guarantee in-order processing. - # Later becomes pushing an event onto a queue to a dedicated worker. - transaction.on_commit( - partial( - pybob_the_blob_replicator_task.delay, - json.dumps( - { - "name": self.name, - "bucket": self.bucket, - } - ) - ), - using=using, - ) + def _emit_blob_change_event(self, using: str | None=None): + queue_for_replication(self.bucket, self.name, using=using) class ResolvedMaterial(models.Model): diff --git a/ietf/blobdb/storage.py b/ietf/blobdb/storage.py index 4213ec801df..e304dabc5dd 100644 --- a/ietf/blobdb/storage.py +++ b/ietf/blobdb/storage.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2025, All Rights Reserved +# Copyright The IETF Trust 2025-2026, All Rights Reserved from typing import Optional from django.core.exceptions import SuspiciousFileOperation @@ -10,6 +10,7 @@ from ietf.utils.storage import MetadataFile from .models import Blob +from .utils import queue_for_replication class BlobFile(MetadataFile): @@ -94,3 +95,12 @@ def get_available_name(self, name, max_length=None): f"asked to store the name '{name[:5]}...{name[-5:]} of length {len(name)}" ) return name # overwrite is permitted + + def force_replication(self, name: str): + """Force replication of a blob by name + + Be careful with this - replication includes replicating deletion of blobs, so + if you call it with a name that does not exist in blobdb, it will be removed + from R2 if it exists there! + """ + queue_for_replication(bucket=self.bucket_name, name=name) diff --git a/ietf/blobdb/utils.py b/ietf/blobdb/utils.py new file mode 100644 index 00000000000..93f8f2f5217 --- /dev/null +++ b/ietf/blobdb/utils.py @@ -0,0 +1,32 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import json +from functools import partial + +from django.db import transaction + +from ietf.blobdb.replication import replication_enabled +from ietf.blobdb.tasks import pybob_the_blob_replicator_task + + +def queue_for_replication(bucket: str, name: str, using: str | None=None): + """Queue a blob for replication + + This is private to the blobdb app. Do not call it directly from other apps. + """ + if not replication_enabled(bucket): + return + + # For now, fire a celery task we've arranged to guarantee in-order processing. + # Later becomes pushing an event onto a queue to a dedicated worker. + transaction.on_commit( + partial( + pybob_the_blob_replicator_task.delay, + json.dumps( + { + "name": name, + "bucket": bucket, + } + ) + ), + using=using, + ) diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index 0d04e8db3ad..757d3da9f97 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -5,6 +5,7 @@ from django.contrib import admin from django.db import models from django import forms +from django.db.models import QuerySet from rangefilter.filters import DateRangeQuickSelectListFilterBuilder from .models import (StateType, State, RelatedDocument, DocumentAuthor, Document, RelatedDocHistory, @@ -18,6 +19,9 @@ from ietf.utils.admin import SaferTabularInline from ietf.utils.validators import validate_external_resource_value +from .storage_utils import force_replication +from .utils import replicate_stored_objects_for_document + class StateTypeAdmin(admin.ModelAdmin): list_display = ["slug", "label"] @@ -73,7 +77,9 @@ class DocumentAuthorAdmin(admin.ModelAdmin): search_fields = ['document__name', 'person__name', 'email__address', 'affiliation', 'country'] raw_id_fields = ["document", "person", "email"] admin.site.register(DocumentAuthor, DocumentAuthorAdmin) - + + + class DocumentAdmin(admin.ModelAdmin): list_display = ['name', 'rev', 'group', 'pages', 'intended_std_level', 'author_list', 'time'] search_fields = ['name'] @@ -81,6 +87,7 @@ class DocumentAdmin(admin.ModelAdmin): raw_id_fields = ['group', 'shepherd', 'ad'] inlines = [DocAuthorInline, DocActionHolderInline, RelatedDocumentInline, AdditionalUrlInLine] form = DocumentForm + actions = ["replicate_stored_objects"] def save_model(self, request, obj, form, change): e = DocEvent.objects.create( @@ -95,6 +102,22 @@ def save_model(self, request, obj, form, change): def state(self, instance): return self.get_state() + @admin.action(description="Replicate related blobs") + def replicate_stored_objects(self, request, queryset: QuerySet[Document]): + doc_count = 0 + stored_obj_count = 0 + for doc in queryset.all(): + doc_count += 1 + if isinstance(doc, Document): + stored_obj_count += replicate_stored_objects_for_document(doc) + self.message_user( + request, + ( + f"Queued replication of a total of {stored_obj_count} StoredObject(s) " + f"for {doc_count} Document(s)" + ) + ) + admin.site.register(Document, DocumentAdmin) class DocHistoryAdmin(admin.ModelAdmin): @@ -232,11 +255,24 @@ class StoredObjectAdmin(admin.ModelAdmin): ] search_fields = ['name', 'doc_name', 'doc_rev'] list_display_links = ['name'] + actions = ["replicate_stored_object"] @admin.display(boolean=True, description="Deleted?", ordering="deleted") def is_deleted(self, instance): return instance.deleted is not None - + + @admin.action(description="Replicate related blobs") + def replicate_stored_object(self, request, queryset: QuerySet[StoredObject]): + stored_obj_count = 0 + for stored_object in queryset.all(): + if isinstance(stored_object, StoredObject): + force_replication(kind=stored_object.store, name=stored_object.name) + stored_obj_count += 1 + self.message_user( + request, + f"Queued replication of a total of {stored_obj_count} StoredObject(s)", + ) + admin.site.register(StoredObject, StoredObjectAdmin) diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index 9c18bb8a8af..c7cc6989cd0 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -192,3 +192,12 @@ def retrieve_str(kind: str, name: str) -> str: log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") raise return content + + +def force_replication(kind: str, name: str): + if not settings.ENABLE_BLOBSTORAGE: + return + storage = _get_storage(kind) + from ietf.blobdb.storage import BlobdbStorage + if isinstance(storage, BlobdbStorage): + storage.force_replication(name) diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 8cbe5e8f3ec..6f32ed454f2 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -39,12 +39,15 @@ DocHistoryAuthor, Document, DocumentAuthor, + EditedRfcAuthorsDocEvent, RfcAuthor, - State, EditedRfcAuthorsDocEvent, + State, + StoredObject, ) from ietf.doc.models import RelatedDocument, RelatedDocHistory, BallotType, DocReminder from ietf.doc.models import DocEvent, ConsensusDocEvent, BallotDocEvent, IRSGBallotDocEvent, NewRevisionDocEvent, StateDocEvent from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent, BallotPositionDocEvent +from ietf.doc.storage_utils import force_replication from ietf.name.models import DocReminderTypeName, DocRelationshipName from ietf.group.models import Role, Group, GroupFeatures from ietf.ietfauth.utils import has_role, is_authorized_in_doc_stream, is_individual_draft_author, is_bofreq_editor @@ -1713,3 +1716,23 @@ def update_or_create_draft_bibxml_file(doc, rev): def ensure_draft_bibxml_path_exists(): (Path(settings.BIBXML_BASE_PATH) / "bibxml-ids").mkdir(exist_ok=True) + + +def replicate_stored_objects_for_document(doc: Document) -> int: + """Sync all StoredObjects associated with doc to the replica blob store + + Returns count of StoredObjects queued for replication (which may or may not + be replicated, depending on whether replication is enabled / the storages are + actually BlobdbStorage instances, etc). + """ + # n.b., StoredObjects have a nullable doc_rev field, but Documents do not. + # Until / unless we straighten that out, treat "" and None equivalently when + # matching rev. + qs_matching_rev = StoredObject.objects.filter(doc_rev=doc.rev) + if doc.rev == "": + qs_matching_rev |= StoredObject.objects.filter(doc_rev__isnull=True) + count = 0 + for stored_object in qs_matching_rev.filter(doc_name=doc.name): + force_replication(kind=stored_object.store, name=stored_object.name) + count += 1 + return count From e3e080fa2341d4cae501dcfbebf8e438e49b6f46 Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Tue, 28 Apr 2026 19:02:07 +0200 Subject: [PATCH 229/313] feat: add some per country/affiliation meeting registration statistics (#10550) * Draft for meeting registrations * Add totals, + nicer JS code * Coherent URL parameters * Handle error case when per-continent stats is requested * Remove redundant code * Dynamic get the current IETF meeting * Display %-age when hovering * Add test for meeting statistics * Add test * Add statistics per affiliation * More code coverage for test * Nicer canonical affiliation * Allow navigation by buttons * Also add participants count in the legend * Add timeline over meetings (total and per country) * Default index refers to current meeting stats by number * Remove unused JS code * Add test coverage for timeline statistics * No need to import test coverage * Use stacked lines of onsite/remote when displaying the total timeline * fix a comment * Add timeline for affiliation * Expanding the test coverage to affiliation timeline * Remove unused botocore (unsure how it was added though) * Remove unused package * Code clean-up, add pan & zoom on timelines * Fix button type * refactor: avoid inline JS; safer JSON handling * chore: lint * feat: cache timeline stats Pins the top_n parameter, which had not been plumbed to be dynamically adjustable. * chore: timeout->settings + drop stale setting STATS_NAMES_LIMIT does not appear anywhere else in the codebase * refactor: wait for DOMContentLoaded + restyle * fix: fix null checks * test: update test_meeting_stats() --------- Co-authored-by: Jennifer Richards --- ietf/settings.py | 3 +- ietf/static/js/meeting_stats.js | 57 ++ ietf/static/js/meeting_timeline.js | 84 +++ ietf/stats/tests.py | 63 +- ietf/stats/urls.py | 5 +- ietf/stats/views.py | 612 ++++++++++++++++++-- ietf/templates/base/menu.html | 11 +- ietf/templates/stats/index.html | 8 +- ietf/templates/stats/meeting_stats.html | 61 ++ ietf/templates/stats/meetings_timeline.html | 78 +++ package.json | 2 + 11 files changed, 916 insertions(+), 68 deletions(-) create mode 100644 ietf/static/js/meeting_stats.js create mode 100644 ietf/static/js/meeting_timeline.js create mode 100644 ietf/templates/stats/meeting_stats.html create mode 100644 ietf/templates/stats/meetings_timeline.html diff --git a/ietf/settings.py b/ietf/settings.py index 3aa45a453c0..50e069ff1a5 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -231,6 +231,7 @@ AGENDA_CACHE_TIMEOUT_DEFAULT = 8 * 24 * 60 * 60 # 8 days AGENDA_CACHE_TIMEOUT_CURRENT_MEETING = 6 * 60 # 6 minutes + WSGI_APPLICATION = "ietf.wsgi.application" AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', ) @@ -1270,7 +1271,7 @@ def skip_unreadable_post(record): except ImportError: pass -STATS_NAMES_LIMIT = 25 +STATS_TIMELINE_CACHE_TIMEOUT = 86400 UTILS_MEETING_CONFERENCE_DOMAINS = ['webex.com', 'zoom.us', 'jitsi.org', 'meetecho.com', 'gather.town', ] UTILS_TEST_RANDOM_STATE_FILE = '.factoryboy_random_state' diff --git a/ietf/static/js/meeting_stats.js b/ietf/static/js/meeting_stats.js new file mode 100644 index 00000000000..70b18a0f03e --- /dev/null +++ b/ietf/static/js/meeting_stats.js @@ -0,0 +1,57 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +document.addEventListener('DOMContentLoaded', () => { + // Need to use autocolors plug-in else all slices are gray... + const autocolors = window['chartjs-plugin-autocolors'] + Chart.register(autocolors) + // ── Safely parse JSON data injected from Django view ── + const totalChartData = JSON.parse(document.getElementById('total-chart-data').textContent) + const inPersonChartData = JSON.parse(document.getElementById('in-person-chart-data').textContent) + + function displayChart (id, data) { + const ctx = document.getElementById(id).getContext('2d') + new Chart(ctx, { + type: 'pie', // Change to 'doughnut' for a donut chart + data: data, + options: { + responsive: true, + plugins: { + autocolors: { + mode: 'data' // Required for Pie charts to color individual slices + }, + legend: { + position: 'bottom', + labels: { + padding: 20, + font: { size: 13 }, + color: '#475569', + generateLabels: function (chart) { + const dataset = chart.data.datasets[0] + return chart.data.labels.map((label, i) => ({ + text: `${label}: ${dataset.data[i]}`, + fillStyle: dataset.backgroundColor[i], + hidden: false, + index: i, + })) + } + } + }, + tooltip: { + callbacks: { + label: function (context) { + const label = context.label || '' + const value = context.raw + const total = context.dataset.data.reduce((a, b) => a + b, 0) + const percentage = ((value / total) * 100).toFixed(1) + + return `${label}: ${value} (${percentage}%)` + } + } + } + } + } + }) + } + + displayChart('totalRegistrationChart', totalChartData) + displayChart('inPersonRegistrationChart', inPersonChartData) +}) diff --git a/ietf/static/js/meeting_timeline.js b/ietf/static/js/meeting_timeline.js new file mode 100644 index 00000000000..161cead0ec5 --- /dev/null +++ b/ietf/static/js/meeting_timeline.js @@ -0,0 +1,84 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +document.addEventListener('DOMContentLoaded', () => { + // ── Safely parse JSON data injected from Django view ── + const totalChartData = JSON.parse(document.getElementById('total-chart-data').textContent) + const inPersonChartData = JSON.parse(document.getElementById('in-person-chart-data').textContent) + const statsType = JSON.parse(document.getElementById('stats-type-data').textContent) + const stackedLines = statsType === 'total' + + function displayChart (id, data) { + const ctx = document.getElementById(id).getContext('2d') + return new Chart(ctx, { + type: 'line', // Change to 'doughnut' for a donut chart + data: data, + options: { + responsive: true, + scales: { + y: { + stacked: stackedLines, + }, + x: { + title: { + display: true, + text: 'IETF Meeting Number', + }, + }, + }, + plugins: { + legend: { + position: 'bottom', + labels: { + usePointStyle: true, + padding: 15, + font: { size: 12 }, + }, + }, + tooltip: { + backgroundColor: 'rgba(0,0,0,0.8)', + titleFont: { size: 14 }, + bodyFont: { size: 13 }, + callbacks: { + title: function (items) { + return `IETF Meeting ${items[0].label}` + }, + label: function (context) { + return ` ${context.dataset.label}: ${context.parsed.y} participants` + } + } + }, + zoom: { + zoom: { + wheel: { enabled: true }, // scroll to zoom + pinch: { enabled: true }, // pinch on mobile + drag: { enabled: true }, // drag to select range + mode: 'xy', // zoom X-axis and Y-axis + }, + pan: { + enabled: true, + mode: 'xy', // pan X-axis and Y-axis + }, + }, + } + } + }) + } + + const totalChart = displayChart('totalRegistrationChart', totalChartData) + if (inPersonChartData !== null) { + inPersonChart = displayChart('inPersonRegistrationChart', inPersonChartData) + } + document.addEventListener('keydown', (event) => { + if (event.key === 'Escape') { + totalChart.resetZoom() + if (inPersonChart !== null) { + inPersonChart.resetZoom() + } + } + }) + document.getElementById('resetButton').addEventListener('click', () => { + totalChart.resetZoom() + if (inPersonChart !== null) { + inPersonChart.resetZoom() + } + }) +}) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 48552c8fbac..373f06e3431 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -4,12 +4,14 @@ import calendar import json +import datetime from pyquery import PyQuery import debug # pyflakes:ignore from django.urls import reverse as urlreverse +from django.utils import timezone from ietf.utils.test_utils import login_testing_unauthorized, TestCase import ietf.stats.views @@ -18,24 +20,73 @@ from ietf.group.factories import RoleFactory from ietf.person.factories import PersonFactory from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory +from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory from ietf.utils.timezone import date_today class StatisticsTests(TestCase): def test_stats_index(self): + # Create a meeting as the index page needs to know the current meeting + MeetingFactory(type_id='ietf', number='124', date=timezone.now()) url = urlreverse(ietf.stats.views.stats_index) r = self.client.get(url) self.assertEqual(r.status_code, 200) def test_document_stats(self): - r = self.client.get(urlreverse("ietf.stats.views.document_stats")) - self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index")) - + # Create a meeting as the index page needs to know the current meeting + MeetingFactory(type_id='ietf', number='124', date=timezone.now()) + r = self.client.get(urlreverse(ietf.stats.views.document_stats)) + self.assertRedirects(r, urlreverse(ietf.stats.views.stats_index)) def test_meeting_stats(self): - r = self.client.get(urlreverse("ietf.stats.views.meeting_stats")) - self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index")) - + meeting124 = MeetingFactory(type_id='ietf', number='124', date=timezone.now()) + meeting125 = MeetingFactory(type_id='ietf', number='125', date=timezone.now() + datetime.timedelta(days=120)) + RegistrationFactory.create_batch(15, meeting=meeting124, with_ticket={'attendance_type_id': 'onsite'}, attended=True) + RegistrationFactory(meeting=meeting124, with_ticket={'attendance_type_id': 'onsite'}, attended=False) + RegistrationFactory.create_batch(14, meeting=meeting124, with_ticket={'attendance_type_id': 'remote'}, attended=True) + RegistrationFactory(meeting=meeting124, with_ticket={'attendance_type_id': 'remote'}, attended=False) + RegistrationFactory.create_batch(15, meeting=meeting125, affiliation='Test LLC', with_ticket={'attendance_type_id': 'remote'}, attended=False) + RegistrationFactory.create_batch(25, meeting=meeting125, affiliation='Example, Ltd', with_ticket={'attendance_type_id': 'onsite'}, attended=False) + # Test the meeting specific statitistics per affiliation and per country + r = self.client.get(urlreverse(ietf.stats.views.meeting_stats, kwargs={"meeting_number": "124", "stats_type": "affiliation"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Total Registrations by Affiliation (31 in total)") + self.assertContains(r, "In Person Registrations by Affiliation (16 in total)") + self.assertContains(r, "/stats/meeting/124/affiliation") + self.assertContains(r, "/stats/meeting/125/affiliation") + r = self.client.get(urlreverse(ietf.stats.views.meeting_stats, kwargs={"meeting_number": "124", "stats_type": "country"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Total Registrations by Country (31 in total)") + self.assertContains(r, "In Person Registrations by Country (16 in total)") + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + # Test the meetings timeline per country + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "country"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + self.assertContains(r, "This page provides a timeline of meeting registrations by country") + # Test the meetings timeline per affiliation + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "affiliation"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/affiliation") + self.assertContains(r, "/stats/meeting/125/affiliation") + self.assertContains(r, "This page provides a timeline of meeting registrations by affiliation") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + in_person_data = json.loads(pq.find("script#in-person-chart-data").text()) + self.assertTrue( + any( + ds["label"] == "Example" and ds["data"] == [0, 25] + for ds in in_person_data["datasets"] + ) + ) + # Test the global meetings timeline + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "total"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + self.assertContains(r, "This page provides a timeline of meeting registrations.") def test_known_country_list(self): # check redirect diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index d2993759d2a..01b8758c840 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -11,7 +11,8 @@ url(r"^$", views.stats_index), url(r"^document/(?:(?Pauthors|pages|words|format|formlang|author/(?:documents|affiliation|country|continent|citations|hindex)|yearly/(?:affiliation|country|continent))/)?$", views.document_stats), url(r"^knowncountries/$", views.known_countries_list), - url(r"^meeting/(?P\d+)/(?Pcountry|continent)/$", views.meeting_stats), - url(r"^meeting/(?:(?Poverview|country|continent)/)?$", views.meeting_stats), + url(r"^meeting/$", views.meetings_timeline), + url(r"^meeting/(?P\d+)/(?Paffiliation|country)/$", views.meeting_stats), + url(r"^meeting/(?:(?Paffiliation|country|total)/)?$", views.meetings_timeline), url(r"^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), ] diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 504d84e86d6..d61b6730754 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -9,11 +9,13 @@ import dateutil.relativedelta from collections import defaultdict +from django.conf import settings from django.contrib.auth.decorators import login_required +from django.core.cache import cache from django.http import HttpResponseRedirect from django.shortcuts import render from django.urls import reverse as urlreverse - +from django.db.models import Count import debug # pyflakes:ignore @@ -25,15 +27,32 @@ from ietf.group.models import Role, Group from ietf.person.models import Person from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName +from ietf.meeting.models import Registration from ietf.ietfauth.utils import has_role from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, DEADLINE_TZINFO +from ietf.meeting.helpers import get_current_ietf_meeting_num, get_ietf_meeting +# Color palette for lines +colors = [ + '#FF6384', '#36A2EB', '#FFCE56', '#4BC0C0', '#9966FF', + '#FF9F40', '#C9CBCF', '#7BC043', '#F37735', '#00ABA9', + '#2B5797', '#E81123', '#00A4EF', '#7FBA00', '#FFB900', + '#D83B01', '#B4009E', '#5C2D91', '#008575', '#E3008C', +] def stats_index(request): - return render(request, "stats/index.html") + """Render the statistics index page with the current meeting number as it is required by the meeting menu item.""" + current_meeting = get_current_ietf_meeting_num() + return render(request, "stats/index.html", { + "current_meeting": current_meeting + }) def generate_query_string(query_dict, overrides): + """ + Returns: + A query string starting with '?' if there are parameters, empty string otherwise. + """ query_part = "" if query_dict or overrides: @@ -58,9 +77,20 @@ def generate_query_string(query_dict, overrides): return query_part def get_choice(request, get_parameter, possible_choices, multiple=False): - # the statistics are built with links to make navigation faster, - # so we don't really have a form in most cases, so just use this - # helper instead to select between the choices + """Extract a choice from the request GET parameters. + + Since statistics pages use links for navigation instead of forms, + this helper selects between possible choices from the URL parameters. + + Args: + request: The HTTP request object. + get_parameter: The name of the GET parameter. + possible_choices: List of tuples (value, label). + multiple: If True, return a list of found values; otherwise return the first found or None. + + Returns: + The selected value(s) or None. + """ values = request.GET.getlist(get_parameter) found = [t[0] for t in possible_choices if t[0] in values] @@ -73,75 +103,553 @@ def get_choice(request, get_parameter, possible_choices, multiple=False): return None def add_url_to_choices(choices, url_builder): + """Add URLs to a list of choices. + + Args: + choices: List of tuples (slug, label). + url_builder: Function that takes a slug and returns a URL. + + Returns: + List of tuples (slug, label, url). + """ return [ (slug, label, url_builder(slug)) for slug, label in choices] -def put_into_bin(value, bin_size): - if value is None: - return (0, '') +def document_stats(request, stats_type=None): + # timeline per year, or per specific year: streams, affiliation, rfc vs I-D + # could also be time between individual/WG I-D to rfc publication/IESG ballot + # DISCUSS resolution time + # Humm also split by authors (affiliation) / documents (the rest) probably + """Redirect to the stats index page. Deprecated view.""" + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - v = (value // bin_size) * bin_size - return (v, "{} - {}".format(v, v + bin_size - 1)) +def known_countries_list(request, stats_type=None, acronym=None): + """Render a list of known countries with their aliases.""" + countries = CountryName.objects.prefetch_related("countryalias_set") + for c in countries: + # the sorting is a bit of a hack - it puts the ISO code first + # since it was added in a migration + c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) -def prune_unknown_bin_with_known(bins): - # remove from the unknown bin all authors within the - # named/known bins - all_known = { n for b, names in bins.items() if b for n in names } - bins[""] = [name for name in bins[""] if name not in all_known] - if not bins[""]: - del bins[""] + return render(request, "stats/known_countries_list.html", { + "countries": countries, + }) -def count_bins(bins): - return len({ n for b, names in bins.items() if b for n in names }) +def canonicalize_affiliation(affiliation): + """Canonicalize an affiliation string by removing common suffixes and standardizing prefixes. + + Args: + affiliation: The affiliation string to canonicalize. + + Returns: + The canonicalized affiliation string, or None if input is None. + """ + if not affiliation or affiliation.lower() in ('n/a', 'none', 'unspecified'): + return None + for suffix in ('ab', 'ag', 'corp', 'corp.', 'corporation', 'gmbh', 'inc.', 'inc', 'international pte ltd', 'llc', 'ltd', 'ltd.', 'private limited', 'pty ltd', 'pvt ltd'): + if affiliation.lower().endswith(', ' + suffix): + affiliation = affiliation[:-(len(suffix)+2)] + elif affiliation.lower().endswith(' ' + suffix): + affiliation = affiliation[:-(len(suffix)+1)] + elif affiliation.lower().endswith(',' + suffix): + affiliation = affiliation[:-(len(suffix)+1)] + for prefix in ('akamai','apple', 'cisco', 'futurewei', 'google', 'hitachi', 'hpe', 'huawei', 'juniper', 'meta', 'nokia', 'ntt', 'siemens'): + if affiliation.lower().startswith(prefix + ' '): + affiliation = prefix + return affiliation.title() + +def get_affiliation_data_for_meetings(attendance_type=None): + """Get affiliation participation data for meetings timeline chart. + + Args: + attendance_type: Optional filter for attendance type (e.g., 'onsite'). + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = f'stats:get_affiliation_data_for_meetings:{attendance_type}' + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + top_n = 20 # could be a parameter, but would need to adjust cache handling + + # Get registration status details + if attendance_type: + registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) + else: + registrations = Registration.objects.all() + registrations = registrations.values('affiliation', 'meeting__number') + + # Count per canonicalized affiliation + organization = dict() + meetings_set = set() + org_totals = defaultdict(int) + data_map = defaultdict(dict) # {org: {meeting: count}} + + for reg in registrations: + meeting = reg['meeting__number'] + meetings_set.add(meeting) + affiliation = canonicalize_affiliation(reg['affiliation']) or "Unspecified" + organization[affiliation] = organization.get(affiliation, 0) + 1 + org_totals[affiliation] = org_totals.get(affiliation, 0) + 1 + data_map[affiliation][meeting] = data_map[affiliation].get(meeting, 0) + 1 + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + + # ── Step 3: Get top N countries ── + top_orgs = sorted( + org_totals.keys(), + key=lambda c: org_totals[c], + reverse=True + )[:top_n] + non_top_orgs = org_totals.keys() - top_orgs + other_totals = defaultdict(int) + for m in sorted_meetings: + other_totals[m] = 0 + for c in non_top_orgs: + other_totals[m] += int(data_map[c].get(m, 0)) + + # ── Step 4: Build Chart.js datasets ── + + datasets = [] + for idx, org in enumerate(top_orgs): + color = colors[idx % len(colors)] + datasets.append({ + 'label': org, + 'data': [data_map[org].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'fill': False, + 'tension': 0.3, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + + # -- Step 4.bis handle the other -- + datasets.append({ + 'label': 'Other', + 'data': [other_totals.get(m, 0) for m in sorted_meetings], + 'borderColor': 'black', + 'fill': False, + 'tension': 0.3, + 'pointColor': 'black', + 'pointBackgroundColor': 'black', + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) -def add_labeled_top_series_from_bins(chart_data, bins, limit): - """Take bins on the form (x, label): [name1, name2, ...], figure out - how many there are per label, take the overall top ones and put - them into sorted series like [(x1, len(names1)), (x2, len(names2)), ...].""" - aggregated_bins = defaultdict(set) - xs = set() - for (x, label), names in bins.items(): - xs.add(x) - aggregated_bins[label].update(names) + return sorted_meetings, datasets - xs = list(sorted(xs)) +def get_country_data_for_meetings(attendance_type=None): + """Get country participation data for meetings timeline chart. - sorted_bins = sorted(aggregated_bins.items(), key=lambda t: len(t[1]), reverse=True) - top = [ label for label, names in list(sorted_bins)[:limit]] + Args: + attendance_type: Optional filter for attendance type (e.g., 'onsite'). - for label in top: - series_data = [] + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = f'stats:get_country_data_for_meetings:{attendance_type}' + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + top_n = 10 # could be a parameter, but would need to adjust cache handling + # Get registration status counts, aggregated by country_code + if attendance_type: + registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) + else: + registrations = Registration.objects.all() + queryset = ( + registrations + .values( + 'meeting__number', # e.g. "118", "119", "120" + 'country_code' # country code of the participant + ) + .annotate(participant_count=Count('id')) + .order_by('meeting__number') # chronological order + ) + + # ── Step 1: Collect all meetings and country totals ── + meetings_set = set() + country_totals = defaultdict(int) + data_map = defaultdict(dict) # {country: {meeting: count}} + + for row in queryset: + meeting = row['meeting__number'] + country = row['country_code'] + count = row['participant_count'] + + meetings_set.add(meeting) + country_totals[country] += count + data_map[country][meeting] = count + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + + # ── Step 3: Get top N countries ── + top_countries = sorted( + country_totals.keys(), + key=lambda c: country_totals[c], + reverse=True + )[:top_n] + + # -- Step 3.bis do the 'other' category -- + non_top_countries = country_totals.keys() - top_countries + other_totals = defaultdict(int) + for m in sorted_meetings: + other_totals[m] = 0 + for c in non_top_countries: + other_totals[m] += int(data_map[c].get(m, 0)) + + # ── Step 4: Build Chart.js datasets ── + + datasets = [] + for idx, country in enumerate(top_countries): + color = colors[idx % len(colors)] + datasets.append({ + 'label': country, + 'data': [data_map[country].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'fill': False, + 'tension': 0.3, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + + # -- Step 4.bis handle the other -- + datasets.append({ + 'label': 'Other', + 'data': [other_totals.get(m, 0) for m in sorted_meetings], + 'borderColor': 'black', + 'fill': False, + 'tension': 0.3, + 'pointColor': 'black', + 'pointBackgroundColor': 'black', + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) - for x in xs: - names = bins.get((x, label), set()) + return sorted_meetings, datasets + +def get_data_for_meetings(): + """Get total participation data by attendance type for meetings timeline chart. + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = "stats:get_data_for_meetings" + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + # Get registration status counts, aggregated by ticket types + registrations = Registration.objects.filter(tickets__attendance_type__in=['onsite', 'remote']) + queryset = ( + registrations + .values( + 'meeting__number', # e.g. "118", "119", "120" + 'tickets__attendance_type' + ) + .annotate(participant_count=Count('id')) + .order_by('meeting__number') # chronological order + ) + + # ── Step 1: Collect all meetings and tickets totals ── + meetings_set = set() + tickets_totals = defaultdict(int) + data_map = defaultdict(dict) # {ticket: {meeting: count}} + + for row in queryset: + meeting = row['meeting__number'] + ticket = row['tickets__attendance_type'] + count = row['participant_count'] + + meetings_set.add(meeting) + tickets_totals[ticket] += count + data_map[ticket][meeting] = count + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + ticket_types = tickets_totals.keys() + + # ── Step 4: Build Chart.js datasets ── + # Color palette for lines + colors = [ '#FF6384', '#36A2EB'] + + datasets = [] + for idx, ticket_type in enumerate(ticket_types): + color = colors[idx % len(colors)] + datasets.append({ + 'label': ticket_type, + 'data': [data_map[ticket_type].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'backgroundColor': color + '99', # 60% opacity fill + 'fill': True, + 'tension': 0.0, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) + return sorted_meetings, datasets + +def meetings_timeline(request, stats_type='country'): + """Render the meetings timeline page with participation statistics over time. + + Args: + request: The HTTP request object. + stats_type: Type of statistics ('country' or 'total'). + top_n: Number of top items to show (for country stats). + + Returns: + Rendered response for the meetings timeline template. + """ + if stats_type == 'total': + total_labels, total_data_sets = get_data_for_meetings() + in_person_labels = ([], []) + in_person_data_sets = ([], []) + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + elif stats_type == 'affiliation': + total_labels, total_data_sets = get_affiliation_data_for_meetings() + in_person_labels, in_person_data_sets = get_affiliation_data_for_meetings(attendance_type='onsite') + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + elif stats_type == 'country': + total_labels, total_data_sets = get_country_data_for_meetings() + in_person_labels, in_person_data_sets = get_country_data_for_meetings(attendance_type='onsite') + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - series_data.append((x, len(names))) + total_chart_data = { + 'labels': total_labels, + 'datasets': total_data_sets, + } - chart_data.append({ - "data": series_data, - "name": label - }) + # On per country/affiliation have a separate graph for inperson + if stats_type == 'total': + in_person_chart_data = None + else: + in_person_chart_data = { + 'labels': in_person_labels, + 'datasets': in_person_data_sets, + } -def document_stats(request, stats_type=None): - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + # Prepare the list of choice buttons for the template + possible_stats_types = [ + ("affiliation", "Per affiliation", urlreverse(meetings_timeline, kwargs={'stats_type': 'affiliation'})), + ("country", "Per country", urlreverse(meetings_timeline, kwargs={'stats_type': 'country'})), + ("total", "Total", urlreverse(meetings_timeline, kwargs={'stats_type': 'total'})), + ] + current_meeting = get_current_ietf_meeting_num() + if stats_type == 'total': + possible_stats_type = 'country' + else: + possible_stats_type = stats_type -def known_countries_list(request, stats_type=None, acronym=None): - countries = CountryName.objects.prefetch_related("countryalias_set") - for c in countries: - # the sorting is a bit of a hack - it puts the ISO code first - # since it was added in a migration - c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) + possible_meeting_numbers = [(int(current_meeting)-1, urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting)-1, 'stats_type': possible_stats_type})), + (int(current_meeting), urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting), 'stats_type': possible_stats_type})), + (int(current_meeting)+1, urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting)+1, 'stats_type': possible_stats_type}))] - return render(request, "stats/known_countries_list.html", { - "countries": countries, + return render(request, "stats/meetings_timeline.html", { + "top_n": top_n, + "possible_stats_types": possible_stats_types, + "possible_meeting_numbers": possible_meeting_numbers, + "stats_type": stats_type, + "total_chart_data": total_chart_data, + "in_person_chart_data": in_person_chart_data, }) -def meeting_stats(request, num=None, stats_type=None): - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) +def get_affiliation_data_for_meeting(meeting_number, minimum_required, attendance_type=None): + """Get affiliation participation data for a specific meeting. + + Args: + meeting_number: The meeting number. + minimum_required: Minimum count to include in main data (others go to 'Other'). + attendance_type: Optional filter for attendance type. + + Returns: + Tuple of (labels, data, total) for chart display. + """ + # Get registration status details + registrations = Registration.objects.filter(meeting__number=meeting_number) + if attendance_type: + registrations = registrations.filter(tickets__attendance_type=attendance_type) + registrations = registrations.values('affiliation') + + # Count per canonicalized affiliation + organization = dict() + for reg in registrations: + affiliation = canonicalize_affiliation(reg['affiliation']) or "Unspecified" + organization[affiliation] = organization.get(affiliation, 0) + 1 + + # Sort to have the largest count first (nicer in pie chart) + sorted_orgs = sorted(organization.items(), key=lambda t: t[1], reverse=True) + labels = [] + data = [] + others_count = 0 + total = 0 + for org, count in sorted_orgs: + total += count + if count > minimum_required: + labels.append(org) + data.append(count) + else: + others_count += count + + if others_count > 0: + labels.append('Other') + data.append(others_count) + + return labels, data, total + +def get_data_for_meeting(meeting_number, minimum_required, attendance_type=None): + """Get country participation data for a specific meeting. + + Args: + meeting_number: The meeting number. + minimum_required: Minimum count to include in main data (others go to 'Other'). + attendance_type: Optional filter for attendance type. + + Returns: + Tuple of (labels, data, total) for chart display. + """ + # Get registration status counts, aggregated by country_code + registration_counts = Registration.objects.filter(meeting__number=meeting_number) + if attendance_type: + registration_counts = registration_counts.filter(tickets__attendance_type=attendance_type) + registration_counts = registration_counts.values('country_code').annotate(count=Count('country_code')).order_by('-count') + + labels = [] + data = [] + others_count = 0 + total = 0 + for item in registration_counts: + total += item['count'] + if item['count'] > minimum_required: + labels.append(item['country_code']) + data.append(item['count']) + else: + others_count += item['count'] + + if others_count > 0: + labels.append('Other') + data.append(others_count) + + return labels, data, total + +def meeting_stats(request, meeting_number=None, stats_type='country'): + """Render statistics for a specific meeting. + + Args: + request: The HTTP request object. + meeting_number: The meeting number (defaults to current). + stats_type: Type of statistics ('country' or 'affiliation'). + + Returns: + Rendered response for the meeting stats template. + """ + + current_meeting = get_current_ietf_meeting_num() + if meeting_number is None: + meeting_number = current_meeting + + this_meeting = get_ietf_meeting(meeting_number) + + if stats_type == 'affiliation': + minimum_required = 4 + total_labels, total_data, total_total = get_affiliation_data_for_meeting(meeting_number, minimum_required) + in_person_labels, in_person_data, in_person_total = get_affiliation_data_for_meeting(meeting_number, minimum_required, attendance_type='onsite') + elif stats_type == 'country': + minimum_required = 10 + total_labels, total_data, total_total = get_data_for_meeting(meeting_number, minimum_required) + in_person_labels, in_person_data, in_person_total = get_data_for_meeting(meeting_number, minimum_required, attendance_type='onsite') + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + + total_chart_data = { + 'labels': total_labels, + 'datasets': [{ + 'label': 'Total Registrations by ' + stats_type, + 'data': total_data, + 'borderColor': '#ffffff', + 'borderWidth': 2, + }] + } + in_person_chart_data = { + 'labels': in_person_labels, + 'datasets': [{ + 'label': 'In Person Registrations by ' + stats_type, + 'data': in_person_data, + 'borderColor': '#ffffff', + 'borderWidth': 2, + }] + } + + # Prepare the list of choice buttons for the template + possible_stats_types = [ + ("affiliation", "Per affiliation", urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': 'affiliation'})), + ("country", "Per country", urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': 'country'})), + ] + + # Prepare the list of meeting number buttons for the template + possible_meeting_numbers = [('All', urlreverse(meetings_timeline, kwargs={'stats_type': stats_type}))] + if int(meeting_number) > 72: # No registration data before IETF-72 + possible_meeting_numbers.append((int(meeting_number)-1, urlreverse(meeting_stats, kwargs={'meeting_number': int(meeting_number)-1, 'stats_type': stats_type}))) + possible_meeting_numbers.append((meeting_number, urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': stats_type}))) + if int(meeting_number) <= int(current_meeting): # Allow current meeting +1 + possible_meeting_numbers.append((int(meeting_number)+1, urlreverse(meeting_stats, kwargs={'meeting_number': int(meeting_number)+1, 'stats_type': stats_type}))) + + return render(request, "stats/meeting_stats.html", { + "meeting_number": meeting_number, + "meeting_date": this_meeting.date, + "meeting_country": this_meeting.country, + "meeting_city": this_meeting.city, + "possible_stats_types": possible_stats_types, + "possible_meeting_numbers": possible_meeting_numbers, + "stats_type": stats_type, + "minimum_required": minimum_required, + "total_chart_data": total_chart_data, + "total_total": total_total, + "in_person_chart_data": in_person_chart_data, + "in_person_total": in_person_total + }) @login_required def review_stats(request, stats_type=None, acronym=None): + """Render review statistics page with tables and charts for review assignments. + + Shows completion status, results, assignment states, and time series data. + Supports both team-level and reviewer-level views with filtering options. + + Args: + request: The HTTP request object. + stats_type: Type of statistics ('completion', 'results', 'states', 'time'). + acronym: Team acronym for reviewer-level view (None for team view). + + Returns: + Rendered response for the review stats template. + """ # This view is a bit complex because we want to show a bunch of # tables with various filtering options, and both a team overview # and a reviewers-within-team overview - and a time series chart. diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html index 8ff6e952daf..43ca025e28b 100644 --- a/ietf/templates/base/menu.html +++ b/ietf/templates/base/menu.html @@ -428,12 +428,11 @@ Downref registry
    • -
  • - +
  • + Statistics -
  • diff --git a/ietf/templates/stats/annual_report_inputs.html b/ietf/templates/stats/annual_report_inputs.html new file mode 100644 index 00000000000..15add7ece3c --- /dev/null +++ b/ietf/templates/stats/annual_report_inputs.html @@ -0,0 +1,55 @@ +{# Copyright The IETF Trust 2026, All Rights Reserved #} +{% extends "base.html" %} +{% load origin %} +{% load ietf_filters static %} +{% block content %} + {% origin %} +

    {% block title %}Annual Report Inputs for {{ year }}{% endblock %}

    +
    +
    + + + +
    +
    +

    Summary

    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Email addressesUnique persons foundAddresses with no person recordApproximate unique people
    Authors{{ author_count }}{{ author_person_count }}{{ author_noperson_count }}{{ author_person_count|add:author_noperson_count }}
    Submitters{{ submitter_count }}{{ submitter_person_count }}{{ submitter_noperson_count }}{{ submitter_person_count|add:submitter_noperson_count }}
    +

    Drafts submitted in {{ year }}: {{ draft_count }}

    +

    Downloads

    + +{% endblock %} diff --git a/ietf/utils/reports.py b/ietf/utils/reports.py new file mode 100755 index 00000000000..9a969a52178 --- /dev/null +++ b/ietf/utils/reports.py @@ -0,0 +1,49 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved + +from typing import List, Set, Tuple +from django.db.models import QuerySet + +from email.utils import parseaddr + +from ietf.person.models import Person +from ietf.submit.models import Submission + + +def authors_by_year(year: int) -> Set[str]: + """Email addresses provided by I-D authors for drafts that were submitted in the given year.""" + addresses = set() + for submission in Submission.objects.filter( + submission_date__year=year, state="posted" + ): + addresses.update([a["email"] for a in submission.authors]) + return addresses + + +def submitters_by_year(year: int) -> Set[str]: + """Email addresses provided by I-D submitters for drafts that were submitted in the given year.""" + return set( + [ + parseaddr(a)[1] + for a in Submission.objects.filter( + submitter__contains="@", submission_date__year=year, state="posted" + ).values_list("submitter", flat=True) + ] + ) + + +def unique_people(addresses: List[str]) -> Tuple["QuerySet[Person]", Set]: + """Identify Person records matching email addresses and email addresses with no Person record. + + Given a list of email addresses, return + ( + a list of unique Person records with a matching email address, + a list of unique email addresses with no matching Person record + ) + The sum of the lengths of these lists is a best-approximation for how + many unique people the list of addresses belong to. + """ + persons = Person.objects.filter(email__address__in=addresses).distinct() + known_email = set(persons.values_list("email__address", flat=True)) + return (persons, set(addresses) - set(known_email)) + + diff --git a/ietf/utils/tests_reports.py b/ietf/utils/tests_reports.py new file mode 100755 index 00000000000..83daa15cc14 --- /dev/null +++ b/ietf/utils/tests_reports.py @@ -0,0 +1,189 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime + +import debug # pyflakes:ignore + +from ietf.doc.factories import IndividualDraftFactory +from ietf.person.factories import EmailFactory +from ietf.person.models import Person, Email +from ietf.submit.factories import SubmissionFactory +from ietf.utils.reports import authors_by_year, submitters_by_year, unique_people +from ietf.utils.test_utils import TestCase + + +class ReportTests(TestCase): + def setUp(self): + super().setUp() + + # Build 5 drafts submitted across two years, with 6 unique authors, + # one of which has more than one email address (author0@example.com and + # author0@example.net). The drafts are submitted by two of the six authors, + # again using multiple addresses for author0, and two identities that are not authors. + # Then build a draft where the submission's submitter info doesn't contain an email + # address (we have those in the production database) to make sure that submitter isn't + # counted. + + self.make_draft_submission( + year=2020, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 1", "author1@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=3, + day=3, + submitter_name="NotanAuthor 0", + submitter_email="notanauthor0@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Note alternate email + ("Author 3", "author3@example.net"), + ], + ) + + self.make_draft_submission( + year=2020, + month=12, + day=31, + submitter_name="Author 3", + submitter_email="author3@example.net", + author_nameaddrs=[("Author 3", "author3@example.net")], + ) + + self.make_draft_submission( + year=2021, + month=1, + day=1, + submitter_name="Author 0", + submitter_email="author0@example.com", # Note alternate email + author_nameaddrs=[ + ("Author 0", "author0@example.com"), # Again, alternate email + ("Author 4", "author4@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="NoatanAuthor 2", + submitter_email="notanauthor2@example.net", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 3", "author3@example.net"), + ("Author 5", "author5@example.net"), + ], + ) + + self.make_draft_submission( + year=2021, + month=12, + day=31, + submitter_name="Trouble Maker", + submitter_email="", + author_nameaddrs=[ + ("Author 0", "author0@example.net"), + ("Author 2", "author2@example.net"), + ], + ) + + def make_draft_submission( + self, year, month, day, submitter_name, submitter_email, author_nameaddrs + ): + + authors = [] + for name, addr in author_nameaddrs: + person = Person.objects.filter(name=name).first() + if not person: + person = EmailFactory(person__name=name, address=addr).person + elif not Email.objects.filter(address=addr).exists(): + EmailFactory(person=person, address=addr) + authors.append(person) + + submission = SubmissionFactory( + submission_date=datetime.date(year, month, day), + submitter_name=submitter_name, + submitter_email=submitter_email, + state_id="posted", + ) + submission.authors = [ + { + "name": f"{name}", + "email": f"{addr}", + "affiliation": "", + "country": "", + "errors": [], + } + for name, addr in author_nameaddrs + ] + + submission.save() + IndividualDraftFactory(submission=submission, authors=authors) + + def test_authors_by_year(self): + authors2020 = authors_by_year(2020) + self.assertEqual( + authors2020, + set( + [ + "author0@example.net", + "author0@example.com", + "author1@example.net", + "author3@example.net", + ] + ), + ) + authors2021 = authors_by_year(2021) + self.assertEqual( + authors2021, + set( + [ + "author0@example.net", + "author0@example.com", + "author2@example.net", + "author3@example.net", + "author4@example.net", + "author5@example.net", + ] + ), + ) + + def test_submitters_by_year(self): + sub2020 = submitters_by_year(2020) + self.assertEqual( + sub2020, + set( + [ + "author0@example.net", + "author3@example.net", + "notanauthor0@example.net", + ] + ), + ) + sub2021 = submitters_by_year(2021) + self.assertEqual( + sub2021, set(["author0@example.com", "notanauthor2@example.net"]) + ) + + def test_unique_people(self): + persons, addrs = unique_people( + [ + "notanauthor0@example.com", + "author0@example.net", + "author0@example.com", + "author1@example.net", + "notanauthor0@example.com", + ] + ) + self.assertEqual(addrs, set(["notanauthor0@example.com"])) + self.assertEqual( + set(persons), set(Person.objects.filter(name__in=("Author 0", "Author 1"))) + ) + self.assertEqual(len(persons) + len(addrs), 3) From 7dff4a7af8844a22a302a579ace59c8372a5348b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 8 Jun 2026 16:22:23 -0300 Subject: [PATCH 276/313] ci: remove strategy from dt manifests (#11004) --- k8s/auth.yaml | 2 -- k8s/datatracker.yaml | 2 -- 2 files changed, 4 deletions(-) diff --git a/k8s/auth.yaml b/k8s/auth.yaml index 6e63001e02b..ef8c259933b 100644 --- a/k8s/auth.yaml +++ b/k8s/auth.yaml @@ -8,8 +8,6 @@ spec: selector: matchLabels: app: auth - strategy: - type: DEPLOY_STRATEGY template: metadata: labels: diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index af2bb6295c9..5183893bc8c 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -8,8 +8,6 @@ spec: selector: matchLabels: app: datatracker - strategy: - type: DEPLOY_STRATEGY template: metadata: labels: From 7e3e3cb503aa9ec2f757cabab85c944771ade8f9 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 9 Jun 2026 18:51:19 -0500 Subject: [PATCH 277/313] fix: tweak logging settings to avoid spurious keyerrors (#11009) --- ietf/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ietf/settings.py b/ietf/settings.py index 95f2ffefd76..1286e70e75c 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -329,7 +329,8 @@ def skip_unreadable_post(record): "formatters": { "django.server": { "()": "django.utils.log.ServerFormatter", - "format": "[%(server_time)s] %(message)s", + "format": "[{server_time}] {message}", + "style": "{", }, "plain": { "style": "{", From c342bef7938e2ba56b23984c17b3af740c52ac8e Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 9 Jun 2026 19:08:23 -0500 Subject: [PATCH 278/313] fix: improved source and keywords for rfc json (#11008) --- ietf/doc/tests_utils_rfc_json.py | 58 ++++++++++++++++++++++++++++++-- ietf/doc/utils_rfc_json.py | 20 +++++------ 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/ietf/doc/tests_utils_rfc_json.py b/ietf/doc/tests_utils_rfc_json.py index b94fc2cbea1..075b50a24f3 100644 --- a/ietf/doc/tests_utils_rfc_json.py +++ b/ietf/doc/tests_utils_rfc_json.py @@ -13,6 +13,7 @@ PublishedRfcDocEventFactory, RfcAuthorFactory, RfcFactory, + RgRfcFactory, WgRfcFactory, ) from ietf.doc.models import RelatedDocument @@ -245,6 +246,19 @@ def test_april_first_date_format(self): self.assertEqual(data["pub_date"], "1 April 2020") + def test_empty_keywords_filtered(self): + """Empty-string keywords are stripped from the keywords list.""" + rfc = PublishedRfcDocEventFactory( + doc=WgRfcFactory(keywords=["foo", "", "bar"]), + ).doc + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["keywords"], ["foo", "bar"]) + def test_non_april_first_april_date(self): """An April publication that is NOT in the April Fools list gets 'April YYYY'.""" rfc = PublishedRfcDocEventFactory( @@ -260,7 +274,7 @@ def test_non_april_first_april_date(self): self.assertEqual(data["pub_date"], "April 2020") def test_source_ietf_wg(self): - """IETF-stream WG RFC: source is 'acronym (area)'.""" + """IETF-stream WG RFC: source is the group's full name.""" area = GroupFactory(type_id="area") wg = GroupFactory(type_id="wg", parent=area) rfc = PublishedRfcDocEventFactory( @@ -272,7 +286,7 @@ def test_source_ietf_wg(self): generate_rfc_json(rfc.rfc_number) data = _read_json(rfc.rfc_number) - self.assertEqual(data["source"], f"{wg.acronym} ({area.acronym})") + self.assertEqual(data["source"], wg.name) def test_source_ietf_no_wg(self): """IETF-stream individual RFC (group acronym 'none'): source is 'IETF - NON WORKING GROUP'.""" @@ -293,6 +307,22 @@ def test_source_ietf_no_wg(self): self.assertEqual(data["source"], "IETF - NON WORKING GROUP") + def test_source_ietf_area(self): + """IETF-stream RFC with area-type group: source is 'IETF - NON WORKING GROUP'.""" + area = GroupFactory(type_id="area") + area.parent = GroupFactory() + area.save() + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(group=area, stream_id="ietf"), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IETF - NON WORKING GROUP") + def test_source_iab(self): """IAB-stream RFC: source is 'IAB'.""" rfc = PublishedRfcDocEventFactory( @@ -319,6 +349,30 @@ def test_source_ise(self): self.assertEqual(data["source"], "INDEPENDENT") + def test_source_irtf_rg(self): + """IRTF-stream RG RFC: source is the group's full name.""" + rfc = PublishedRfcDocEventFactory(doc=RgRfcFactory()).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], rfc.group.name) + + def test_source_irtf_no_rg(self): + """IRTF-stream RFC with no specific RG (group acronym 'none'): source is 'IRTF'.""" + rfc = PublishedRfcDocEventFactory( + doc=RfcFactory(stream_id="irtf", group=GroupFactory(acronym="none")), + ).doc + _put_pub_levels(rfc.rfc_number, "inf") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["source"], "IRTF") + def test_pub_levels_passed_in(self): """When pub_levels is passed in, get_publication_std_levels() is not called.""" rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc diff --git a/ietf/doc/utils_rfc_json.py b/ietf/doc/utils_rfc_json.py index 6030a7064c5..af916e0d7fc 100644 --- a/ietf/doc/utils_rfc_json.py +++ b/ietf/doc/utils_rfc_json.py @@ -105,7 +105,6 @@ def generate_rfc_json(rfc_number: int, *, pub_levels=None) -> None: stream_slug = rfc.stream.slug group_acronym = rfc.group.acronym - area_acronym = None if stream_slug == "ietf": if rfc.group.parent is None: assertion("rfc.group.parent is not None") @@ -113,23 +112,22 @@ def generate_rfc_json(rfc_number: int, *, pub_levels=None) -> None: f"Malformed document object encountered for rfc{rfc_number}. Aborting update of rfc{rfc_number}.json" ) return - else: - area_acronym = rfc.group.parent.acronym if stream_slug == "ise": source = "INDEPENDENT" elif stream_slug == "iab": source = "IAB" elif stream_slug == "ietf" and ( - group_acronym in ("none", "gen") or not area_acronym + group_acronym == "none" or rfc.group.type_id == "area" ): source = "IETF - NON WORKING GROUP" - elif group_acronym not in ("none", ""): - source = group_acronym - if stream_slug == "ietf" and area_acronym: - source += f" ({area_acronym})" - elif stream_slug: - source += f" ({stream_slug})" + elif stream_slug == "irtf": + if group_acronym == "none": + source = "IRTF" + else: + source = rfc.group.name + elif group_acronym not in ("none", "") and stream_slug in ["ietf", "editorial"]: + source = rfc.group.name elif stream_slug: source = "Legacy" if stream_slug == "legacy" else stream_slug.upper() else: @@ -207,7 +205,7 @@ def _rfc_list(qs, attr): "source": source, "abstract": rfc.abstract, "pub_date": pub_date, - "keywords": rfc.keywords, + "keywords": [kw for kw in rfc.keywords if kw], "obsoletes": obsoletes, "obsoleted_by": obsoleted_by, "updates": updates, From 1cccb088947cab50a5b8c837ed584f2396de2cdd Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 9 Jun 2026 19:09:36 -0500 Subject: [PATCH 279/313] test: make check for April 1 resilient across line-breaks (#11006) * test: make check for April 1 resilient across line-breaks * test: be resilient against faker creating a 63 character long title --- ietf/sync/tests_rfcindex.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index 2b70924db39..1a45d5d9cf4 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -139,7 +139,10 @@ def test_create_rfc_txt_index(self, mock_save_blob, mock_save_file): f"{self.april_fools_rfc.rfc_number} {self.april_fools_rfc.title}", stripped_contents, ) - self.assertIn("1 April 2020", contents) # from the April 1 RFC + # "1 April 2020" may be split across a line wrap (e.g. "1 April\n 2020") + # when the randomly-generated title is long enough to push the date off the line. + # assertRegex handles both wrapped and non-wrapped cases explicitly. + self.assertRegex(contents, r"1\s+April\s+2020") # from the April 1 RFC self.assertIn( f"{self.rfc.rfc_number} {self.rfc.title}", stripped_contents, From 28a5e967725b8569bde7ca09821b182a5bcbe9b5 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 10 Jun 2026 12:08:45 -0300 Subject: [PATCH 280/313] chore(dev): fix nginx config (#11014) keepalive 0 is not supported by the version of nginx in the dev container --- docker/configs/nginx-proxy.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/configs/nginx-proxy.conf b/docker/configs/nginx-proxy.conf index a02ab2ff06a..0a9dde04eb4 100644 --- a/docker/configs/nginx-proxy.conf +++ b/docker/configs/nginx-proxy.conf @@ -1,6 +1,7 @@ upstream datatracker_backend { server 127.0.0.1:8001; - keepalive 0; # default = 32 since nginx 1.29.7 +# Uncomment when changing to nginx 1.29.7 or later. +# keepalive 0; # default = 32 since nginx 1.29.7 } server { From 6b185be2961952e25a3ec226310593a4544a4a10 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 10 Jun 2026 18:39:32 -0500 Subject: [PATCH 281/313] fix: sort the search results document column correctly (#11015) * fix: sort the group document's document column correcty * test: confirm column sort information is present --- ietf/group/tests_info.py | 9 ++++++++- ietf/templates/doc/search/search_result_row.html | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 3f24e2e3d6b..97ec7ebdb15 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -27,7 +27,7 @@ from ietf.community.models import CommunityList from ietf.community.utils import reset_name_contains_index_for_rule -from ietf.doc.factories import WgDraftFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory +from ietf.doc.factories import WgDraftFactory, WgRfcFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory from ietf.doc.models import Document, DocEvent, State from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils_charter import charter_name_for_group @@ -396,6 +396,7 @@ def test_group_documents(self): draft7 = WgDraftFactory(group=group) draft7.set_state(State.objects.get(type='draft', slug='expired')) draft7.set_state(State.objects.get(type='draft-stream-%s' % draft7.stream_id, slug='dead')) # Expired WG draft, marked as dead + rfc = WgRfcFactory(group=group) clist = CommunityList.objects.get(group=group) related_docs_rule = clist.searchrule_set.get(rule_type='name_contains') @@ -426,6 +427,12 @@ def test_group_documents(self): q = PyQuery(r.content) self.assertTrue(any([draft2.name in x.attrib['href'] for x in q('table td a.track-untrack-doc')])) + # RFC rows must use the RFC number as the sort key so that numeric sort + # is not disrupted by the page-count text that precedes the name in the cell. + # Draft rows must use the document name. + self.assertTrue(q(f'td.doc[data-sort-number="{rfc.rfc_number}"]')) + self.assertTrue(q(f'td.doc[data-sort-number="{draft.name}"]')) + # Let's also check the IRTF stream rg = GroupFactory(type_id='rg') setup_default_community_list_for_group(rg) diff --git a/ietf/templates/doc/search/search_result_row.html b/ietf/templates/doc/search/search_result_row.html index 476c81f5986..cb50a1b214e 100644 --- a/ietf/templates/doc/search/search_result_row.html +++ b/ietf/templates/doc/search/search_result_row.html @@ -45,7 +45,7 @@ {% endfor %} - + {% if doc.pages %}{{ doc.pages }} page{{ doc.pages|pluralize }}{% endif %}
    From f77514e98f76d9cc263c829f616bc6822864b01b Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 10 Jun 2026 18:41:03 -0500 Subject: [PATCH 282/313] fix: correct sorting in nomcom feedback tables (#11016) --- ietf/nomcom/tests.py | 14 ++++++++++++++ ietf/templates/nomcom/view_feedback.html | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index 210788ce079..28152ef79b1 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -1640,6 +1640,20 @@ def test_feedback_topic_badges(self): q = PyQuery(response.content) self.assertEqual( len(q('.text-bg-success')), 0 ) + def test_feedback_index_sort_keys(self): + url = reverse('ietf.nomcom.views.view_feedback', kwargs={'year': self.nc.year()}) + login_testing_unauthorized(self, self.member.user.username, url) + provide_private_key_to_test_client(self) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + q = PyQuery(response.content) + # Feedback count cells must carry a numeric data-sort-number so that + # a "New" badge appearing before the count doesn't corrupt the sort key. + sort_cells = q('td[data-sort-number]') + self.assertTrue(len(sort_cells) > 0) + for cell in sort_cells.items(): + self.assertRegex(cell.attr('data-sort-number'), r'^\d+$') + class NewActiveNomComTests(TestCase): def setUp(self): diff --git a/ietf/templates/nomcom/view_feedback.html b/ietf/templates/nomcom/view_feedback.html index 93d61b7f420..0758329d606 100644 --- a/ietf/templates/nomcom/view_feedback.html +++ b/ietf/templates/nomcom/view_feedback.html @@ -43,7 +43,7 @@

    Declined each nominated position

     {% for fbtype_name, fbtype_count, fbtype_newflag in fb_dict.feedback %} - + {% if fbtype_newflag %}New{% endif %} {{ fbtype_count }} @@ -82,7 +82,7 @@

    Feedback related to topics

    {% for fbtype_name, fbtype_count, fbtype_newflag in fb_dict.feedback %} - + {% if fbtype_newflag %}New{% endif %} {{ fbtype_count }} From 65dcdc942956d5a9dc21bf7029664b04d49c8d00 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 10 Jun 2026 20:42:58 -0300 Subject: [PATCH 283/313] refactor: use correct red bucket paths by default (#11007) * refactor: RFCINDEX_INPUT_PATH dflt -> prod value * refactor: RFCINDEX_OUTPUT_PATH dflt -> prod value * ci: only change RFCINDEX_* paths if set * refactor: clearer settings * test: update / fix / add tests --- ietf/settings.py | 4 +++ ietf/sync/rfcindex.py | 19 ++++++++------ ietf/sync/tests_rfcindex.py | 50 +++++++++++++++++++++++++++++-------- k8s/settings_local.py | 6 +++-- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index 1286e70e75c..d509a877c61 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -977,6 +977,10 @@ def skip_unreadable_post(record): ) RFC_FILE_TYPES = IDSUBMIT_FILE_TYPES +# Paths in the red bucket +RFCINDEX_INPUT_PATH = "other/" +RFCINDEX_OUTPUT_PATH = "other/" + IDSUBMIT_MAX_DRAFT_SIZE = { 'txt': 2*1024*1024, # Max size of txt draft file in bytes 'xml': 3*1024*1024, # Max size of xml draft file in bytes diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py index be55a6866e3..f47974f9002 100644 --- a/ietf/sync/rfcindex.py +++ b/ietf/sync/rfcindex.py @@ -48,9 +48,17 @@ def errata_url(rfc: Document): return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}") +def red_bucket_input_path(filename: str) -> str: + return str(Path(settings.RFCINDEX_INPUT_PATH) / filename) + + +def red_bucket_output_path(filename: str) -> str: + return str(Path(settings.RFCINDEX_OUTPUT_PATH) / filename) + + def save_to_red_bucket(filename: str, content: str | bytes): red_bucket = storages["red_bucket"] - bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename) + bucket_path = red_bucket_output_path(filename) if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True): # Django 4.2's FileSystemStorage does not support allow_overwrite. red_bucket.delete(bucket_path) @@ -87,8 +95,7 @@ class UnusableRfcNumber: def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: - FILENAME = "unusable-rfc-numbers.json" - bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + bucket_path = red_bucket_input_path("unusable-rfc-numbers.json") try: with storages["red_bucket"].open(bucket_path) as urn_file: records = json.load(urn_file) @@ -115,8 +122,7 @@ def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: def get_april1_rfc_numbers() -> Container[int]: - FILENAME = "april-first-rfc-numbers.json" - bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + bucket_path = red_bucket_input_path("april-first-rfc-numbers.json") try: with storages["red_bucket"].open(bucket_path) as urn_file: records = json.load(urn_file) @@ -139,8 +145,7 @@ def get_april1_rfc_numbers() -> Container[int]: def get_publication_std_levels() -> dict[int, StdLevelName]: - FILENAME = "publication-std-levels.json" - bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + bucket_path = red_bucket_input_path("publication-std-levels.json") values: dict[int, StdLevelName] = {} try: with storages["red_bucket"].open(bucket_path) as urn_file: diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py index 1a45d5d9cf4..226b41af946 100644 --- a/ietf/sync/tests_rfcindex.py +++ b/ietf/sync/tests_rfcindex.py @@ -28,9 +28,11 @@ get_april1_rfc_numbers, get_publication_std_levels, get_unusable_rfc_numbers, + red_bucket_input_path, + red_bucket_output_path, + save_to_filesystem, save_to_red_bucket, subseries_text_line, - save_to_filesystem, ) from ietf.utils.test_utils import TestCase @@ -401,25 +403,47 @@ def test_create_fyi_txt_index(self, mock_save_blob, mock_save_file): ) +@override_settings(RFCINDEX_INPUT_PATH="input/", RFCINDEX_OUTPUT_PATH="output/") class HelperTests(TestCase): + INPUT_PATH = "input" + OUTPUT_PATH = "output" + def test_format_rfc_number(self): self.assertEqual(format_rfc_number(10), "10") with override_settings(RFCINDEX_MATCH_LEGACY_XML=True): self.assertEqual(format_rfc_number(10), "0010") + def test_red_bucket_input_path(self): + with override_settings(RFCINDEX_INPUT_PATH="bar"): + self.assertEqual(red_bucket_input_path("foo"), "bar/foo") + with override_settings(RFCINDEX_INPUT_PATH="bar/"): + self.assertEqual(red_bucket_input_path("foo"), "bar/foo") + + def test_red_bucket_output_path(self): + self.assertEqual(red_bucket_input_path("foo"), f"{self.INPUT_PATH}/foo") + with override_settings(RFCINDEX_OUTPUT_PATH="bar"): + self.assertEqual(red_bucket_output_path("foo"), "bar/foo") + with override_settings(RFCINDEX_OUTPUT_PATH="bar/"): + self.assertEqual(red_bucket_output_path("foo"), "bar/foo") + def test_save_to_red_bucket(self): red_bucket = storages["red_bucket"] with override_settings(RFCINDEX_DELETE_THEN_WRITE=False): save_to_red_bucket("test", "contents \U0001f600") # Read as binary and explicitly decode to confirm encoding - with red_bucket.open("test", "rb") as f: + with red_bucket.open(f"{self.OUTPUT_PATH}/test", "rb") as f: self.assertEqual(f.read().decode("utf-8"), "contents \U0001f600") with override_settings(RFCINDEX_DELETE_THEN_WRITE=True): save_to_red_bucket("test", "new contents \U0001fae0".encode("utf-8")) # Read as binary and explicitly decode to confirm encoding - with red_bucket.open("test", "rb") as f: + with red_bucket.open(f"{self.OUTPUT_PATH}/test", "rb") as f: self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0") - red_bucket.delete("test") # clean up like a good child + red_bucket.delete(f"{self.OUTPUT_PATH}/test") # clean up like a good child + # check that we can override the path + with override_settings(RFCINDEX_OUTPUT_PATH="fruit"): + save_to_red_bucket("test", "content") + self.assertTrue(red_bucket.exists("fruit/test")) + red_bucket.delete("fruit/test") # clean up like a good child def test_save_to_filesystem(self): rfc_path = Path(settings.RFC_PATH) @@ -445,30 +469,36 @@ def test_get_unusable_rfc_numbers_raises(self): with self.assertRaises(FileNotFoundError): get_unusable_rfc_numbers() red_bucket = storages["red_bucket"] - red_bucket.save("unusable-rfc-numbers.json", ContentFile("not json")) + red_bucket.save( + f"{self.INPUT_PATH}/unusable-rfc-numbers.json", ContentFile("not json") + ) with self.assertRaises(json.JSONDecodeError): get_unusable_rfc_numbers() - red_bucket.delete("unusable-rfc-numbers.json") + red_bucket.delete(f"{self.INPUT_PATH}/unusable-rfc-numbers.json") def test_get_april1_rfc_numbers_raises(self): """get_april1_rfc_numbers should bail on errors""" with self.assertRaises(FileNotFoundError): get_april1_rfc_numbers() red_bucket = storages["red_bucket"] - red_bucket.save("april-first-rfc-numbers.json", ContentFile("not json")) + red_bucket.save( + f"{self.INPUT_PATH}/april-first-rfc-numbers.json", ContentFile("not json") + ) with self.assertRaises(json.JSONDecodeError): get_april1_rfc_numbers() - red_bucket.delete("april-first-rfc-numbers.json") + red_bucket.delete(f"{self.INPUT_PATH}/april-first-rfc-numbers.json") def test_get_publication_std_levels_raises(self): """get_publication_std_levels should bail on errors""" with self.assertRaises(FileNotFoundError): get_publication_std_levels() red_bucket = storages["red_bucket"] - red_bucket.save("publication-std-levels.json", ContentFile("not json")) + red_bucket.save( + f"{self.INPUT_PATH}/publication-std-levels.json", ContentFile("not json") + ) with self.assertRaises(json.JSONDecodeError): get_publication_std_levels() - red_bucket.delete("publication-std-levels.json") + red_bucket.delete(f"{self.INPUT_PATH}/publication-std-levels.json") def test_subseries_text_line(self): text = "foobar" diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 20c5252ff03..5dc31bac0e5 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -472,8 +472,10 @@ def _multiline_to_list(s): ), } RFCINDEX_DELETE_THEN_WRITE = False # S3Storage allows file_overwrite by default -RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/") -RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH", "") +if "DATATRACKER_RFCINDEX_OUTPUT_PATH" in os.environ: + RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH") +if "DATATRACKER_RFCINDEX_INPUT_PATH" in os.environ: + RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH") # Configure the blobdb app for artifact storage _blobdb_replication_enabled = ( From e79b0575cd36273837a2e6f2d82847889cf34541 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 11 Jun 2026 12:27:41 -0500 Subject: [PATCH 284/313] fix: include draft revision in rfc json (#11024) --- ietf/doc/tests_utils_rfc_json.py | 14 ++++++++++++++ ietf/doc/utils_rfc_json.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/ietf/doc/tests_utils_rfc_json.py b/ietf/doc/tests_utils_rfc_json.py index 075b50a24f3..ec3cd85893f 100644 --- a/ietf/doc/tests_utils_rfc_json.py +++ b/ietf/doc/tests_utils_rfc_json.py @@ -14,6 +14,7 @@ RfcAuthorFactory, RfcFactory, RgRfcFactory, + WgDraftFactory, WgRfcFactory, ) from ietf.doc.models import RelatedDocument @@ -199,6 +200,19 @@ def test_errata_url_set_when_errata_exist(self): f"https://www.rfc-editor.org/errata/rfc{rfc.rfc_number}", ) + def test_draft_field_includes_revision(self): + """draft field is '-' when the RFC originated from a draft.""" + draft = WgDraftFactory(rev="07") + rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc + draft.relateddocument_set.create(relationship_id="became_rfc", target=rfc) + _put_pub_levels(rfc.rfc_number, "ps") + _put_empty_errata() + + generate_rfc_json(rfc.rfc_number) + data = _read_json(rfc.rfc_number) + + self.assertEqual(data["draft"], f"{draft.name}-07") + def test_errata_url_none_when_no_errata(self): """errata_url is None when errata.json has no entries for the RFC.""" rfc = PublishedRfcDocEventFactory(doc=WgRfcFactory()).doc diff --git a/ietf/doc/utils_rfc_json.py b/ietf/doc/utils_rfc_json.py index af916e0d7fc..1f134556869 100644 --- a/ietf/doc/utils_rfc_json.py +++ b/ietf/doc/utils_rfc_json.py @@ -55,7 +55,7 @@ def generate_rfc_json(rfc_number: int, *, pub_levels=None) -> None: # draft name draft_doc = rfc.came_from_draft() - draft = draft_doc.name if draft_doc else None + draft = f"{draft_doc.name}-{draft_doc.rev}" if draft_doc else None # authors: ordered list of display strings authors = [] From 7666ae23c88671eb8f72b13ac72156c7046d5039 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Thu, 11 Jun 2026 18:19:52 -0400 Subject: [PATCH 285/313] ci: fix lock-threads workflow --- .github/workflows/lock-threads.yml | 2 +- .github/workflows/tests-az.yml | 109 ----------------------------- 2 files changed, 1 insertion(+), 110 deletions(-) delete mode 100644 .github/workflows/tests-az.yml diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml index 22652dab889..ab2c1a7cf6b 100644 --- a/.github/workflows/lock-threads.yml +++ b/.github/workflows/lock-threads.yml @@ -16,7 +16,7 @@ jobs: action: runs-on: ubuntu-latest steps: - - uses: ietf-tools/lock-threads@v3.1.1 + - uses: dessant/lock-threads@v6 with: github-token: ${{ github.token }} issue-inactive-days: 7 diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml deleted file mode 100644 index 833ca89befd..00000000000 --- a/.github/workflows/tests-az.yml +++ /dev/null @@ -1,109 +0,0 @@ -name: Tests (Azure Test) - -on: - workflow_dispatch: - -jobs: - main: - name: Run Tests on Azure temp VM - runs-on: ubuntu-latest - - permissions: - contents: read - - steps: - - name: Launch VM on Azure - id: azlaunch - run: | - echo "Authenticating to Azure..." - az login --service-principal -u ${{ secrets.AZ_TESTS_APP_ID }} -p ${{ secrets.AZ_TESTS_PWD }} --tenant ${{ secrets.AZ_TESTS_TENANT_ID }} - echo "Creating VM..." - vminfo=$(az vm create \ - --resource-group ghaDatatrackerTests \ - --name tmpGhaVM2 \ - --image Ubuntu2204 \ - --admin-username azureuser \ - --generate-ssh-keys \ - --priority Spot \ - --size Standard_D4as_v5 \ - --max-price -1 \ - --os-disk-size-gb 30 \ - --eviction-policy Delete \ - --nic-delete-option Delete \ - --output tsv \ - --query "publicIpAddress") - echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT" - echo "VM Public IP: $vminfo" - cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key - ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts - - - name: Remote SSH into VM - uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - host: ${{ steps.azlaunch.outputs.ipaddr }} - port: 22 - username: azureuser - command_timeout: 60m - key_path: ${{ github.workspace }}/prvkey.key - envs: GITHUB_TOKEN - script_stop: true - script: | - export DEBIAN_FRONTEND=noninteractive - lsb_release -a - sudo apt-get update - sudo apt-get upgrade -y - - echo "Installing Docker..." - curl -fsSL https://get.docker.com -o get-docker.sh - sudo sh get-docker.sh - - echo "Starting Containers..." - sudo docker network create dtnet - sudo docker run -d --name db --network=dtnet ghcr.io/ietf-tools/datatracker-db:latest & - sudo docker run -d --name app --network=dtnet ghcr.io/ietf-tools/datatracker-app-base:latest sleep infinity & - wait - - echo "Cloning datatracker repo..." - sudo docker exec app git clone --depth=1 https://github.com/ietf-tools/datatracker.git . - echo "Prepare tests..." - sudo docker exec app chmod +x ./dev/tests/prepare.sh - sudo docker exec app sh ./dev/tests/prepare.sh - echo "Running checks..." - sudo docker exec app ietf/manage.py check - sudo docker exec app ietf/manage.py migrate --fake-initial - echo "Running tests..." - sudo docker exec app ietf/manage.py test -v2 --validate-html-harder --settings=settings_test - - - name: Destroy VM + resources - if: always() - shell: pwsh - run: | - echo "Destroying VM..." - az vm delete -g ghaDatatrackerTests -n tmpGhaVM2 --yes --force-deletion true - - $resourceOrderRemovalOrder = [ordered]@{ - "Microsoft.Compute/virtualMachines" = 0 - "Microsoft.Compute/disks" = 1 - "Microsoft.Network/networkInterfaces" = 2 - "Microsoft.Network/publicIpAddresses" = 3 - "Microsoft.Network/networkSecurityGroups" = 4 - "Microsoft.Network/virtualNetworks" = 5 - } - echo "Fetching remaining resources..." - $resources = az resource list --resource-group ghaDatatrackerTests | ConvertFrom-Json - - $orderedResources = $resources - | Sort-Object @{ - Expression = {$resourceOrderRemovalOrder[$_.type]} - Descending = $False - } - - echo "Deleting remaining resources..." - $orderedResources | ForEach-Object { - az resource delete --resource-group ghaDatatrackerTests --ids $_.id --verbose - } - - echo "Logout from Azure..." - az logout From 65e7d51cd4ada4c6c817b9348f5671f45633f899 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Thu, 11 Jun 2026 18:22:12 -0400 Subject: [PATCH 286/313] ci: fix lock-threads workflow --- .github/workflows/lock-threads.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lock-threads.yml b/.github/workflows/lock-threads.yml index ab2c1a7cf6b..b90300c09b2 100644 --- a/.github/workflows/lock-threads.yml +++ b/.github/workflows/lock-threads.yml @@ -19,6 +19,7 @@ jobs: - uses: dessant/lock-threads@v6 with: github-token: ${{ github.token }} + process-only: 'issues, prs' issue-inactive-days: 7 pr-inactive-days: 3 log-output: true From ecebc399dda88494a04ccb59fd8ec24fa4473a4d Mon Sep 17 00:00:00 2001 From: NGPixel Date: Thu, 11 Jun 2026 22:23:03 -0400 Subject: [PATCH 287/313] ci: refactor build + deploy workflows --- .github/workflows/build.yml | 349 +++--------------- .github/workflows/ci-run-tests.yml | 2 +- .github/workflows/release.yml | 209 +++++++++++ .github/workflows/reusable-build.yml | 226 ++++++++++++ .../{tests.yml => reusable-tests.yml} | 0 5 files changed, 489 insertions(+), 297 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/reusable-build.yml rename .github/workflows/{tests.yml => reusable-tests.yml} (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a89bac46e74..2c449b53ce8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,7 @@ -name: Build and Release -run-name: ${{ github.ref_name == 'release' && '[Prod]' || '[Dev]' }} Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} +name: Build Dev and Deploy +run-name: Dev Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} on: - push: - branches: [release] - workflow_dispatch: inputs: deploy: @@ -14,24 +11,28 @@ on: type: choice options: - Skip - - Staging Only - - Staging + Prod - dev: - description: 'Deploy to Dev' - default: true + - Dev + - Staging + deployStrategy: + description: 'Deploy Strategy' + default: 'Recreate' required: true - type: boolean + type: choice + options: + - Recreate + - RollingUpdate + - Current devNoDbRefresh: description: 'Dev Disable Daily DB Refresh' default: false required: true type: boolean - skiptests: + skipTests: description: 'Skip Tests' default: false required: true type: boolean - skiparm: + skipArmBuild: description: 'Skip ARM64 Build' default: false required: true @@ -56,14 +57,12 @@ jobs: # PREPARE # ----------------------------------------------------------------- prepare: - name: Prepare Release + name: Prepare runs-on: ubuntu-latest outputs: - should_deploy: ${{ steps.buildvars.outputs.should_deploy }} - pkg_version: ${{ steps.buildvars.outputs.pkg_version }} - from_tag: ${{ steps.semver.outputs.nextStrict }} - to_tag: ${{ steps.semver.outputs.current }} - base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} + buildVersion: ${{ steps.buildvars.outputs.buildVersion }} + previousVersion: ${{ steps.semver.outputs.current }} + baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} steps: - uses: actions/checkout@v6 @@ -71,19 +70,8 @@ jobs: fetch-depth: 1 fetch-tags: false - - name: Get Next Version (Prod) - if: ${{ github.ref_name == 'release' }} - id: semver - uses: ietf-tools/semver-action@v1 - with: - token: ${{ github.token }} - branch: release - skipInvalidTags: true - patchList: fix, bugfix, perf, refactor, test, tests, chore - - name: Get Dev Version - if: ${{ github.ref_name != 'release' }} - id: semverdev + id: semver uses: ietf-tools/semver-action@v1 with: token: ${{ github.token }} @@ -91,260 +79,53 @@ jobs: skipInvalidTags: true noVersionBumpBehavior: 'current' noNewCommitBehavior: 'current' - - - name: Set Release Flag - if: ${{ github.ref_name == 'release' }} - run: | - echo "IS_RELEASE=true" >> $GITHUB_ENV - - - name: Create Draft Release - uses: ncipollo/release-action@v1.21.0 - if: ${{ github.ref_name == 'release' }} - with: - prerelease: true - draft: false - commit: ${{ github.sha }} - tag: ${{ steps.semver.outputs.nextStrict }} - name: ${{ steps.semver.outputs.nextStrict }} - body: '*pending*' - token: ${{ secrets.GITHUB_TOKEN }} - name: Set Build Variables id: buildvars run: | - if [[ $IS_RELEASE ]]; then - echo "Using AUTO SEMVER mode: ${{ steps.semver.outputs.nextStrict }}" - echo "should_deploy=true" >> $GITHUB_OUTPUT - echo "pkg_version=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT - echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME" - else - echo "Using TEST mode: ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" - echo "should_deploy=false" >> $GITHUB_OUTPUT - echo "pkg_version=${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT - echo "::notice::Non-production build ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME" - fi + SHORT_SHA="${GITHUB_SHA:0:7}" + echo "Will set build version to: ${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA" + echo "buildVersion=${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA" >> $GITHUB_OUTPUT + echo "::notice::Non-production build ${{ steps.semver.outputs.nextMajorStrict }}.0.0-dev.$SHORT_SHA created using branch $GITHUB_REF_NAME" - name: Get Base Image Target Version id: baseimgversion run: | - echo "base_image_version=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + echo "baseImageVersion=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT # ----------------------------------------------------------------- # TESTS # ----------------------------------------------------------------- - tests: name: Run Tests - uses: ./.github/workflows/tests.yml - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} + uses: ./.github/workflows/reusable-tests.yml + if: ${{ inputs.skipTests == 'false' }} needs: [prepare] secrets: inherit with: - ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }} + ignoreLowerCoverage: ${{ inputs.ignoreLowerCoverage == 'true' }} skipSelenium: true - targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }} + targetBaseVersion: ${{ needs.prepare.outputs.baseImageVersion }} # ----------------------------------------------------------------- - # RELEASE + # BUILD IMAGE # ----------------------------------------------------------------- - release: - name: Make Release + build: + name: Build Image + uses: ./.github/workflows/reusable-build.yml if: ${{ !failure() && !cancelled() }} needs: [tests, prepare] - runs-on: - group: hperf-8c32r permissions: contents: write packages: write - env: - SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}} - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - FROM_TAG: ${{needs.prepare.outputs.from_tag}} - TO_TAG: ${{needs.prepare.outputs.to_tag}} - TARGET_BASE: ${{needs.prepare.outputs.base_image_version}} - - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 1 - fetch-tags: false - - - name: Setup Node.js environment - uses: actions/setup-node@v6 - with: - node-version: 18.x - - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: "3.x" - - - name: Setup AWS CLI - uses: unfor19/install-aws-cli-action@v1 - with: - version: 2.22.35 - - - name: Download a Coverage Results - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v8.0.1 - with: - name: coverage - - - name: Make Release Build - env: - DEBIAN_FRONTEND: noninteractive - BROWSERSLIST_IGNORE_OLD_DATA: 1 - run: | - echo "PKG_VERSION: $PKG_VERSION" - echo "GITHUB_SHA: $GITHUB_SHA" - echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" - echo "Running frontend build script..." - echo "Compiling native node packages..." - yarn rebuild - echo "Packaging static assets..." - yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/ - yarn legacy:build - echo "Setting version $PKG_VERSION..." - sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py - sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py - sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py - - - name: Set Production Flags - if: ${{ env.SHOULD_DEPLOY == 'true' }} - run: | - echo "Setting production flags in settings.py..." - sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py - - - name: Make Release Tarball - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Build release tarball..." - mkdir -p /home/runner/work/release - tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . - - - name: Collect + Push Statics - env: - DEBIAN_FRONTEND: noninteractive - AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} - AWS_DEFAULT_REGION: auto - AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} - run: | - echo "Collecting statics..." - echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}" - docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh - echo "Pushing statics..." - cd static - aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors - - - name: Augment dockerignore for docker image build - env: - DEBIAN_FRONTEND: noninteractive - run: | - cat >> .dockerignore <> $GITHUB_ENV - - - name: Build Images - uses: docker/build-push-action@v7 - env: - DOCKER_BUILD_SUMMARY: false - with: - context: . - file: dev/build/Dockerfile - platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }} - push: true - tags: | - ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} - ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} - - - name: Update CHANGELOG - id: changelog - uses: Requarks/changelog-action@v1 - if: ${{ env.SHOULD_DEPLOY == 'true' }} - with: - token: ${{ github.token }} - fromTag: ${{ env.FROM_TAG }} - toTag: ${{ env.TO_TAG }} - writeToFile: false - - - name: Download Coverage Results - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v8.0.1 - with: - name: coverage - - - name: Prepare Coverage Action - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - working-directory: ./dev/coverage-action - run: npm install - - - name: Process Coverage Stats + Chart - id: covprocess - uses: ./dev/coverage-action/ - if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - with: - token: ${{ github.token }} - tokenCommon: ${{ secrets.GH_COMMON_TOKEN }} - repoCommon: common - version: ${{needs.prepare.outputs.pkg_version}} - changelog: ${{ steps.changelog.outputs.changes }} - summary: '' - coverageResultsPath: coverage.json - histCoveragePath: historical-coverage.json - - - name: Create Release - uses: ncipollo/release-action@v1.21.0 - if: ${{ env.SHOULD_DEPLOY == 'true' }} - with: - allowUpdates: true - makeLatest: true - draft: false - tag: ${{ env.PKG_VERSION }} - name: ${{ env.PKG_VERSION }} - body: ${{ steps.covprocess.outputs.changelog }} - artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Update Baseline Coverage - uses: ncipollo/release-action@v1.21.0 - if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }} - with: - allowUpdates: true - tag: baseline - omitBodyDuringUpdate: true - omitNameDuringUpdate: true - omitPrereleaseDuringUpdate: true - replacesArtifacts: true - artifacts: "coverage.json" - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Upload Build Artifacts - uses: actions/upload-artifact@v7 - with: - name: release-${{ env.PKG_VERSION }} - path: /home/runner/work/release/release.tar.gz + with: + buildVersion: ${{ needs.prepare.outputs.buildVersion }} + isReleaseBuild: false + previousVersion: ${{ needs.prepare.outputs.previousVersion }} + handleCoverageResults: ${{ inputs.skipTests == 'false' }} + updateCoverageBaseline: ${{ inputs.updateCoverage }} + baseImageVersion: ${{ needs.prepare.outputs.baseImageVersion }} + skipArmBuild: ${{ inputs.skipArmBuild }} # ----------------------------------------------------------------- # NOTIFY @@ -352,10 +133,10 @@ jobs: notify: name: Notify if: ${{ always() }} - needs: [prepare, tests, release] + needs: [prepare, tests, build] runs-on: ubuntu-latest env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - name: Notify on Slack (Success) @@ -366,7 +147,7 @@ jobs: method: chat.postMessage payload: | channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} - text: "Datatracker Build by ${{ github.triggering_actor }}" + text: "Datatracker Dev Build by ${{ github.triggering_actor }}" attachments: - color: "28a745" fields: @@ -381,26 +162,26 @@ jobs: method: chat.postMessage payload: | channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} - text: "Datatracker Build by ${{ github.triggering_actor }}" + text: "Datatracker Dev Build by ${{ github.triggering_actor }}" attachments: - color: "a82929" fields: - title: "Status" short: true - value: "Failed" + value: "Failed 😱" # ----------------------------------------------------------------- # DEV # ----------------------------------------------------------------- dev: name: Deploy to Dev - if: ${{ !failure() && !cancelled() && github.event.inputs.dev == 'true' }} - needs: [prepare, release] + if: ${{ !failure() && !cancelled() && inputs.deploy == 'Dev' }} + needs: [prepare, build] runs-on: ubuntu-latest environment: name: dev env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - uses: actions/checkout@v6 @@ -424,7 +205,7 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' + inputs: '{ "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }}, "deployStrategy":"${{ inputs.deployStrategy }}" }' waitForCompletionTimeout: 60m # ----------------------------------------------------------------- @@ -432,13 +213,13 @@ jobs: # ----------------------------------------------------------------- staging: name: Deploy to Staging - if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging Only' || github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }} - needs: [prepare, release] + if: ${{ !failure() && !cancelled() && inputs.deploy == 'Staging' }} + needs: [prepare, build] runs-on: ubuntu-latest environment: name: staging env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - name: Refresh Staging DB @@ -458,29 +239,5 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' - waitForCompletionTimeout: 30m - - # ----------------------------------------------------------------- - # PROD - # ----------------------------------------------------------------- - prod: - name: Deploy to Production - if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }} - needs: [prepare, staging] - runs-on: ubuntu-latest - environment: - name: production - env: - PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} - - steps: - - name: Deploy to production - uses: ietf-tools/workflow-dispatch-action@v1 - with: - workflow: deploy.yml - repo: ietf-tools/infra-k8s - ref: main - token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' waitForCompletionTimeout: 30m diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml index 5349f1ac7a6..4d4e034ed64 100644 --- a/.github/workflows/ci-run-tests.yml +++ b/.github/workflows/ci-run-tests.yml @@ -38,7 +38,7 @@ jobs: # ----------------------------------------------------------------- tests: name: Run Tests - uses: ./.github/workflows/tests.yml + uses: ./.github/workflows/reusable-tests.yml needs: [prepare] with: ignoreLowerCoverage: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..437ed1baac8 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,209 @@ +name: Make Release and Deploy +run-name: RELEASE ${{ github.run_number }} by @${{ github.actor }} + +on: + workflow_dispatch: + inputs: + deployStrategy: + description: 'Deploy Strategy' + default: 'Recreate' + required: true + type: choice + options: + - Recreate + - RollingUpdate + - Current + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + # ----------------------------------------------------------------- + # PREPARE + # ----------------------------------------------------------------- + prepare: + name: Prepare Release + runs-on: ubuntu-latest + outputs: + buildVersion: ${{ steps.buildvars.outputs.buildVersion }} + previousVersion: ${{ steps.semver.outputs.current }} + baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} + + steps: + - name: Ensure release branch + if: github.ref != 'refs/heads/release' + run: | + echo "This workflow can only run on the 'release' branch (current: ${{ github.ref_name }})" + exit 1 + + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Get Next Version + id: semver + uses: ietf-tools/semver-action@v1 + with: + token: ${{ github.token }} + branch: release + skipInvalidTags: true + patchList: fix, bugfix, perf, refactor, test, tests, chore + + - name: Create Draft Release + uses: ncipollo/release-action@v1.21.0 + with: + prerelease: true + draft: false + commit: ${{ github.sha }} + tag: ${{ steps.semver.outputs.nextStrict }} + name: ${{ steps.semver.outputs.nextStrict }} + body: '*pending*' + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set Build Variables + id: buildvars + run: | + echo "Will set build version to: ${{ steps.semver.outputs.nextStrict }}" + echo "buildVersion=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT + echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME" + + - name: Get Base Image Target Version + id: baseimgversion + run: | + echo "baseImageVersion=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + + # ----------------------------------------------------------------- + # TESTS + # ----------------------------------------------------------------- + tests: + name: Run Tests + uses: ./.github/workflows/reusable-tests.yml + needs: [prepare] + secrets: inherit + with: + ignoreLowerCoverage: false + skipSelenium: true + targetBaseVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # BUILD RELEASE + # ----------------------------------------------------------------- + build: + name: Build Release Image + uses: ./.github/workflows/reusable-build.yml + if: ${{ !failure() && !cancelled() }} + needs: [tests, prepare] + permissions: + contents: write + packages: write + secrets: inherit + with: + buildVersion: ${{needs.prepare.outputs.buildVersion}} + isReleaseBuild: true + previousVersion: ${{needs.prepare.outputs.previousVersion}} + handleCoverageResults: true + updateCoverageBaseline: true + baseImageVersion: ${{ needs.prepare.outputs.baseImageVersion }} + + # ----------------------------------------------------------------- + # NOTIFY + # ----------------------------------------------------------------- + notify: + name: Notify + if: ${{ always() }} + needs: [prepare, tests, build] + runs-on: ubuntu-latest + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Notify on Slack (Success) + if: ${{ !contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 + with: + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Release Build by ${{ github.triggering_actor }}" + attachments: + - color: "28a745" + fields: + - title: "Status" + short: true + value: "Completed" + - name: Notify on Slack (Failure) + if: ${{ contains(join(needs.*.result, ','), 'failure') }} + uses: slackapi/slack-github-action@v3 + with: + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage + payload: | + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Release Build by ${{ github.triggering_actor }}" + attachments: + - color: "a82929" + fields: + - title: "Status" + short: true + value: "Failed 😱" + + # ----------------------------------------------------------------- + # STAGING + # ----------------------------------------------------------------- + staging: + name: Deploy to Staging + if: ${{ !failure() && !cancelled() }} + needs: [prepare, build] + runs-on: ubuntu-latest + environment: + name: staging + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Refresh Staging DB + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy-db.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }' + waitForCompletionTimeout: 120m + + - name: Deploy to staging + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m + + # ----------------------------------------------------------------- + # PROD + # ----------------------------------------------------------------- + prod: + name: Deploy to Production + if: ${{ !failure() && !cancelled() }} + needs: [prepare, staging] + runs-on: ubuntu-latest + environment: + name: production + env: + BUILD_VERSION: ${{needs.prepare.outputs.buildVersion}} + + steps: + - name: Deploy to production + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "deployStrategy":"${{ inputs.deployStrategy }}" }' + waitForCompletionTimeout: 30m diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml new file mode 100644 index 00000000000..93867d3400e --- /dev/null +++ b/.github/workflows/reusable-build.yml @@ -0,0 +1,226 @@ +name: Reusable Build Workflow + +on: + workflow_call: + inputs: + buildVersion: + required: true + type: string + isReleaseBuild: + default: false + required: false + type: boolean + previousVersion: + required: false + type: string + handleCoverageResults: + default: true + required: false + type: boolean + updateCoverageBaseline: + default: false + required: false + type: boolean + baseImageVersion: + default: latest + required: false + type: string + skipArmBuild: + default: false + required: false + type: boolean + +jobs: + # ----------------------------------------------------------------- + # BUILD IMAGE + # ----------------------------------------------------------------- + build: + name: Build Image + runs-on: + group: hperf-8c32r + permissions: + contents: write + packages: write + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 1 + fetch-tags: false + + - name: Setup Node.js environment + uses: actions/setup-node@v6 + with: + node-version: 18.x + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.x" + + - name: Setup AWS CLI + uses: unfor19/install-aws-cli-action@v1 + with: + version: 2.22.35 + + - name: Download a Coverage Results + if: ${{ inputs.handleCoverageResults }} + uses: actions/download-artifact@v8.0.1 + with: + name: coverage + + - name: Make Build + env: + DEBIAN_FRONTEND: noninteractive + BROWSERSLIST_IGNORE_OLD_DATA: 1 + run: | + echo "BUILD_VERSION: ${{ inputs.buildVersion }}" + echo "GITHUB_SHA: $GITHUB_SHA" + echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" + echo "Running frontend build script..." + echo "Compiling native node packages..." + yarn rebuild + echo "Packaging static assets..." + yarn build --base=https://static.ietf.org/dt/${{ inputs.buildVersion }}/ + yarn legacy:build + echo "Setting version ${{ inputs.buildVersion }}..." + sed -i -r -e "s|^__version__ += '.*'$|__version__ = '${{ inputs.buildVersion }}'|" ietf/__init__.py + sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py + sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py + + - name: Set Production Flags + if: ${{ inputs.isReleaseBuild }} + run: | + echo "Setting production flags in settings.py..." + sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py + + - name: Make Tarball + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "Build release tarball..." + mkdir -p /home/runner/work/release + tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . + + - name: Collect + Push Statics + env: + DEBIAN_FRONTEND: noninteractive + AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} + AWS_DEFAULT_REGION: auto + AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} + run: | + echo "Collecting statics..." + echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.baseImageVersion }}" + docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.baseImageVersion }} sh dev/build/collectstatics.sh + echo "Pushing statics..." + cd static + aws s3 sync . s3://static/dt/${{ inputs.buildVersion }} --only-show-errors + + - name: Augment dockerignore for docker image build + env: + DEBIAN_FRONTEND: noninteractive + run: | + cat >> .dockerignore <> $GITHUB_ENV + + - name: Build Images + uses: docker/build-push-action@v7 + env: + DOCKER_BUILD_SUMMARY: false + with: + context: . + file: dev/build/Dockerfile + platforms: ${{ inputs.skipArmBuild && 'linux/amd64' || 'linux/amd64,linux/arm64' }} + push: true + tags: | + ghcr.io/ietf-tools/datatracker:${{ inputs.buildVersion }} + ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} + + - name: Update CHANGELOG + id: changelog + uses: Requarks/changelog-action@v1 + if: ${{ inputs.isReleaseBuild }} + with: + token: ${{ github.token }} + fromTag: ${{ inputs.buildVersion }} + toTag: ${{ inputs.previousVersion }} + writeToFile: false + + - name: Download Coverage Results + if: ${{ inputs.handleCoverageResults }} + uses: actions/download-artifact@v8.0.1 + with: + name: coverage + + - name: Prepare Coverage Action + if: ${{ inputs.handleCoverageResults }} + working-directory: ./dev/coverage-action + run: npm install + + - name: Process Coverage Stats + Chart + id: covprocess + uses: ./dev/coverage-action/ + if: ${{ inputs.handleCoverageResults }} + with: + token: ${{ github.token }} + tokenCommon: ${{ secrets.GH_COMMON_TOKEN }} + repoCommon: common + version: ${{ inputs.buildVersion }} + changelog: ${{ steps.changelog.outputs.changes }} + summary: '' + coverageResultsPath: coverage.json + histCoveragePath: historical-coverage.json + + - name: Create Release + uses: ncipollo/release-action@v1.21.0 + if: ${{ inputs.isReleaseBuild }} + with: + allowUpdates: true + makeLatest: true + draft: false + tag: ${{ inputs.buildVersion }} + name: ${{ inputs.buildVersion }} + body: ${{ steps.covprocess.outputs.changelog }} + artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Update Baseline Coverage + uses: ncipollo/release-action@v1.21.0 + if: ${{ inputs.updateCoverageBaseline }} + with: + allowUpdates: true + tag: baseline + omitBodyDuringUpdate: true + omitNameDuringUpdate: true + omitPrereleaseDuringUpdate: true + replacesArtifacts: true + artifacts: "coverage.json" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload Build Artifacts + uses: actions/upload-artifact@v7 + with: + name: release-${{ inputs.buildVersion }} + path: /home/runner/work/release/release.tar.gz diff --git a/.github/workflows/tests.yml b/.github/workflows/reusable-tests.yml similarity index 100% rename from .github/workflows/tests.yml rename to .github/workflows/reusable-tests.yml From 5dd0fc6ecbc2d9337ebd7e8cf5957dd8a0907b90 Mon Sep 17 00:00:00 2001 From: NGPixel Date: Thu, 11 Jun 2026 22:33:52 -0400 Subject: [PATCH 288/313] ci: fix build workflow --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c449b53ce8..4b2bd03f68f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,6 +118,7 @@ jobs: permissions: contents: write packages: write + secrets: inherit with: buildVersion: ${{ needs.prepare.outputs.buildVersion }} isReleaseBuild: false From 70ed39f7c00f58f67d69243cbd16280eeca0cf06 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Fri, 12 Jun 2026 17:03:45 -0400 Subject: [PATCH 289/313] ci: fix build workflow --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4b2bd03f68f..bfc77b2910e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,7 +14,7 @@ on: - Dev - Staging deployStrategy: - description: 'Deploy Strategy' + description: 'Deploy Strategy (Staging Only)' default: 'Recreate' required: true type: choice @@ -23,7 +23,7 @@ on: - RollingUpdate - Current devNoDbRefresh: - description: 'Dev Disable Daily DB Refresh' + description: 'Disable Daily DB Refresh (Dev Only)' default: false required: true type: boolean @@ -206,7 +206,7 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }}, "deployStrategy":"${{ inputs.deployStrategy }}" }' + inputs: '{ "app":"datatracker", "appVersion":"${{ env.BUILD_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' waitForCompletionTimeout: 60m # ----------------------------------------------------------------- From af603788927fc11746be350b4fbb4d6a15ca27b8 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 16 Jun 2026 09:01:58 +1200 Subject: [PATCH 290/313] fix: scrub control characters from object values headed towards lxml (#11040) * fix: scrub control characters from object values headed towards lxml * chore: clearer comment --- ietf/api/__init__.py | 28 +++++++++++++++++----------- ietf/api/tests.py | 27 +++++++++++++++++---------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index d4562f97ddf..00733717c4f 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -177,8 +177,15 @@ def dehydrate(self, bundle, for_list=True): return dehydrated + +# XML 1.0 forbids all control characters except tab (#x9), LF (#xA), and CR (#xD). +# Replace each with its Unicode control picture (U+2400 + codepoint) so the +# substitution is lossless and the result is valid XML. +_XML_INVALID_CTRL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]") + + class Serializer(tastypie.serializers.Serializer): - OPTION_ESCAPE_NULLS = "datatracker-escape-nulls" + OPTION_ESCAPE_XML_INVALID = "datatracker-escape-xml-invalid" def format_datetime(self, data): return data.astimezone(datetime.UTC).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" @@ -186,18 +193,17 @@ def format_datetime(self, data): def to_simple(self, data, options): options = options or {} simple_data = super().to_simple(data, options) - if ( - options.get(self.OPTION_ESCAPE_NULLS, False) - and isinstance(simple_data, str) - ): - # replace nulls with unicode "symbol for null character", \u2400 - simple_data = simple_data.replace("\x00", "\u2400") + if options.get(self.OPTION_ESCAPE_XML_INVALID, False) and isinstance(simple_data, str): + # Replace control chars invalid in XML 1.0 with their Unicode + # control pictures (U+2400-U+241F) so lxml won't reject the string. + simple_data = _XML_INVALID_CTRL_RE.sub( + lambda m: chr(ord(m.group()) + 0x2400), simple_data + ) return simple_data def to_etree(self, data, options=None, name=None, depth=0): - # lxml does not escape nulls on its own, so ask to_simple() to do it. - # This is mostly (only?) an issue when generating errors responses for - # fuzzers. + # lxml rejects control characters that are invalid in XML 1.0. + # Ask to_simple() to escape them before they reach lxml. options = options or {} - options[self.OPTION_ESCAPE_NULLS] = True + options[self.OPTION_ESCAPE_XML_INVALID] = True return super().to_etree(data, options, name, depth) diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 2a44791a5cf..887969cec15 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -1542,20 +1542,27 @@ def test_all_model_resources_exist(self): self.assertIn(model._meta.model_name, list(app_resources.keys()), "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,)) - def test_serializer_to_etree_handles_nulls(self): - """Serializer to_etree() should handle a null character""" + def test_serializer_to_etree_handles_xml_invalid_control_chars(self): + """Serializer to_etree() must not raise ValueError for any XML-invalid control character.""" serializer = Serializer() + # Ordinary strings and strings with valid whitespace must pass through unchanged. try: - serializer.to_etree("string with no nulls in it") + serializer.to_etree("string with no special chars") + serializer.to_etree("tab\there lf\nhere cr\rhere") except ValueError: self.fail("serializer.to_etree raised ValueError on an ordinary string") - try: - serializer.to_etree("string with a \x00 in it") - except ValueError: - self.fail( - "serializer.to_etree raised ValueError on a string " - "containing a null character" - ) + # Every control character that XML 1.0 forbids must be escaped rather than + # causing a ValueError. This is the class of characters that triggered the + # production exception (lxml.etree._utf8 rejects them all). + invalid_chars = [chr(c) for c in list(range(0x00, 0x09)) + [0x0b, 0x0c] + list(range(0x0e, 0x20))] + for ch in invalid_chars: + try: + serializer.to_etree(f"string with {ch!r} in it") + except ValueError: + self.fail( + f"serializer.to_etree raised ValueError on a string " + f"containing control character U+{ord(ch):04X}" + ) class RfcdiffSupportTests(TestCase): From d69e5457544e061177236800b592c6986bb5b0f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jun 2026 10:56:03 +1200 Subject: [PATCH 291/313] chore(deps): bump actions/checkout from 6 to 7 (#11074) Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-base-app.yml | 2 +- .github/workflows/build-devblobstore.yml | 2 +- .github/workflows/build-mq-broker.yml | 2 +- .github/workflows/build.yml | 4 ++-- .github/workflows/ci-run-tests.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/dev-assets-sync-nightly.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/reusable-build.yml | 2 +- .github/workflows/reusable-tests.yml | 6 +++--- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 35172aa2996..a2abe089ce6 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -18,7 +18,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: token: ${{ secrets.GH_COMMON_TOKEN }} diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml index 14c4b1a135a..429aa08b006 100644 --- a/.github/workflows/build-devblobstore.yml +++ b/.github/workflows/build-devblobstore.yml @@ -20,7 +20,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index b297e34b476..7832e65a3a3 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -24,7 +24,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Set up QEMU uses: docker/setup-qemu-action@v4 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bfc77b2910e..5088d763e92 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,7 +65,7 @@ jobs: baseImageVersion: ${{ steps.baseimgversion.outputs.baseImageVersion }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false @@ -185,7 +185,7 @@ jobs: BUILD_VERSION: ${{ needs.prepare.outputs.buildVersion }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: ref: main diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml index 4d4e034ed64..50ec4c1943b 100644 --- a/.github/workflows/ci-run-tests.yml +++ b/.github/workflows/ci-run-tests.yml @@ -23,7 +23,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bc20779ae6c..15b2231ce39 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Initialize CodeQL uses: github/codeql-action/init@v4 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index e255b270ff9..7393e83b829 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: 'Dependency Review' uses: actions/dependency-review-action@v4 with: diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index cd986f06f33..83121354e64 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -29,7 +29,7 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Login to GitHub Container Registry uses: docker/login-action@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 437ed1baac8..2de02eb2852 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: echo "This workflow can only run on the 'release' branch (current: ${{ github.ref_name }})" exit 1 - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index 93867d3400e..4d9a3b4bf11 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -43,7 +43,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 1 fetch-tags: false diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml index ad2e35408d4..10d04c8265e 100644 --- a/.github/workflows/reusable-tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -32,7 +32,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-devblobstore:latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Prepare for tests run: | @@ -102,7 +102,7 @@ jobs: project: [chromium, firefox] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: actions/setup-node@v6 with: @@ -144,7 +144,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-db:latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Prepare for tests run: | From a89f27bcf5bb7d45119ae2cad5b6b553f91888c9 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 24 Jun 2026 04:19:39 +1200 Subject: [PATCH 292/313] fix: Use non-four digit RFC numbers for DOI (#11080) --- ietf/doc/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 156bac4e779..3685ab65519 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -163,7 +163,7 @@ class DocumentInfo(models.Model): @property def doi(self) -> str | None: if self.type_id == "rfc" and self.rfc_number is not None: - return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number:04d}" + return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number}" return None def file_extension(self): From b05e281a7f7ad15106bd06ad124ef52ee546c6f0 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 1 Jul 2026 07:51:31 +1200 Subject: [PATCH 293/313] fix: include URL to I-D Archive in WGLC messages (#11056) --- ietf/doc/models.py | 5 ++++ ietf/doc/tests_draft.py | 53 ++++++++++++++++++++++++++++++++++++++++- ietf/doc/views_draft.py | 1 + 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 3685ab65519..77b2c86efe5 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -344,6 +344,11 @@ def _get_ref(self, meeting=None, versioned=True): setattr(self, cache_attr, href) return getattr(self, cache_attr) + def refresh_from_db(self, using=None, fields=None, **kwargs): + super().refresh_from_db(using=using, fields=fields, **kwargs) + self.state_cache = None + self._cached_state_slug = {} + def set_state(self, state): """Switch state type implicit in state to state. This just sets the state, doesn't log the change.""" diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index db9dbc2baff..688b933aad3 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -2187,9 +2187,34 @@ def _assert_rejected(testcase, doc, person): chair = RoleFactory(name_id="chair", group=doc.group).person url = urlreverse("ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name)) login_testing_unauthorized(self, chair.user.username, url) + + # GET: check 200 and that the pre-populated body contains the archive URL r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) + prefilled_body = q("textarea#id_body").text() + expected_archive_url = f"{settings.IETF_ID_ARCHIVE_URL}{doc.name}-{doc.rev}.txt" + self.assertIn( + expected_archive_url, + prefilled_body, + "Pre-populated body must contain the I-D archive .txt URL", + ) + + # Invalid POST: end_date in the past must not change state + bad_postdict = dict( + end_date=date_today(DEADLINE_TZINFO), + to=q("input#id_to").attr("value"), + subject=q("input#id_subject").attr("value"), + body=prefilled_body, + ) + r = self.client.post(url, bad_postdict) + self.assertEqual(r.status_code, 200, "Invalid POST should re-render the form") + self.assertNotEqual( + doc.get_state_slug("draft-stream-ietf"), + "wg-lc", + "State must not change on invalid POST", + ) + postdict = dict() postdict["end_date"] = q("input#id_end_date").attr("value") postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.org" @@ -2199,15 +2224,36 @@ def _assert_rejected(testcase, doc, person): else: postdict["cc"] = "extracc@example.org" postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words" - postdict["body"] = q("textarea#id_body").text() + "FGgqbQ$UNeXs" + postdict["body"] = prefilled_body + "FGgqbQ$UNeXs" empty_outbox() r = self.client.post( url, postdict, ) self.assertEqual(r.status_code, 302) + doc.refresh_from_db() self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "wg-lc") + + # A StateDocEvent recording the transition must exist + self.assertTrue( + doc.docevent_set.filter(type="changed_state").exists(), + "Expected a changed_state event after issuing WG LC", + ) + + # A stream-s reminder must be active for the doc + self.assertTrue( + DocReminder.objects.filter( + event__doc=doc, type_id="stream-s", active=True + ).exists(), + "Expected an active stream-s reminder after issuing WG LC", + ) + self.assertEqual(len(outbox), 2) + + # outbox[0] is the stream-state-changed notification + self.assertIn(doc.name, outbox[0]["Subject"]) + + # outbox[1] is the WGLC announcement sent to the WG list self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"]) self.assertIn("extrato@example.org", outbox[1]["To"]) self.assertIn("extracc@example.org", outbox[1]["Cc"]) @@ -2216,6 +2262,11 @@ def _assert_rejected(testcase, doc, person): body = get_payload_text(outbox[1]) self.assertIn("disclosure obligations", body) self.assertIn("FGgqbQ$UNeXs", body) + self.assertIn( + expected_archive_url, + body, + "WGLC email body must contain the I-D archive .txt URL", + ) def test_issue_wg_call_for_adoption_form(self): end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1) diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py index 0f5ea49f5d4..a64d0a53fe9 100644 --- a/ietf/doc/views_draft.py +++ b/ietf/doc/views_draft.py @@ -1973,6 +1973,7 @@ def issue_wg_lc(request, name): doc=doc, end_date=end_date, wg_list=doc.group.list_email, + url=f"{settings.IETF_ID_ARCHIVE_URL}{doc.name}-{doc.rev}.txt", settings=settings, ), ) From f158244c292d725487c7e810de8b174bd46f5765 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 1 Jul 2026 07:55:57 +1200 Subject: [PATCH 294/313] fix: remove unreachable old RFC-Editor sync code. Flatten redirects. (#10967) --- ietf/doc/models.py | 3 - ietf/settings.py | 1 - ietf/sync/rfceditor.py | 845 ------------------ ietf/sync/tasks.py | 89 -- ietf/sync/tests.py | 687 +------------- ietf/sync/views.py | 19 +- ietf/templates/doc/document_draft.html | 2 +- ietf/templates/doc/search/status_columns.html | 2 +- .../management/commands/periodic_tasks.py | 28 - ietf/utils/timezone.py | 2 +- 10 files changed, 6 insertions(+), 1672 deletions(-) delete mode 100644 ietf/sync/rfceditor.py diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 77b2c86efe5..ff879dd71c6 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1283,9 +1283,6 @@ def submission(self): def pub_datetime(self): """Get the publication datetime of this document""" if self.type_id == "rfc": - # As of Sept 2022, in ietf.sync.rfceditor.update_docs_from_rfc_index() `published_rfc` events are - # created with a timestamp whose date *in the PST8PDT timezone* is the official publication date - # assigned by the RFC editor. event = self.latest_event(type='published_rfc') else: event = self.latest_event(type='new_revision') diff --git a/ietf/settings.py b/ietf/settings.py index d509a877c61..7af9c7ae5be 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -895,7 +895,6 @@ def skip_unreadable_post(record): IANA_SYNC_PROTOCOLS_URL = "https://www.iana.org/protocols/" RFC_EDITOR_GROUP_NOTIFICATION_EMAIL = "webmaster@rfc-editor.org" -RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" RFC_EDITOR_ERRATA_JSON_URL = "https://www.rfc-editor.org/errata.json" RFC_EDITOR_INLINE_ERRATA_URL = "https://www.rfc-editor.org/rfc/inline-errata/rfc{rfc_number}.html" diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py deleted file mode 100644 index 347b58efbb2..00000000000 --- a/ietf/sync/rfceditor.py +++ /dev/null @@ -1,845 +0,0 @@ -# Copyright The IETF Trust 2012-2025, All Rights Reserved -# -*- coding: utf-8 -*- - - -import datetime -import re - -from typing import Iterator, Optional, Union -from xml.dom import pulldom, Node - -from django.db import transaction -from django.db.models import Subquery, OuterRef, F, Q -from django.utils import timezone - -import debug # pyflakes:ignore - -from ietf.doc.models import ( Document, State, StateType, DocEvent, DocRelationshipName, - DocTagName, RelatedDocument, RelatedDocHistory ) -from ietf.doc.expire import move_draft_files_to_archive -from ietf.doc.utils import add_state_change_event, new_state_change_event, prettify_std_name, update_action_holders -from ietf.group.models import Group -from ietf.ipr.models import IprDocRel -from ietf.name.models import StdLevelName, StreamName -from ietf.person.models import Person -from ietf.utils.log import log -from ietf.utils.mail import send_mail_text -from ietf.utils.timezone import datetime_from_date, RPC_TZINFO - -# QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" -# INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" -# POST_APPROVED_DRAFT_URL = "https://www.rfc-editor.org/sdev/jsonexp/jsonparser.php" - -MIN_ERRATA_RESULTS = 5000 -MIN_INDEX_RESULTS = 8000 -MIN_QUEUE_RESULTS = 10 - -def get_child_text(parent_node, tag_name): - text = [] - for node in parent_node.childNodes: - if node.nodeType == Node.ELEMENT_NODE and node.localName == tag_name: - text.append(node.firstChild.data) - return '\n\n'.join(text) - - -def parse_queue(response): - """Parse RFC Editor queue XML into a bunch of tuples + warnings.""" - - events = pulldom.parse(response) - drafts = [] - warnings = [] - stream = None - - for event, node in events: - try: - if event == pulldom.START_ELEMENT and node.tagName == "entry": - events.expandNode(node) - node.normalize() - draft_name = get_child_text(node, "draft").strip() - draft_name = re.sub(r"(-\d\d)?(.txt){1,2}$", "", draft_name) - date_received = get_child_text(node, "date-received") - - state = "" - tags = [] - missref_generation = "" - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "state": - state = child.firstChild.data - # state has some extra annotations encoded, parse - # them out - if '*R' in state: - tags.append("ref") - state = state.replace("*R", "") - if '*A' in state: - tags.append("iana") - state = state.replace("*A", "") - m = re.search(r"\(([0-9]+)G\)", state) - if m: - missref_generation = m.group(1) - state = state.replace("(%sG)" % missref_generation, "") - - # AUTH48 link - auth48 = "" - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "auth48-url": - auth48 = child.firstChild.data - - # cluster link (if it ever gets implemented) - cluster = "" - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "cluster-url": - cluster = child.firstChild.data - - refs = [] - for child in node.childNodes: - if child.nodeType == Node.ELEMENT_NODE and child.localName == "normRef": - ref_name = get_child_text(child, "ref-name") - ref_state = get_child_text(child, "ref-state") - in_queue = ref_state.startswith("IN-QUEUE") - refs.append((ref_name, ref_state, in_queue)) - - drafts.append((draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs)) - - elif event == pulldom.START_ELEMENT and node.tagName == "section": - name = node.getAttribute('name') - if name.startswith("IETF"): - stream = "ietf" - elif name.startswith("IAB"): - stream = "iab" - elif name.startswith("IRTF"): - stream = "irtf" - elif name.startswith("INDEPENDENT"): - stream = "ise" - elif name.startswith("Editorial Stream"): - stream = "editorial" - else: - stream = None - warnings.append("unrecognized section " + name) - except Exception as e: - log("Exception when processing an RFC queue entry: %s" % e) - log("node: %s" % node) - raise - - return drafts, warnings - -def update_drafts_from_queue(drafts): - """Given a list of parsed drafts from the RFC Editor queue, update the - documents in the database. Return those that were changed.""" - - tag_mapping = { - 'IANA': DocTagName.objects.get(slug='iana'), - 'REF': DocTagName.objects.get(slug='ref') - } - - slookup = dict((s.slug, s) - for s in State.objects.filter(used=True, type=StateType.objects.get(slug="draft-rfceditor"))) - state_mapping = { - 'AUTH': slookup['auth'], - 'AUTH48': slookup['auth48'], - 'AUTH48-DONE': slookup['auth48-done'], - 'EDIT': slookup['edit'], - 'IANA': slookup['iana'], - 'IESG': slookup['iesg'], - 'ISR': slookup['isr'], - 'ISR-AUTH': slookup['isr-auth'], - 'REF': slookup['ref'], - 'RFC-EDITOR': slookup['rfc-edit'], - 'TI': slookup['tooling-issue'], - 'TO': slookup['timeout'], - 'MISSREF': slookup['missref'], - } - - system = Person.objects.get(name="(System)") - - warnings = [] - - names = [t[0] for t in drafts] - - drafts_in_db = dict((d.name, d) - for d in Document.objects.filter(type="draft", name__in=names)) - - changed = set() - - for name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs in drafts: - if name not in drafts_in_db: - warnings.append("unknown document %s" % name) - continue - - if not state or state not in state_mapping: - warnings.append("unknown state '%s' for %s" % (state, name)) - continue - - d = drafts_in_db[name] - - prev_state = d.get_state("draft-rfceditor") - next_state = state_mapping[state] - events = [] - - # check if we've noted it's been received - if d.get_state_slug("draft-iesg") == "ann" and not prev_state and not d.latest_event(DocEvent, type="rfc_editor_received_announcement"): - e = DocEvent(doc=d, rev=d.rev, by=system, type="rfc_editor_received_announcement") - e.desc = "Announcement was received by RFC Editor" - e.save() - send_mail_text(None, "iesg-secretary@ietf.org", None, - '%s in RFC Editor queue' % d.name, - 'The announcement for %s has been received by the RFC Editor.' % d.name) - # change draft-iesg state to RFC Ed Queue - prev_iesg_state = State.objects.get(used=True, type="draft-iesg", slug="ann") - next_iesg_state = State.objects.get(used=True, type="draft-iesg", slug="rfcqueue") - - d.set_state(next_iesg_state) - e = add_state_change_event(d, system, prev_iesg_state, next_iesg_state) - if e: - events.append(e) - e = update_action_holders(d, prev_iesg_state, next_iesg_state) - if e: - events.append(e) - changed.add(name) - - # check draft-rfceditor state - if prev_state != next_state: - d.set_state(next_state) - - e = new_state_change_event(d, system, prev_state, next_state) # unsaved - if e: - if auth48: - e.desc = re.sub(r"(.*)", "\\1" % auth48, e.desc) - e.save() - events.append(e) - - if auth48: - # Create or update the auth48 URL whether or not this is a state expected to have one. - d.documenturl_set.update_or_create( - tag_id='auth48', # look up existing based on this field - defaults=dict(url=auth48) # create or update with this field - ) - else: - # Remove any existing auth48 URL when an update does not have one. - d.documenturl_set.filter(tag_id='auth48').delete() - - changed.add(name) - - t = DocTagName.objects.filter(slug__in=tags) - if set(t) != set(d.tags.all()): - d.tags.clear() - d.tags.set(t) - changed.add(name) - - if events: - d.save_with_history(events) - - - # remove tags and states for those not in the queue anymore - for d in Document.objects.exclude(name__in=names).filter(states__type="draft-rfceditor").distinct(): - d.tags.remove(*list(tag_mapping.values())) - d.unset_state("draft-rfceditor") - # we do not add a history entry here - most likely we already - # have something that explains what happened - changed.add(name) - - return changed, warnings - - -def parse_index(response): - """Parse RFC Editor index XML into a bunch of tuples.""" - - def normalize_std_name(std_name): - # remove zero padding - prefix = std_name[:3] - if prefix in ("RFC", "FYI", "BCP", "STD"): - try: - return prefix + str(int(std_name[3:])) - except ValueError: - pass - return std_name - - def extract_doc_list(parentNode, tagName): - l = [] - for u in parentNode.getElementsByTagName(tagName): - for d in u.getElementsByTagName("doc-id"): - l.append(normalize_std_name(d.firstChild.data)) - return l - - also_list = {} - data = [] - events = pulldom.parse(response) - for event, node in events: - try: - if event == pulldom.START_ELEMENT and node.tagName in ["bcp-entry", "fyi-entry", "std-entry"]: - events.expandNode(node) - node.normalize() - bcpid = normalize_std_name(get_child_text(node, "doc-id")) - doclist = extract_doc_list(node, "is-also") - for docid in doclist: - if docid in also_list: - also_list[docid].append(bcpid) - else: - also_list[docid] = [bcpid] - - elif event == pulldom.START_ELEMENT and node.tagName == "rfc-entry": - events.expandNode(node) - node.normalize() - rfc_number = int(get_child_text(node, "doc-id")[3:]) - title = get_child_text(node, "title") - - authors = [] - for author in node.getElementsByTagName("author"): - authors.append(get_child_text(author, "name")) - - d = node.getElementsByTagName("date")[0] - year = int(get_child_text(d, "year")) - month = get_child_text(d, "month") - month = ["January","February","March","April","May","June","July","August","September","October","November","December"].index(month)+1 - rfc_published_date = datetime.date(year, month, 1) - - current_status = get_child_text(node, "current-status").title() - - updates = extract_doc_list(node, "updates") - updated_by = extract_doc_list(node, "updated-by") - obsoletes = extract_doc_list(node, "obsoletes") - obsoleted_by = extract_doc_list(node, "obsoleted-by") - pages = get_child_text(node, "page-count") - stream = get_child_text(node, "stream") - wg = get_child_text(node, "wg_acronym") - if wg and ((wg == "NON WORKING GROUP") or len(wg) > 15): - wg = None - - l = [] - for fmt in node.getElementsByTagName("format"): - l.append(get_child_text(fmt, "file-format")) - file_formats = (",".join(l)).lower() - - abstract = "" - for abstract in node.getElementsByTagName("abstract"): - abstract = get_child_text(abstract, "p") - - draft = get_child_text(node, "draft") - if draft and re.search(r"-\d\d$", draft): - draft = draft[0:-3] - - if len(node.getElementsByTagName("errata-url")) > 0: - has_errata = 1 - else: - has_errata = 0 - - data.append((rfc_number,title,authors,rfc_published_date,current_status,updates,updated_by,obsoletes,obsoleted_by,[],draft,has_errata,stream,wg,file_formats,pages,abstract)) - except Exception as e: - log("Exception when processing an RFC index entry: %s" % e) - log("node: %s" % node) - raise - for d in data: - k = "RFC%d" % d[0] - if k in also_list: - d[9].extend(also_list[k]) - return data - - -def update_docs_from_rfc_index( - index_data, errata_data, skip_older_than_date: Optional[datetime.date] = None -) -> Iterator[tuple[int, list[str], Document, bool]]: - """Given parsed data from the RFC Editor index, update the documents in the database - - Returns an iterator that yields (rfc_number, change_list, doc, rfc_published) for the - RFC document and, if applicable, the I-D that it came from. - - The skip_older_than_date is a bare date, not a datetime. - """ - # Create dict mapping doc-id to list of errata records that apply to it - errata: dict[str, list[dict]] = {} - for item in errata_data: - name = item["doc-id"] - if name.upper().startswith("RFC"): - name = f"RFC{int(name[3:])}" # removes leading 0s on the rfc number - if not name in errata: - errata[name] = [] - errata[name].append(item) - - std_level_mapping = { - "Standard": StdLevelName.objects.get(slug="std"), - "Internet Standard": StdLevelName.objects.get(slug="std"), - "Draft Standard": StdLevelName.objects.get(slug="ds"), - "Proposed Standard": StdLevelName.objects.get(slug="ps"), - "Informational": StdLevelName.objects.get(slug="inf"), - "Experimental": StdLevelName.objects.get(slug="exp"), - "Best Current Practice": StdLevelName.objects.get(slug="bcp"), - "Historic": StdLevelName.objects.get(slug="hist"), - "Unknown": StdLevelName.objects.get(slug="unkn"), - } - - stream_mapping = { - "IETF": StreamName.objects.get(slug="ietf"), - "INDEPENDENT": StreamName.objects.get(slug="ise"), - "IRTF": StreamName.objects.get(slug="irtf"), - "IAB": StreamName.objects.get(slug="iab"), - "Editorial": StreamName.objects.get(slug="editorial"), - "Legacy": StreamName.objects.get(slug="legacy"), - } - - tag_has_errata = DocTagName.objects.get(slug="errata") - tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata") - relationship_obsoletes = DocRelationshipName.objects.get(slug="obs") - relationship_updates = DocRelationshipName.objects.get(slug="updates") - rfc_published_state = State.objects.get(type_id="rfc", slug="published") - - system = Person.objects.get(name="(System)") - - first_sync_creating_subseries = not Document.objects.filter(type_id__in=["bcp","std","fyi"]).exists() - - for ( - rfc_number, - title, - authors, - rfc_published_date, - current_status, - updates, - updated_by, - obsoletes, - obsoleted_by, - also, - draft_name, - has_errata, - stream, - wg, - file_formats, - pages, - abstract, - ) in index_data: - if skip_older_than_date and rfc_published_date < skip_older_than_date: - # speed up the process by skipping old entries (n.b., the comparison above is a - # lexical comparison between "YYYY-MM-DD"-formatted dates) - continue - - # we assume two things can happen: we get a new RFC, or an - # attribute has been updated at the RFC Editor (RFC Editor - # attributes take precedence over our local attributes) - rfc_events = [] - rfc_changes = [] - rfc_published = False - - # Find the draft, if any - draft = None - if draft_name: - try: - draft = Document.objects.get(name=draft_name, type_id="draft") - except Document.DoesNotExist: - pass - # Logging below warning turns out to be unhelpful - there are many references - # to such things in the index: - # * all april-1 RFCs have an internal name that looks like a draft name, but there - # was never such a draft. More of these will exist in the future - # * Several documents were created with out-of-band input to the RFC-editor, for a - # variety of reasons. - # - # What this exposes is that the rfc-index needs to stop talking about these things. - # If there is no draft to point to, don't point to one, even if there was an RPC - # internal name in use (and in the RPC database). This will be a requirement on the - # reimplementation of the creation of the rfc-index. - # - # log(f"Warning: RFC index for {rfc_number} referred to unknown draft {draft_name}") - - # Find or create the RFC document - creation_args: dict[str, Optional[Union[str, int]]] = {"name": f"rfc{rfc_number}"} - if draft: - creation_args.update( - { - "title": draft.title, - "stream": draft.stream, - "group": draft.group, - "abstract": draft.abstract, - "pages": draft.pages, - "words": draft.words, - "std_level": draft.std_level, - "ad": draft.ad, - "external_url": draft.external_url, - "uploaded_filename": draft.uploaded_filename, - "note": draft.note, - } - ) - doc, created_rfc = Document.objects.get_or_create( - rfc_number=rfc_number, type_id="rfc", defaults=creation_args - ) - if created_rfc: - rfc_changes.append(f"created document {prettify_std_name(doc.name)}") - doc.set_state(rfc_published_state) - if draft: - doc.formal_languages.set(draft.formal_languages.all()) - # Create authors based on the last draft in the datatracker. This - # path will go away when we publish via the modernized RPC workflow - # but until then, these are the only data we have for authors that - # are easily connected to Person records. - for documentauthor in draft.documentauthor_set.all(): - # Copy the author but point at the new doc. - # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances - documentauthor.pk = None - documentauthor.id = None - documentauthor._state.adding = True - documentauthor.document = doc - documentauthor.save() - - if draft: - draft_events = [] - draft_changes = [] - - # Ensure the draft is in the "rfc" state and move its files to the archive - # if necessary. - if draft.get_state_slug() != "rfc": - draft.set_state( - State.objects.get(used=True, type="draft", slug="rfc") - ) - move_draft_files_to_archive(draft, draft.rev) - draft_changes.append(f"changed state to {draft.get_state()}") - - # Ensure the draft and rfc are linked with a "became_rfc" relationship - r, created_relateddoc = RelatedDocument.objects.get_or_create( - source=draft, target=doc, relationship_id="became_rfc" - ) - if created_relateddoc: - change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format( - rel_name=r.relationship.name.lower(), - pretty_draft_name=prettify_std_name(draft_name), - pretty_rfc_name=prettify_std_name(doc.name), - ) - draft_changes.append(change) - rfc_changes.append(change) - - # Always set the "draft-iesg" state. This state should be set for all drafts, so - # log a warning if it is not set. What should happen here is that ietf stream - # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index. - # Other stream documents should normally be "idexists" and be left that way. The - # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub", - # and changes any other state to "pub". If unset, it changes it to "idexists". - # This reflects historical behavior and should probably be updated, but a migration - # of existing drafts (and validation of the change) is needed before we change the - # handling. - prev_iesg_state = draft.get_state("draft-iesg") - if prev_iesg_state is None: - log(f'Warning while processing {doc.name}: {draft.name} has no "draft-iesg" state') - new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists") - elif prev_iesg_state.slug not in ("pub", "idexists"): - if prev_iesg_state.slug != "rfcqueue": - log( - 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format( - doc.name, draft.name, prev_iesg_state.slug - ) - ) - new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub") - else: - new_iesg_state = prev_iesg_state - - if new_iesg_state != prev_iesg_state: - draft.set_state(new_iesg_state) - draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}") - e = update_action_holders(draft, prev_iesg_state, new_iesg_state) - if e: - draft_events.append(e) - - # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain. - if draft.stream != doc.stream: - log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( - doc.name, draft.name, draft.stream, doc.stream - )) - elif draft.stream.slug in ["iab", "irtf", "ise"]: - stream_slug = f"draft-stream-{draft.stream.slug}" - prev_state = draft.get_state(stream_slug) - if prev_state is not None and prev_state.slug != "pub": - new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub") - draft.set_state(new_state) - draft_changes.append( - f"changed {new_state.type.label} to {new_state}" - ) - e = update_action_holders(draft, prev_state, new_state) - if e: - draft_events.append(e) - if draft_changes: - draft_events.append( - DocEvent.objects.create( - doc=draft, - rev=doc.rev, - by=system, - type="sync_from_rfc_editor", - desc=f"Received changes through RFC Editor sync ({', '.join(draft_changes)})", - ) - ) - draft.save_with_history(draft_events) - yield rfc_number, draft_changes, draft, False # yield changes to the draft - - # check attributes - verbed = "set" if created_rfc else "changed" - if title != doc.title: - doc.title = title - rfc_changes.append(f"{verbed} title to '{doc.title}'") - - if abstract and abstract != doc.abstract: - doc.abstract = abstract - rfc_changes.append(f"{verbed} abstract to '{doc.abstract}'") - - if pages and int(pages) != doc.pages: - doc.pages = int(pages) - rfc_changes.append(f"{verbed} pages to {doc.pages}") - - if std_level_mapping[current_status] != doc.std_level: - doc.std_level = std_level_mapping[current_status] - rfc_changes.append(f"{verbed} standardization level to {doc.std_level}") - - if doc.stream != stream_mapping[stream]: - doc.stream = stream_mapping[stream] - rfc_changes.append(f"{verbed} stream to {doc.stream}") - - if doc.get_state() != rfc_published_state: - doc.set_state(rfc_published_state) - rfc_changes.append(f"{verbed} {rfc_published_state.type.label} to {rfc_published_state}") - - # if we have no group assigned, check if RFC Editor has a suggestion - if not doc.group: - if wg: - doc.group = Group.objects.get(acronym=wg) - rfc_changes.append(f"set group to {doc.group}") - else: - doc.group = Group.objects.get( - type="individ" - ) # fallback for newly created doc - - if not doc.latest_event(type="published_rfc"): - e = DocEvent(doc=doc, rev=doc.rev, type="published_rfc") - # unfortunately, rfc_published_date doesn't include the correct day - # at the moment because the data only has month/year, so - # try to deduce it - # - # Note: This is in done PST8PDT to preserve compatibility with events created when - # USE_TZ was False. The published_rfc event was created with a timestamp whose - # server-local datetime (PST8PDT) matched the publication date from the RFC index. - # When switching to USE_TZ=True, the timestamps were migrated so they still - # matched the publication date in PST8PDT. When interpreting the event timestamp - # as a publication date, you must treat it in the PST8PDT time zone. The - # RPC_TZINFO constant in ietf.utils.timezone is defined for this purpose. - d = datetime_from_date(rfc_published_date, RPC_TZINFO) - synthesized = timezone.now().astimezone(RPC_TZINFO) - if abs(d - synthesized) > datetime.timedelta(days=60): - synthesized = d - else: - direction = -1 if (d - synthesized).total_seconds() < 0 else +1 - while synthesized.month != d.month or synthesized.year != d.year: - synthesized += datetime.timedelta(days=direction) - e.time = synthesized - e.by = system - e.desc = "RFC published" - e.save() - rfc_events.append(e) - - rfc_changes.append( - f"added RFC published event at {e.time.strftime('%Y-%m-%d')}" - ) - rfc_published = True - - def parse_relation_list(rel_list: list[str]) -> list[Document]: - return list( - Document.objects.filter( - name__in=[name.strip().lower() for name in rel_list], - type_id="rfc" - ) - ) - - # Create missing obsoletes relations - docs_this_obsoletes = parse_relation_list(obsoletes) - for obs_doc in docs_this_obsoletes: - if not doc.relateddocument_set.filter( - target=obs_doc, relationship=relationship_obsoletes - ): - r = doc.relateddocument_set.create( - target=obs_doc, relationship=relationship_obsoletes - ) - rfc_changes.append( - "created {rel_name} relation between {src} and {tgt}".format( - rel_name=r.relationship.name.lower(), - src=prettify_std_name(r.source.name), - tgt=prettify_std_name(r.target.name), - ) - ) - # Remove stale obsoletes relations - for r in doc.relateddocument_set.filter( - relationship=relationship_obsoletes - ).exclude(target_id__in=[d.pk for d in docs_this_obsoletes]): - r.delete() - rfc_changes.append( - "removed {rel_name} relation between {src} and {tgt}".format( - rel_name=r.relationship.name.lower(), - src=prettify_std_name(r.source.name), - tgt=prettify_std_name(r.target.name), - ) - ) - - docs_this_updates = parse_relation_list(updates) - for upd_doc in docs_this_updates: - if not RelatedDocument.objects.filter( - source=doc, target=upd_doc, relationship=relationship_updates - ): - r = doc.relateddocument_set.create( - target=upd_doc, relationship=relationship_updates - ) - rfc_changes.append( - "created {rel_name} relation between {src} and {tgt}".format( - rel_name=r.relationship.name.lower(), - src=prettify_std_name(r.source.name), - tgt=prettify_std_name(r.target.name), - ) - ) - # Remove stale updates relations - for r in doc.relateddocument_set.filter( - relationship=relationship_updates - ).exclude(target_id__in=[d.pk for d in docs_this_updates]): - r.delete() - rfc_changes.append( - "removed {rel_name} relation between {src} and {tgt}".format( - rel_name=r.relationship.name.lower(), - src=prettify_std_name(r.source.name), - tgt=prettify_std_name(r.target.name), - ) - ) - - if also: - # recondition also to have proper subseries document names: - conditioned_also = [] - for a in also: - a = a.lower() - subseries_slug = a[:3] - if subseries_slug not in ["bcp", "std", "fyi"]: - log(f"Unexpected 'also' relationship of {a} encountered for {doc}") - continue - maybe_number = a[3:].strip() - if not maybe_number.isdigit(): - log(f"Unexpected 'also' subseries element identifier {a} encountered for {doc}") - continue - else: - subseries_number = int(maybe_number) - conditioned_also.append(f"{subseries_slug}{subseries_number}") # Note the lack of leading zeros - also = conditioned_also - - for a in also: - subseries_doc_name = a - subseries_slug=a[:3] - # Leaving most things to the default intentionally - # Of note, title and stream are left to the defaults of "" and none. - subseries_doc, created = Document.objects.get_or_create(type_id=subseries_slug, name=subseries_doc_name) - if created: - if first_sync_creating_subseries: - subseries_doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of this {subseries_slug.upper()} document is currently available in the datatracker before this point") - subseries_doc.docevent_set.create(type=f"{subseries_slug}_doc_created", by=system, desc=f"Imported {subseries_doc_name} into the datatracker via sync to the rfc-index") - else: - subseries_doc.docevent_set.create(type=f"{subseries_slug}_doc_created", by=system, desc=f"Created {subseries_doc_name} via sync to the rfc-index") - _, relationship_created = subseries_doc.relateddocument_set.get_or_create(relationship_id="contains", target=doc) - if relationship_created: - if first_sync_creating_subseries: - subseries_doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Imported membership of {doc.name} in {subseries_doc.name} via sync to the rfc-index") - rfc_events.append(doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of {subseries_doc.name.upper()} is currently available in the datatracker before this point")) - rfc_events.append(doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Imported membership of {doc.name} in {subseries_doc.name} via sync to the rfc-index")) - else: - subseries_doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Added {doc.name} to {subseries_doc.name}") - rfc_events.append(doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Added {doc.name} to {subseries_doc.name}")) - - # Delete subseries relations that are no longer current. Use a transaction - # so we are sure we iterate over the same relations that we delete! - with transaction.atomic(): - stale_subseries_relations = doc.relations_that("contains").exclude( - source__name__in=also - ) - for stale_relation in stale_subseries_relations: - stale_subseries_doc = stale_relation.source - rfc_events.append( - doc.docevent_set.create( - type="sync_from_rfc_editor", - by=system, - desc=f"Removed {doc.name} from {stale_subseries_doc.name}", - ) - ) - stale_subseries_doc.docevent_set.create( - type="sync_from_rfc_editor", - by=system, - desc=f"Removed {doc.name} from {stale_subseries_doc.name}", - ) - stale_subseries_relations.delete() - - doc_errata = errata.get(f"RFC{rfc_number}", []) - all_rejected = doc_errata and all( - er["errata_status_code"] == "Rejected" for er in doc_errata - ) - if has_errata and not all_rejected: - if not doc.tags.filter(pk=tag_has_errata.pk).exists(): - doc.tags.add(tag_has_errata) - rfc_changes.append("added Errata tag") - has_verified_errata = any( - [er["errata_status_code"] == "Verified" for er in doc_errata] - ) - if ( - has_verified_errata - and not doc.tags.filter(pk=tag_has_verified_errata.pk).exists() - ): - doc.tags.add(tag_has_verified_errata) - rfc_changes.append("added Verified Errata tag") - else: - if doc.tags.filter(pk=tag_has_errata.pk): - doc.tags.remove(tag_has_errata) - if all_rejected: - rfc_changes.append("removed Errata tag (all errata rejected)") - else: - rfc_changes.append("removed Errata tag") - if doc.tags.filter(pk=tag_has_verified_errata.pk): - doc.tags.remove(tag_has_verified_errata) - rfc_changes.append("removed Verified Errata tag") - - if rfc_changes: - rfc_events.append( - DocEvent.objects.create( - doc=doc, - rev=doc.rev, - by=system, - type="sync_from_rfc_editor", - desc=f"Received changes through RFC Editor sync ({', '.join(rfc_changes)})", - ) - ) - doc.save_with_history(rfc_events) - yield rfc_number, rfc_changes, doc, rfc_published # yield changes to the RFC - - if first_sync_creating_subseries: - # First - create the known subseries documents that have ghosted. - # The RFC editor (as of 31 Oct 2023) claims these subseries docs do not exist. - # The datatracker, on the other hand, will say that the series doc currently contains no RFCs. - for name in ["fyi17", "std1", "bcp12", "bcp113", "bcp66"]: - # Leaving most things to the default intentionally - # Of note, title and stream are left to the defaults of "" and none. - subseries_doc, created = Document.objects.get_or_create(type_id=name[:3], name=name) - if not created: - log(f"Warning: {name} unexpectedly already exists") - else: - subseries_slug = name[:3] - subseries_doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of this {subseries_slug.upper()} document is currently available in the datatracker before this point") - - RelatedDocument.objects.filter( - Q(originaltargetaliasname__startswith="bcp") | - Q(originaltargetaliasname__startswith="std") | - Q(originaltargetaliasname__startswith="fyi") - ).annotate( - subseries_target=Subquery( - Document.objects.filter(name=OuterRef("originaltargetaliasname")).values_list("pk",flat=True)[:1] - ) - ).update(target=F("subseries_target")) - RelatedDocHistory.objects.filter( - Q(originaltargetaliasname__startswith="bcp") | - Q(originaltargetaliasname__startswith="std") | - Q(originaltargetaliasname__startswith="fyi") - ).annotate( - subseries_target=Subquery( - Document.objects.filter(name=OuterRef("originaltargetaliasname")).values_list("pk",flat=True)[:1] - ) - ).update(target=F("subseries_target")) - IprDocRel.objects.filter( - Q(originaldocumentaliasname__startswith="bcp") | - Q(originaldocumentaliasname__startswith="std") | - Q(originaldocumentaliasname__startswith="fyi") - ).annotate( - subseries_target=Subquery( - Document.objects.filter(name=OuterRef("originaldocumentaliasname")).values_list("pk",flat=True)[:1] - ) - ).update(document=F("subseries_target")) - - - diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index 4ccd5db4bb9..c61e754ed30 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -3,7 +3,6 @@ # Celery task definitions # import datetime -import io from pathlib import Path from tempfile import NamedTemporaryFile import requests @@ -19,13 +18,11 @@ from ietf.person.models import Person from ietf.utils.mail import send_mail_text from ietf.sync import iana -from ietf.sync import rfceditor from ietf.sync.errata import ( errata_are_dirty, mark_errata_as_processed, update_errata_from_rfceditor, ) -from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue from ietf.sync.rfcindex import ( create_bcp_txt_index, create_fyi_txt_index, @@ -36,92 +33,6 @@ ) from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper from ietf.utils import log -from ietf.utils.timezone import date_today - - -@shared_task -def rfc_editor_index_update_task(full_index=False): - """Update metadata from the RFC index - - Default is to examine only changes in the past 365 days. Call with full_index=True to update - the full RFC index. - - According to comments on the original script, a year's worth took about 20s on production as of - August 2022 - - The original rfc-editor-index-update script had a long-disabled provision for running the - rebuild_reference_relations scripts after the update. That has not been brought over - at all because it should be implemented as its own task if it is needed. - """ - skip_date = None if full_index else date_today() - datetime.timedelta(days=365) - log.log( - "Updating document metadata from RFC index going back to {since}, from {url}".format( - since=skip_date if skip_date is not None else "the beginning", - url=settings.RFC_EDITOR_INDEX_URL, - ) - ) - try: - response = requests.get( - settings.RFC_EDITOR_INDEX_URL, - timeout=30, # seconds - ) - except requests.Timeout as exc: - log.log(f"GET request timed out retrieving RFC editor index: {exc}") - return # failed - rfc_index_xml = response.text - index_data = rfceditor.parse_index(io.StringIO(rfc_index_xml)) - try: - response = requests.get( - settings.RFC_EDITOR_ERRATA_JSON_URL, - timeout=30, # seconds - ) - except requests.Timeout as exc: - log.log(f"GET request timed out retrieving RFC editor errata: {exc}") - return # failed - errata_data = response.json() - if len(index_data) < rfceditor.MIN_INDEX_RESULTS: - log.log("Not enough index entries, only %s" % len(index_data)) - return # failed - if len(errata_data) < rfceditor.MIN_ERRATA_RESULTS: - log.log("Not enough errata entries, only %s" % len(errata_data)) - return # failed - newly_published = set() - for rfc_number, changes, doc, rfc_published in rfceditor.update_docs_from_rfc_index( - index_data, errata_data, skip_older_than_date=skip_date - ): - for c in changes: - log.log("RFC%s, %s: %s" % (rfc_number, doc.name, c)) - if rfc_published: - newly_published.add(rfc_number) - if len(newly_published) > 0: - rsync_rfcs_from_rfceditor_task.delay(list(newly_published)) - - -@shared_task -def rfc_editor_queue_updates_task(): - log.log(f"Updating RFC Editor queue states from {settings.RFC_EDITOR_QUEUE_URL}") - try: - response = requests.get( - settings.RFC_EDITOR_QUEUE_URL, - timeout=30, # seconds - ) - except requests.Timeout as exc: - log.log(f"GET request timed out retrieving RFC editor queue: {exc}") - return # failed - drafts, warnings = parse_queue(io.StringIO(response.text)) - for w in warnings: - log.log(f"Warning: {w}") - - if len(drafts) < MIN_QUEUE_RESULTS: - log.log("Not enough results, only %s" % len(drafts)) - return # failed - - changed, warnings = update_drafts_from_queue(drafts) - for w in warnings: - log.log(f"Warning: {w}") - - for c in changed: - log.log(f"Updated {c}") @shared_task diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 99ca7b0008a..e13dcd19fd7 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -1,17 +1,11 @@ # Copyright The IETF Trust 2012-2026, All Rights Reserved - -import os -import io import json import datetime from unittest import mock import quopri import requests -from dataclasses import dataclass - -from django.conf import settings from django.core.files.base import ContentFile from django.core.files.storage import storages from django.urls import reverse as urlreverse @@ -24,24 +18,21 @@ from ietf.doc.factories import ( WgDraftFactory, RfcFactory, - DocumentAuthorFactory, DocEventFactory, - BcpFactory, WgRfcFactory, + WgRfcFactory, ) from ietf.doc.models import ( Document, DocEvent, DeletedEvent, DocTagName, - RelatedDocument, State, StateDocEvent, ) from ietf.doc.utils import add_state_change_event -from ietf.group.factories import GroupFactory from ietf.person.factories import PersonFactory from ietf.person.models import Person -from ietf.sync import iana, rfceditor, tasks +from ietf.sync import iana, tasks from ietf.sync.errata import ( update_errata_from_rfceditor, get_errata_last_updated, @@ -56,7 +47,6 @@ from ietf.utils.models import DirtyBits from ietf.utils.test_utils import login_testing_unauthorized from ietf.utils.test_utils import TestCase -from ietf.utils.timezone import date_today, RPC_TZINFO class IANASyncTests(TestCase): @@ -306,494 +296,6 @@ def test_notify_page(self): self.assertContains(r, "new changes at") # we don't actually try posting as that would trigger a real run - - -class RFCSyncTests(TestCase): - def write_draft_file(self, name, size): - with io.open(os.path.join(settings.INTERNET_DRAFT_PATH, name), 'w') as f: - f.write("a" * size) - - def test_rfc_index(self): - area = GroupFactory(type_id='area') - draft_doc = WgDraftFactory( - group__parent=area, - states=[('draft-iesg','rfcqueue')], - ad=Person.objects.get(user__username='ad'), - external_url="http://my-external-url.example.com", - note="this is a note", - pages=54, # make sure this is not 42 - ) - DocumentAuthorFactory.create_batch(2, document=draft_doc) - draft_doc.action_holders.add(draft_doc.ad) # not normally set, but add to be sure it's cleared - - RfcFactory(rfc_number=123) - - today = date_today() - - t = ''' - - - BCP0001 - - RFC1234 - RFC2345 - - - - FYI0001 - - RFC1234 - - - - STD0002 - Test - - RFC1234 - - - - RFC1234 - A Testing RFC - - A. Irector - - - %(month)s - %(year)s - - - ASCII - - 42 - - test - -

    This is some interesting text.

     - %(name)s-%(rev)s - - RFC123 - - - BCP0001 - - PROPOSED STANDARD - PROPOSED STANDARD - IETF - %(area)s - %(group)s - http://www.rfc-editor.org/errata_search.php?rfc=1234 -     -    ''' % dict(year=today.strftime("%Y"), - month=today.strftime("%B"), - name=draft_doc.name, - rev=draft_doc.rev, - area=draft_doc.group.parent.acronym, - group=draft_doc.group.acronym) - - errata = [{ - "errata_id":1, - "doc-id":"RFC123", # n.b. this is not the same RFC as in the above index XML! - "errata_status_code":"Verified", - "errata_type_code":"Editorial", - "section": "4.1", - "orig_text":" S: 220-smtp.example.com ESMTP Server", - "correct_text":" S: 220 smtp.example.com ESMTP Server", - "notes":"There are 3 instances of this (one on p. 7 and two on p. 8). \n", - "submit_date":"2007-07-19", - "submitter_name":"Rob Siemborski", - "verifier_id":99, - "verifier_name":None, - "update_date":"2019-09-10 09:09:03"}, - ] - - data = rfceditor.parse_index(io.StringIO(t)) - self.assertEqual(len(data), 1) - rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract = data[0] - - # currently, we only check what we actually use - self.assertEqual(rfc_number, 1234) - self.assertEqual(title, "A Testing RFC") - self.assertEqual(rfc_published_date.year, today.year) - self.assertEqual(rfc_published_date.month, today.month) - self.assertEqual(current_status, "Proposed Standard") - self.assertEqual(updates, ["RFC123"]) - self.assertEqual(set(also), set(["BCP1", "FYI1", "STD2"])) - self.assertEqual(draft, draft_doc.name) - self.assertEqual(wg, draft_doc.group.acronym) - self.assertEqual(has_errata, True) - self.assertEqual(stream, "IETF") - self.assertEqual(pages, "42") - self.assertEqual(abstract, "This is some interesting text.") - - draft_filename = "%s-%s.txt" % (draft_doc.name, draft_doc.rev) - self.write_draft_file(draft_filename, 5000) - - event_count_before = draft_doc.docevent_set.count() - draft_title_before = draft_doc.title - draft_abstract_before = draft_doc.abstract - draft_pages_before = draft_doc.pages - - changes = [] - with mock.patch("ietf.sync.rfceditor.log") as mock_log: - for rfc_number, _, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)): - changes.append({"doc_pk": d.pk, "rfc_published": rfc_published}) # we ignore the actual change list - self.assertEqual(rfc_number, 1234) - if rfc_published: - self.assertEqual(d.type_id, "rfc") - self.assertEqual(d.rfc_number, rfc_number) - else: - self.assertEqual(d.type_id, "draft") - self.assertIsNone(d.rfc_number) - - self.assertFalse(mock_log.called, "No log messages expected") - - draft_doc = Document.objects.get(name=draft_doc.name) - draft_events = draft_doc.docevent_set.all() - self.assertEqual(len(draft_events) - event_count_before, 2) - self.assertEqual(draft_events[0].type, "sync_from_rfc_editor") - self.assertEqual(draft_events[1].type, "changed_action_holders") - self.assertEqual(draft_doc.get_state_slug(), "rfc") - self.assertEqual(draft_doc.get_state_slug("draft-iesg"), "pub") - self.assertCountEqual(draft_doc.action_holders.all(), []) - self.assertEqual(draft_doc.title, draft_title_before) - self.assertEqual(draft_doc.abstract, draft_abstract_before) - self.assertEqual(draft_doc.pages, draft_pages_before) - self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, draft_filename))) - self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, draft_filename))) - - rfc_doc = Document.objects.filter(rfc_number=1234, type_id="rfc").first() - self.assertIsNotNone(rfc_doc, "RFC document should have been created") - self.assertEqual(rfc_doc.author_persons_or_names(), draft_doc.author_persons_or_names()) - rfc_events = rfc_doc.docevent_set.all() - self.assertEqual(len(rfc_events), 8) - expected_events = [ - ["sync_from_rfc_editor", ""], # Not looking for exact desc match here - see detailed tests below - ["sync_from_rfc_editor", "Imported membership of rfc1234 in std2 via sync to the rfc-index"], - ["std_history_marker", "No history of STD2 is currently available in the datatracker before this point"], - ["sync_from_rfc_editor", "Imported membership of rfc1234 in fyi1 via sync to the rfc-index"], - ["fyi_history_marker", "No history of FYI1 is currently available in the datatracker before this point"], - ["sync_from_rfc_editor", "Imported membership of rfc1234 in bcp1 via sync to the rfc-index"], - ["bcp_history_marker", "No history of BCP1 is currently available in the datatracker before this point"], - ["published_rfc", "RFC published"] - ] - for index, [event_type, desc] in enumerate(expected_events): - self.assertEqual(rfc_events[index].type, event_type) - if index == 0: - self.assertIn("Received changes through RFC Editor sync (created document RFC 1234,", rfc_events[0].desc) - self.assertIn(f"created became rfc relationship between {rfc_doc.came_from_draft().name} and RFC 1234", rfc_events[0].desc) - self.assertIn("set title to 'A Testing RFC'", rfc_events[0].desc) - self.assertIn("set abstract to 'This is some interesting text.'", rfc_events[0].desc) - self.assertIn("set pages to 42", rfc_events[0].desc) - self.assertIn("set standardization level to Proposed Standard", rfc_events[0].desc) - self.assertIn(f"added RFC published event at {rfc_events[0].time.astimezone(RPC_TZINFO):%Y-%m-%d}", rfc_events[0].desc) - self.assertIn("created updates relation between RFC 1234 and RFC 123", rfc_events[0].desc) - self.assertIn("added Errata tag", rfc_events[0].desc) - else: - self.assertEqual(rfc_events[index].desc, desc) - self.assertEqual(rfc_events[7].time.astimezone(RPC_TZINFO).date(), today) - for subseries_name in ["bcp1", "fyi1", "std2"]: - sub = Document.objects.filter(type_id=subseries_name[:3],name=subseries_name).first() - self.assertIsNotNone(sub, f"{subseries_name} not created") - self.assertTrue(rfc_doc in sub.contains()) - self.assertTrue(sub in rfc_doc.part_of()) - self.assertEqual(rfc_doc.get_state_slug(), "published") - # Should have an "errata" tag because there is an errata-url in the index XML, but no "verified-errata" tag - # because there is no verified item in the errata JSON with doc-id matching the RFC document. - tag_slugs = rfc_doc.tags.values_list("slug", flat=True) - self.assertTrue("errata" in tag_slugs) - self.assertFalse("verified-errata" in tag_slugs) - # TODO: adjust these when we have subseries document types - # self.assertTrue(DocAlias.objects.filter(name="rfc1234", docs=rfc_doc)) - # self.assertTrue(DocAlias.objects.filter(name="bcp1", docs=rfc_doc)) - # self.assertTrue(DocAlias.objects.filter(name="fyi1", docs=rfc_doc)) - # self.assertTrue(DocAlias.objects.filter(name="std1", docs=rfc_doc)) - self.assertTrue(RelatedDocument.objects.filter(source=rfc_doc, target__name="rfc123", relationship="updates").exists()) - self.assertTrue(RelatedDocument.objects.filter(source=draft_doc, target=rfc_doc, relationship="became_rfc").exists()) - self.assertEqual(rfc_doc.title, "A Testing RFC") - self.assertEqual(rfc_doc.abstract, "This is some interesting text.") - self.assertEqual(rfc_doc.std_level_id, "ps") - self.assertEqual(rfc_doc.pages, 42) - self.assertEqual(rfc_doc.stream, draft_doc.stream) - self.assertEqual(rfc_doc.group, draft_doc.group) - self.assertEqual(rfc_doc.words, draft_doc.words) - self.assertEqual(rfc_doc.ad, draft_doc.ad) - self.assertEqual(rfc_doc.external_url, draft_doc.external_url) - self.assertEqual(rfc_doc.note, draft_doc.note) - - # check that we got the expected changes - self.assertEqual(len(changes), 2) - self.assertEqual(changes[0]["doc_pk"], draft_doc.pk) - self.assertEqual(changes[0]["rfc_published"], False) - self.assertEqual(changes[1]["doc_pk"], rfc_doc.pk) - self.assertEqual(changes[1]["rfc_published"], True) - - # make sure we can apply it again with no changes - changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30))) - self.assertEqual(len(changed), 0) - - def test_rfc_index_subseries_replacement(self): - today = date_today() - author = PersonFactory(name="Some Bozo") - - # Start with two BCPs, each containing an rfc - rfc1, rfc2, rfc3 = RfcFactory.create_batch(3, authors=[author]) - bcp1 = BcpFactory(contains=[rfc1]) - bcp2 = BcpFactory(contains=[rfc2]) - - def _nameify(doc): - """Convert a name like 'rfc1' to 'RFC0001""" - return f"{doc.name[:3].upper()}{int(doc.name[3:]):04d}" - - # RFC index that replaces rfc2 with rfc3 in bcp2 - index_xml = f""" - - - {_nameify(bcp1)} - - {_nameify(rfc1)} - - - - {_nameify(bcp2)} - - {_nameify(rfc3)} - - - - {_nameify(rfc1)} - {rfc1.title} - - Some Bozo - - - {today.strftime('%B')} - {today.strftime('%Y')} - - - ASCII - - 42 - - test - -

    This is some interesting text.

     - - {_nameify(bcp1)} - - PROPOSED STANDARD - PROPOSED STANDARD - IETF -     - - {_nameify(rfc2)} - {rfc2.title} - - Some Bozo - - - {today.strftime('%B')} - {today.strftime('%Y')} - - - ASCII - - 42 - - test - -

    This is some interesting text.

     - PROPOSED STANDARD - PROPOSED STANDARD - IETF -     - - {_nameify(rfc3)} - {rfc3.title} - - Some Bozo - - - {today.strftime('%B')} - {today.strftime('%Y')} - - - ASCII - - 42 - - test - -

    This is some interesting text.

     - - {_nameify(bcp2)} - - PROPOSED STANDARD - PROPOSED STANDARD - IETF -     -    """ - data = rfceditor.parse_index(io.StringIO(index_xml)) # parse index - self.assertEqual(len(data), 3) # check that we parsed 3 RFCs - # Process the data by consuming the generator - for _ in rfceditor.update_docs_from_rfc_index(data, []): - pass - # Confirm that the expected changes were made - self.assertCountEqual(rfc1.related_that("contains"), [bcp1]) - self.assertCountEqual(rfc2.related_that("contains"), []) - self.assertCountEqual(rfc3.related_that("contains"), [bcp2]) - - def _generate_rfc_queue_xml(self, draft, state, auth48_url=None): - """Generate an RFC queue xml string for a draft""" - t = ''' -
    - -%(name)s-%(rev)s.txt -2010-09-08 -%(state)s -%(auth48_url)s - -%(ref)s -IN-QUEUE - -A. Author - -%(title)s - -10000000 -%(group)s - -
    -    ''' % dict(name=draft.name, - rev=draft.rev, - title=draft.title, - group=draft.group.name, - ref="draft-ietf-test", - state=state, - auth48_url=(auth48_url or '')) - t = t.replace('\n', '') # strip empty auth48-url tags - return t - - def test_rfc_queue(self): - draft = WgDraftFactory(states=[('draft-iesg','ann')], ad=Person.objects.get(user__username='ad')) - draft.action_holders.add(draft.ad) # add an action holder so we can test that it's removed later - - expected_auth48_url = "http://www.rfc-editor.org/auth48/rfc1234" - t = self._generate_rfc_queue_xml(draft, - state='EDIT*R*A(1G)', - auth48_url=expected_auth48_url) - - drafts, warnings = rfceditor.parse_queue(io.StringIO(t)) - # rfceditor.parse_queue() is tested independently; just sanity check here - self.assertEqual(len(drafts), 1) - self.assertEqual(len(warnings), 0) - - mailbox_before = len(outbox) - - changed, warnings = rfceditor.update_drafts_from_queue(drafts) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - - draft = Document.objects.get(pk=draft.pk) - self.assertEqual(draft.get_state_slug("draft-rfceditor"), "edit") - self.assertEqual(draft.get_state_slug("draft-iesg"), "rfcqueue") - self.assertCountEqual(draft.action_holders.all(), []) - self.assertEqual(set(draft.tags.all()), set(DocTagName.objects.filter(slug__in=("iana", "ref")))) - events = draft.docevent_set.all() - self.assertEqual(events[0].type, "changed_state") # changed draft-iesg state - self.assertEqual(events[1].type, "changed_action_holders") - self.assertEqual(events[2].type, "changed_state") # changed draft-rfceditor state - self.assertEqual(events[3].type, "rfc_editor_received_announcement") - - self.assertEqual(len(outbox), mailbox_before + 1) - self.assertTrue("RFC Editor queue" in outbox[-1]["Subject"]) - - # make sure we can apply it again with no changes - changed, warnings = rfceditor.update_drafts_from_queue(drafts) - self.assertEqual(len(changed), 0) - self.assertEqual(len(warnings), 0) - - def test_rfceditor_parse_queue(self): - """Test that rfceditor.parse_queue() behaves as expected. - - Currently does a limited test - old comment was - "currently, we only check what we actually use". - """ - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - t = self._generate_rfc_queue_xml(draft, - state='EDIT*R*A(1G)', - auth48_url="http://www.rfc-editor.org/auth48/rfc1234") - - drafts, warnings = rfceditor.parse_queue(io.StringIO(t)) - self.assertEqual(len(drafts), 1) - self.assertEqual(len(warnings), 0) - - draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs = drafts[0] - self.assertEqual(draft_name, draft.name) - self.assertEqual(state, "EDIT") - self.assertEqual(set(tags), set(["iana", "ref"])) - self.assertEqual(auth48, "http://www.rfc-editor.org/auth48/rfc1234") - - def test_rfceditor_parse_queue_TI_state(self): - # Test with TI state introduced 11 Sep 2019 - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - t = self._generate_rfc_queue_xml(draft, - state='TI', - auth48_url="http://www.rfc-editor.org/auth48/rfc1234") - __, warnings = rfceditor.parse_queue(io.StringIO(t)) - self.assertEqual(len(warnings), 0) - - def _generate_rfceditor_update(self, draft, state, tags=None, auth48_url=None): - """Helper to generate fake output from rfceditor.parse_queue()""" - return [[ - draft.name, # draft_name - '2020-06-03', # date_received - state, - tags or [], - '1', # missref_generation - 'ietf', # stream - auth48_url or '', - '', # cluster - ['draft-ietf-test'], # refs - ]] - - def test_update_draft_auth48_url(self): - """Test that auth48 URLs are handled correctly.""" - draft = WgDraftFactory(states=[('draft-iesg','ann')]) - - # Step 1 setup: update to a state with no auth48 URL - changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='EDIT') - ) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() - self.assertIsNone(auth48_docurl) - - # Step 2: update to auth48 state with auth48 URL - changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='AUTH48', auth48_url='http://www.rfc-editor.org/rfc1234') - ) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() - self.assertIsNotNone(auth48_docurl) - self.assertEqual(auth48_docurl.url, 'http://www.rfc-editor.org/rfc1234') - - # Step 3: update to auth48-done state without auth48 URL - changed, warnings = rfceditor.update_drafts_from_queue( - self._generate_rfceditor_update(draft, state='AUTH48-DONE') - ) - self.assertEqual(len(changed), 1) - self.assertEqual(len(warnings), 0) - auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() - self.assertIsNone(auth48_docurl) class DiscrepanciesTests(TestCase): def test_discrepancies(self): @@ -1075,192 +577,7 @@ def test_mark_errata_as_processed(self): class TaskTests(TestCase): - @override_settings( - RFC_EDITOR_INDEX_URL="https://rfc-editor.example.com/index/", - RFC_EDITOR_ERRATA_JSON_URL="https://rfc-editor.example.com/errata/", - ) - @mock.patch("ietf.sync.tasks.rfceditor.update_docs_from_rfc_index") - @mock.patch("ietf.sync.tasks.rfceditor.parse_index") - @mock.patch("ietf.sync.tasks.requests.get") - @mock.patch("ietf.sync.tasks.rsync_rfcs_from_rfceditor_task.delay") - def test_rfc_editor_index_update_task( - self, rsync_task_mock, requests_get_mock, parse_index_mock, update_docs_mock - ) -> None: # the annotation here prevents mypy from complaining about annotation-unchecked - """rfc_editor_index_update_task calls helpers correctly - - This tests that data flow is as expected. Assumes the individual helpers are - separately tested to function correctly. - """ - @dataclass - class MockIndexData: - """Mock index item that claims to be a specified length""" - length: int - - def __len__(self): - return self.length - - @dataclass - class MockResponse: - """Mock object that contains text and json() that claims to be a specified length""" - text: str - json_length: int = 0 - - def json(self): - return MockIndexData(length=self.json_length) - - # Response objects - index_response = MockResponse(text="this is the index") - errata_response = MockResponse( - text="these are the errata", json_length=rfceditor.MIN_ERRATA_RESULTS - ) - rfc = RfcFactory() - # Test with full_index = False - rsync_task_mock.return_value = None - requests_get_mock.side_effect = (index_response, errata_response) # will step through these - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) - update_docs_mock.return_value = ( - (rfc.rfc_number, ("something changed",), rfc, False), - ) - - tasks.rfc_editor_index_update_task(full_index=False) - - # Check parse_index() call - self.assertTrue(parse_index_mock.called) - (parse_index_args, _) = parse_index_mock.call_args - self.assertEqual( - parse_index_args[0].read(), # arg is a StringIO - "this is the index", - "parse_index is called with the index text in a StringIO", - ) - - # Check update_docs_from_rfc_index call - self.assertTrue(update_docs_mock.called) - (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args - self.assertEqual( - update_docs_args, (parse_index_mock.return_value, errata_response.json()) - ) - self.assertIsNotNone(update_docs_kwargs["skip_older_than_date"]) - - self.assertFalse(rsync_task_mock.called) - - # Test again with full_index = True - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = (index_response, errata_response) # will step through these - tasks.rfc_editor_index_update_task(full_index=True) - - # Check parse_index() call - self.assertTrue(parse_index_mock.called) - (parse_index_args, _) = parse_index_mock.call_args - self.assertEqual( - parse_index_args[0].read(), # arg is a StringIO - "this is the index", - "parse_index is called with the index text in a StringIO", - ) - - # Check update_docs_from_rfc_index call - self.assertTrue(update_docs_mock.called) - (update_docs_args, update_docs_kwargs) = update_docs_mock.call_args - self.assertEqual( - update_docs_args, (parse_index_mock.return_value, errata_response.json()) - ) - self.assertIsNone(update_docs_kwargs["skip_older_than_date"]) - - self.assertFalse(rsync_task_mock.called) - - # Test again where the index would cause a new RFC to come into existance - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = ( - index_response, - errata_response, - ) # will step through these - update_docs_mock.return_value = ( - (rfc.rfc_number, ("something changed",), rfc, True), - ) - tasks.rfc_editor_index_update_task(full_index=True) - self.assertTrue(rsync_task_mock.called) - rsync_task_args, rsync_task_kwargs = rsync_task_mock.call_args - self.assertEqual((([rfc.rfc_number],), {}), (rsync_task_args, rsync_task_kwargs)) - - # Test error handling - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = requests.Timeout # timeout on every get() - tasks.rfc_editor_index_update_task(full_index=False) - self.assertFalse(parse_index_mock.called) - self.assertFalse(update_docs_mock.called) - self.assertFalse(rsync_task_mock.called) - - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = [index_response, requests.Timeout] # timeout second get() - tasks.rfc_editor_index_update_task(full_index=False) - self.assertFalse(update_docs_mock.called) - self.assertFalse(rsync_task_mock.called) - - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = [index_response, errata_response] - # feed in an index that is too short - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS - 1) - tasks.rfc_editor_index_update_task(full_index=False) - self.assertTrue(parse_index_mock.called) - self.assertFalse(update_docs_mock.called) - self.assertFalse(rsync_task_mock.called) - - requests_get_mock.reset_mock() - parse_index_mock.reset_mock() - update_docs_mock.reset_mock() - rsync_task_mock.reset_mock() - requests_get_mock.side_effect = [index_response, errata_response] - errata_response.json_length = rfceditor.MIN_ERRATA_RESULTS - 1 # too short - parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) - tasks.rfc_editor_index_update_task(full_index=False) - self.assertFalse(update_docs_mock.called) - self.assertFalse(rsync_task_mock.called) - - @override_settings(RFC_EDITOR_QUEUE_URL="https://rfc-editor.example.com/queue/") - @mock.patch("ietf.sync.tasks.update_drafts_from_queue") - @mock.patch("ietf.sync.tasks.parse_queue") - def test_rfc_editor_queue_updates_task(self, mock_parse, mock_update): - # test a request timeout - self.requests_mock.get("https://rfc-editor.example.com/queue/", exc=requests.exceptions.Timeout) - tasks.rfc_editor_queue_updates_task() - self.assertFalse(mock_parse.called) - self.assertFalse(mock_update.called) - - # now return a value rather than an exception - self.requests_mock.get("https://rfc-editor.example.com/queue/", text="the response") - - # mock returning < MIN_QUEUE_RESULTS values - treated as an error, so no update takes place - mock_parse.return_value = ([n for n in range(rfceditor.MIN_QUEUE_RESULTS - 1)], ["a warning"]) - tasks.rfc_editor_queue_updates_task() - self.assertEqual(mock_parse.call_count, 1) - self.assertEqual(mock_parse.call_args[0][0].read(), "the response") - self.assertFalse(mock_update.called) - mock_parse.reset_mock() - - # mock returning +. MIN_QUEUE_RESULTS - should succeed - mock_parse.return_value = ([n for n in range(rfceditor.MIN_QUEUE_RESULTS)], ["a warning"]) - mock_update.return_value = ([1,2,3], ["another warning"]) - tasks.rfc_editor_queue_updates_task() - self.assertEqual(mock_parse.call_count, 1) - self.assertEqual(mock_parse.call_args[0][0].read(), "the response") - self.assertEqual(mock_update.call_count, 1) - self.assertEqual(mock_update.call_args, mock.call([n for n in range(rfceditor.MIN_QUEUE_RESULTS)])) - @override_settings(IANA_SYNC_CHANGES_URL="https://iana.example.com/sync/") @mock.patch("ietf.sync.tasks.iana.update_history_with_changes") @mock.patch("ietf.sync.tasks.iana.parse_changes_json") diff --git a/ietf/sync/views.py b/ietf/sync/views.py index bd539d4e186..58209c2c6ee 100644 --- a/ietf/sync/views.py +++ b/ietf/sync/views.py @@ -35,7 +35,6 @@ def notify(request, org, notification): known_orgs = { "iana": "IANA", - "rfceditor": "RFC Editor", } if org not in known_orgs: @@ -67,29 +66,13 @@ def notify(request, org, notification): known_notifications = { "protocols": "an added reference to an RFC at the IANA protocols page" % settings.IANA_SYNC_PROTOCOLS_URL, "changes": "new changes at the changes JSON dump" % settings.IANA_SYNC_CHANGES_URL, - "queue": "new changes to queue2.xml" % settings.RFC_EDITOR_QUEUE_URL, - "index": "new changes to rfc-index.xml" % settings.RFC_EDITOR_INDEX_URL, } if notification not in known_notifications: raise Http404 if request.method == "POST": - if notification == "index": - log("Queuing RFC Editor index sync from notify view POST") - # Wrap in on_commit in case a transaction is open - # (As of 2024-11-08, this only runs in a transaction during tests) - transaction.on_commit( - lambda: tasks.rfc_editor_index_update_task.delay() - ) - elif notification == "queue": - log("Queuing RFC Editor queue sync from notify view POST") - # Wrap in on_commit in case a transaction is open - # (As of 2024-11-08, this only runs in a transaction during tests) - transaction.on_commit( - lambda: tasks.rfc_editor_queue_updates_task.delay() - ) - elif notification == "changes": + if notification == "changes": log("Queuing IANA changes sync from notify view POST") # Wrap in on_commit in case a transaction is open # (As of 2024-11-08, this only runs in a transaction during tests) diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index eab1d779fba..9b967c013bf 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -601,7 +601,7 @@ diff --git a/ietf/templates/doc/search/status_columns.html b/ietf/templates/doc/search/status_columns.html index 5ba41bb9c4a..50d70fcb379 100644 --- a/ietf/templates/doc/search/status_columns.html +++ b/ietf/templates/doc/search/status_columns.html @@ -11,7 +11,7 @@ {{ doc.friendly_state|safe }} {% endif %} {% if doc|state:"draft-rfceditor" %} - : {{ doc|state:"draft-rfceditor" }} + : {{ doc|state:"draft-rfceditor" }} {% endif %} {{ doc|auth48_alert_badge }} {{ doc|state_age_colored }} diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index 2d34f8361c1..c878ba49f10 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -83,34 +83,6 @@ def create_default_tasks(self): ), ) - PeriodicTask.objects.get_or_create( - name="Partial sync with RFC Editor index", - task="ietf.sync.tasks.rfc_editor_index_update_task", - kwargs=json.dumps(dict(full_index=False)), - defaults=dict( - enabled=False, - crontab=self.crontabs["every_15m_except_midnight"], # don't collide with full sync - description=( - "Reparse the last _year_ of RFC index entries until " - "https://github.com/ietf-tools/datatracker/issues/3734 is addressed. " - "This takes about 20s on production as of 2022-08-11." - ) - ), - ) - - PeriodicTask.objects.get_or_create( - name="Full sync with RFC Editor index", - task="ietf.sync.tasks.rfc_editor_index_update_task", - kwargs=json.dumps(dict(full_index=True)), - defaults=dict( - enabled=False, - crontab=self.crontabs["daily"], - description=( - "Run an extended version of the rfc editor update to catch changes with backdated timestamps" - ), - ), - ) - PeriodicTask.objects.get_or_create( name="Fetch meeting attendance", task="ietf.stats.tasks.fetch_meeting_attendance_task", diff --git a/ietf/utils/timezone.py b/ietf/utils/timezone.py index e08dfa02f2c..5e82c225dc6 100644 --- a/ietf/utils/timezone.py +++ b/ietf/utils/timezone.py @@ -15,7 +15,7 @@ DEADLINE_TZINFO = ZoneInfo('PST8PDT') # Time zone for dates from the RPC. This value is baked into the timestamps on DocEvents -# of type="published_rfc" - see Document.pub_date() and ietf.sync.refceditor.update_docs_from_rfc_index() +# of type="published_rfc" - see Document.pub_date() and the rfc publication workflow # for more information about how that works. RPC_TZINFO = ZoneInfo('PST8PDT') From ea7406f5ca64dbc6ce0fb85a3c9ca029c4a68ca6 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 1 Jul 2026 07:58:52 +1200 Subject: [PATCH 295/313] fix: consistent counts at proceedings and proceedings/attendees (#10935) * fix: consistent counts at proceedings and proceedings/attendees * fix: reconstitute reg objects directly from person objects --------- Co-authored-by: Jennifer Richards --- ietf/meeting/models.py | 107 +++++++++--- ietf/meeting/tests_models.py | 329 ++++++++++++++++++++++++----------- ietf/meeting/tests_views.py | 30 +++- ietf/meeting/utils.py | 11 -- ietf/meeting/views.py | 27 ++- ietf/nomcom/utils.py | 5 +- 6 files changed, 350 insertions(+), 159 deletions(-) diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 7d9e318aabc..8cf386abf29 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -236,16 +236,24 @@ def get_proceedings_materials(self): document__states__slug='active', document__states__type_id='procmaterials' ).order_by('type__order') - def get_attendance(self): - """Get the meeting attendance from the Registrations + def get_attendees(self, sessions=None): + """Get the meeting attendees from the Registrations - Returns a NamedTuple with onsite and remote attributes. Returns None if the record is unavailable - for this meeting. + Returns a pair of sets containing Person objects for persons attending + the meeting either onsite or online (remote). Returns None if the record + is unavailable for this meeting. + + sessions: optional queryset of Sessions to restrict which Attended records + count toward attendance. Defaults to all sessions at this meeting. + Note: restricting sessions does not affect the registration-based path + (checkedin / attended flags), only the Attended-record path. """ number = self.get_number() if number is None or number < 110: return None - Attendance = namedtuple('Attendance', 'onsite remote') + + if sessions is None: + sessions = self.session_set.all() # MeetingRegistration.attended started conflating badge-pickup and session attendance before IETF 114. # We've separated session attendance off to ietf.meeting.Attended, but need to report attendance at older @@ -254,11 +262,8 @@ def get_attendance(self): # Looking up by registration and attendance records separately and joining in # python is far faster than combining the Q objects in the query (~100x). # Further optimization may be possible, but the queries are tricky... - attended_per_meeting_registration = ( - Q(registration__meeting=self) & ( - Q(registration__attended=True) | - Q(registration__checkedin=True) - ) + attended_per_meeting_registration = Q(registration__meeting=self) & ( + Q(registration__attended=True) | Q(registration__checkedin=True) ) attendees_by_reg = set( Person.objects.filter(attended_per_meeting_registration).values_list( @@ -266,29 +271,81 @@ def get_attendance(self): ) ) - attended_per_meeting_attended = ( - Q(attended__session__meeting=self) - # Note that we are not filtering to plenary, wg, or rg sessions - # as we do for nomcom eligibility - if picking up a badge (see above) - # is good enough, just attending e.g. a training session is also good enough - ) attendees_by_att = set( - Person.objects.filter(attended_per_meeting_attended).values_list( + Person.objects.filter(attended__session__in=sessions).values_list( "pk", flat=True ) ) - - attendees = Person.objects.filter( - pk__in=attendees_by_att | attendees_by_reg + + attendees = Person.objects.filter(pk__in=attendees_by_att | attendees_by_reg) + onsite = set( + attendees.filter( + registration__meeting=self, + registration__tickets__attendance_type__slug="onsite", + ) + ) + remote = set( + attendees.filter( + registration__meeting=self, + registration__tickets__attendance_type__slug="remote", + ) ) - onsite = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='onsite')) - remote = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='remote')) remote.difference_update(onsite) - return Attendance( - onsite=len(onsite), - remote=len(remote) + return (onsite, remote) + + def get_attendance(self): + """Get the meeting attendance from the Registrations + + Returns a NamedTuple with onsite and remote attributes. Returns None if the record is unavailable + for this meeting. + """ + attendees = self.get_attendees() + if attendees is None: + return None + Attendance = namedtuple("Attendance", "onsite remote") + onsite, remote = attendees + return Attendance(onsite=len(onsite), remote=len(remote)) + + def nomcom_eligible_participants(self): + """Return (onsite_pks, remote_pks) as frozensets for nomcom eligibility. + + Onsite: has onsite ticket AND (checked in OR attended a qualifying session). + Remote: has remote ticket AND attended a qualifying plenary/wg/rg session. + The registration attended flag alone is not sufficient for remote. + """ + sessions = self.session_set.filter( + Q(type="plenary") | Q(group__type__in=["wg", "rg"]) + ) + attendees = self.get_attendees(sessions=sessions) + if attendees is None: + onsite_pks = frozenset( + self.registration_set.onsite() + .filter(checkedin=True) + .values_list("person", flat=True) + .distinct() + ) + remote_pks = ( + frozenset( + Attended.objects.filter(session__in=sessions) + .values_list("person", flat=True) + .distinct() + ) + - onsite_pks + ) + return (onsite_pks, remote_pks) + onsite, remote = attendees + onsite_pks = frozenset(p.pk for p in onsite) + # Remote requires actual qualifying session attendance; the registration + # attended flag alone (used as a proxy before Attended records existed) + # is not sufficient. + session_attended_pks = frozenset( + Attended.objects.filter(session__in=sessions) + .values_list("person", flat=True) + .distinct() ) + remote_pks = frozenset(p.pk for p in remote) & session_attended_pks + return (onsite_pks, remote_pks) @property def proceedings_format_version(self): diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py index 869d9ec814e..0b9ce3f607f 100644 --- a/ietf/meeting/tests_models.py +++ b/ietf/meeting/tests_models.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2021-2024, All Rights Reserved # -*- coding: utf-8 -*- """Tests of models in the Meeting application""" + import datetime from unittest.mock import patch @@ -10,7 +11,12 @@ import ietf.meeting.models from ietf.group.factories import GroupFactory, GroupHistoryFactory -from ietf.meeting.factories import MeetingFactory, SessionFactory, AttendedFactory, SessionPresentationFactory +from ietf.meeting.factories import ( + MeetingFactory, + SessionFactory, + AttendedFactory, + SessionPresentationFactory, +) from ietf.meeting.factories import RegistrationFactory from ietf.meeting.models import Session from ietf.utils.test_utils import TestCase @@ -18,105 +24,200 @@ class MeetingTests(TestCase): - def test_get_attendance_pre110(self): - """Pre-110 meetings do not calculate attendance""" - meeting = MeetingFactory(type_id='ietf', number='109') - RegistrationFactory.create_batch(3, meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}) - RegistrationFactory.create_batch(4, meeting=meeting, with_ticket={'attendance_type_id': 'remote'}) - RegistrationFactory.create_batch(5, meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}) - self.assertIsNone(meeting.get_attendance()) - - def test_get_attendance_110(self): - """Look at attendance as captured at 110""" - meeting = MeetingFactory(type_id='ietf', number='110') - - # start with attendees that should be ignored - RegistrationFactory.create_batch(3, meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}, attended=True) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}, attended=False) - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 0) - self.assertEqual(attendance.onsite, 0) - - # add online attendees with at least one who registered but did not attend - RegistrationFactory.create_batch(4, meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=True) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=False) - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 4) - self.assertEqual(attendance.onsite, 0) - - # and the same for onsite attendees - RegistrationFactory.create_batch(5, meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=True) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False) - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 4) - self.assertEqual(attendance.onsite, 5) - - # and once more after removing all the online attendees + def test_get_attendees_pre110(self): + """Pre-110 meetings return None""" + meeting = MeetingFactory(type_id="ietf", number="109") + RegistrationFactory.create_batch( + 3, meeting=meeting, with_ticket={"attendance_type_id": "unknown"} + ) + RegistrationFactory.create_batch( + 4, meeting=meeting, with_ticket={"attendance_type_id": "remote"} + ) + RegistrationFactory.create_batch( + 5, meeting=meeting, with_ticket={"attendance_type_id": "onsite"} + ) + self.assertIsNone(meeting.get_attendees()) + + def test_get_attendees_110(self): + """Registration-based attendance (attended flag) used at IETF 110""" + meeting = MeetingFactory(type_id="ietf", number="110") + + # Unknown-ticket attendees are excluded regardless of attended flag + RegistrationFactory.create_batch( + 3, + meeting=meeting, + with_ticket={"attendance_type_id": "unknown"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "unknown"}, + attended=False, + ) + onsite, remote = meeting.get_attendees() + self.assertEqual(len(onsite), 0) + self.assertEqual(len(remote), 0) + + # Remote attendees: only those with attended=True are included + remote_regs = RegistrationFactory.create_batch( + 4, + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + ) + onsite, remote = meeting.get_attendees() + self.assertEqual({p.pk for p in onsite}, set()) + self.assertEqual({p.pk for p in remote}, {r.person.pk for r in remote_regs}) + + # Onsite attendees: only those with attended=True are included + onsite_regs = RegistrationFactory.create_batch( + 5, + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + ) + RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + ) + onsite, remote = meeting.get_attendees() + self.assertEqual({p.pk for p in onsite}, {r.person.pk for r in onsite_regs}) + self.assertEqual({p.pk for p in remote}, {r.person.pk for r in remote_regs}) + + # Deleting remote registrations empties the remote set meeting.registration_set.remote().delete() - attendance = meeting.get_attendance() - self.assertIsNotNone(attendance) - self.assertEqual(attendance.remote, 0) - self.assertEqual(attendance.onsite, 5) - - def test_get_attendance_113(self): - """Simulate IETF 113 attendance gathering data""" - meeting = MeetingFactory(type_id='ietf', number='113') - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=True, checkedin=False) - RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=True) - p1 = RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=False).person - AttendedFactory(session__meeting=meeting, person=p1) - p2 = RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=False, checkedin=False).person - AttendedFactory(session__meeting=meeting, person=p2) - attendance = meeting.get_attendance() - self.assertEqual(attendance.onsite, 3) - self.assertEqual(attendance.remote, 1) - - def test_get_attendance_keeps_meetings_distinct(self): + onsite, remote = meeting.get_attendees() + self.assertEqual({p.pk for p in onsite}, {r.person.pk for r in onsite_regs}) + self.assertEqual(len(remote), 0) + + def test_get_attendees_113(self): + """Simulate IETF 113: mix of attended flag, checkedin flag, and Attended records""" + meeting = MeetingFactory(type_id="ietf", number="113") + p_attended = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + checkedin=False, + ).person + p_checkedin = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + checkedin=True, + ).person + p_onsite_session = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + checkedin=False, + ).person + AttendedFactory(session__meeting=meeting, person=p_onsite_session) + p_remote_session = RegistrationFactory( + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + checkedin=False, + ).person + AttendedFactory(session__meeting=meeting, person=p_remote_session) + onsite, remote = meeting.get_attendees() + self.assertEqual( + {p.pk for p in onsite}, {p_attended.pk, p_checkedin.pk, p_onsite_session.pk} + ) + self.assertEqual({p.pk for p in remote}, {p_remote_session.pk}) + + def test_get_attendees_keeps_meetings_distinct(self): """No cross-talk between attendance for different meetings""" - # numbers are arbitrary here - first_mtg = MeetingFactory(type_id='ietf', number='114') - second_mtg = MeetingFactory(type_id='ietf', number='115') + first_mtg = MeetingFactory(type_id="ietf", number="114") + second_mtg = MeetingFactory(type_id="ietf", number="115") - # Create a person who attended a remote session for first_mtg and onsite for second_mtg without - # checking in for either. - p = RegistrationFactory(meeting=second_mtg, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=False).person + # Person with remote ticket at first_mtg and onsite ticket at second_mtg, attended both via Attended records + p = RegistrationFactory( + meeting=second_mtg, + with_ticket={"attendance_type_id": "onsite"}, + attended=False, + checkedin=False, + ).person + RegistrationFactory( + meeting=first_mtg, + person=p, + with_ticket={"attendance_type_id": "remote"}, + attended=False, + checkedin=False, + ) AttendedFactory(session__meeting=first_mtg, person=p) - RegistrationFactory(meeting=first_mtg, person=p, with_ticket={'attendance_type_id': 'remote'}, attended=False, checkedin=False) AttendedFactory(session__meeting=second_mtg, person=p) - att = first_mtg.get_attendance() - self.assertEqual(att.onsite, 0) - self.assertEqual(att.remote, 1) + first_onsite, first_remote = first_mtg.get_attendees() + self.assertEqual({q.pk for q in first_onsite}, set()) + self.assertEqual({q.pk for q in first_remote}, {p.pk}) + + second_onsite, second_remote = second_mtg.get_attendees() + self.assertEqual({q.pk for q in second_onsite}, {p.pk}) + self.assertEqual({q.pk for q in second_remote}, set()) + + def test_get_attendance(self): + """get_attendance delegates to get_attendees and returns counts as a NamedTuple""" + meeting = MeetingFactory(type_id="ietf", number="120") + onsite_regs = RegistrationFactory.create_batch( + 3, + meeting=meeting, + with_ticket={"attendance_type_id": "onsite"}, + attended=True, + ) + remote_regs = RegistrationFactory.create_batch( + 2, + meeting=meeting, + with_ticket={"attendance_type_id": "remote"}, + attended=True, + ) + att = meeting.get_attendance() + self.assertIsNotNone(att) + self.assertEqual(att.onsite, len(onsite_regs)) + self.assertEqual(att.remote, len(remote_regs)) - att = second_mtg.get_attendance() - self.assertEqual(att.onsite, 1) - self.assertEqual(att.remote, 0) + old_meeting = MeetingFactory(type_id="ietf", number="109") + self.assertIsNone(old_meeting.get_attendance()) def test_vtimezone(self): # normal time zone that should have a zoneinfo file - meeting = MeetingFactory(type_id='ietf', time_zone='America/Los_Angeles', populate_schedule=False) + meeting = MeetingFactory( + type_id="ietf", time_zone="America/Los_Angeles", populate_schedule=False + ) vtz = meeting.vtimezone() self.assertIsNotNone(vtz) self.assertGreater(len(vtz), 0) # time zone that does not have a zoneinfo file should return None - meeting = MeetingFactory(type_id='ietf', time_zone='Fake/Time_Zone', populate_schedule=False) + meeting = MeetingFactory( + type_id="ietf", time_zone="Fake/Time_Zone", populate_schedule=False + ) vtz = meeting.vtimezone() self.assertIsNone(vtz) # ioerror trying to read zoneinfo should return None - meeting = MeetingFactory(type_id='ietf', time_zone='America/Los_Angeles', populate_schedule=False) - with patch('ietf.meeting.models.io.open', side_effect=IOError): + meeting = MeetingFactory( + type_id="ietf", time_zone="America/Los_Angeles", populate_schedule=False + ) + with patch("ietf.meeting.models.io.open", side_effect=IOError): vtz = meeting.vtimezone() self.assertIsNone(vtz) def test_group_at_the_time(self): - m = MeetingFactory(type_id='ietf', date=date_today() - datetime.timedelta(days=10)) + m = MeetingFactory( + type_id="ietf", date=date_today() - datetime.timedelta(days=10) + ) cached_groups = GroupFactory.create_batch(2) m.cached_groups_at_the_time = {g.pk: g for g in cached_groups} # fake the cache - uncached_group_hist = GroupHistoryFactory(time=datetime_today() - datetime.timedelta(days=30)) - self.assertEqual(m.group_at_the_time(uncached_group_hist.group), uncached_group_hist) + uncached_group_hist = GroupHistoryFactory( + time=datetime_today() - datetime.timedelta(days=30) + ) + self.assertEqual( + m.group_at_the_time(uncached_group_hist.group), uncached_group_hist + ) self.assertIn(uncached_group_hist.group.pk, m.cached_groups_at_the_time) @@ -127,27 +228,36 @@ def test_chat_archive_url(self): meeting__number=120, # needs to use proceedings_format_version > 1 ) with override_settings(): - if hasattr(settings, 'CHAT_ARCHIVE_URL_PATTERN'): + if hasattr(settings, "CHAT_ARCHIVE_URL_PATTERN"): del settings.CHAT_ARCHIVE_URL_PATTERN self.assertEqual(session.chat_archive_url(), session.chat_room_url()) - settings.CHAT_ARCHIVE_URL_PATTERN = 'http://chat.example.com' - self.assertEqual(session.chat_archive_url(), 'http://chat.example.com') - chatlog = SessionPresentationFactory(session=session, document__type_id='chatlog').document + settings.CHAT_ARCHIVE_URL_PATTERN = "http://chat.example.com" + self.assertEqual(session.chat_archive_url(), "http://chat.example.com") + chatlog = SessionPresentationFactory( + session=session, document__type_id="chatlog" + ).document self.assertEqual(session.chat_archive_url(), chatlog.get_href()) # datatracker 8.8.0 rolled out on 2022-07-15. Before that, chat logs were jabber logs hosted at www.ietf.org. - session_with_jabber = SessionFactory(group__acronym='fakeacronym', meeting__date=datetime.date(2022,7,14)) - self.assertEqual(session_with_jabber.chat_archive_url(), 'https://www.ietf.org/jabber/logs/fakeacronym?C=M;O=D') - chatlog = SessionPresentationFactory(session=session_with_jabber, document__type_id='chatlog').document + session_with_jabber = SessionFactory( + group__acronym="fakeacronym", meeting__date=datetime.date(2022, 7, 14) + ) + self.assertEqual( + session_with_jabber.chat_archive_url(), + "https://www.ietf.org/jabber/logs/fakeacronym?C=M;O=D", + ) + chatlog = SessionPresentationFactory( + session=session_with_jabber, document__type_id="chatlog" + ).document self.assertEqual(session_with_jabber.chat_archive_url(), chatlog.get_href()) def test_chat_room_name(self): - session = SessionFactory(group__acronym='xyzzy') - self.assertEqual(session.chat_room_name(), 'xyzzy') - session.type_id = 'plenary' - self.assertEqual(session.chat_room_name(), 'plenary') - session.chat_room = 'fnord' - self.assertEqual(session.chat_room_name(), 'fnord') + session = SessionFactory(group__acronym="xyzzy") + self.assertEqual(session.chat_room_name(), "xyzzy") + session.type_id = "plenary" + self.assertEqual(session.chat_room_name(), "plenary") + session.chat_room = "fnord" + self.assertEqual(session.chat_room_name(), "fnord") def test_alpha_str(self): self.assertEqual(Session._alpha_str(0), "a") @@ -157,7 +267,11 @@ def test_alpha_str(self): self.assertEqual(Session._alpha_str(27 * 26 - 1), "zz") self.assertEqual(Session._alpha_str(27 * 26), "aaa") - @patch.object(ietf.meeting.models.Session, "_session_recording_url_label", return_value="LABEL") + @patch.object( + ietf.meeting.models.Session, + "_session_recording_url_label", + return_value="LABEL", + ) def test_session_recording_url(self, mock): for session_type in ["ietf", "interim"]: session = SessionFactory(meeting__type_id=session_type) @@ -165,20 +279,29 @@ def test_session_recording_url(self, mock): if hasattr(settings, "MEETECHO_SESSION_RECORDING_URL"): del settings.MEETECHO_SESSION_RECORDING_URL self.assertIsNone(session.session_recording_url()) - + settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com" - self.assertEqual(session.session_recording_url(), "http://player.example.com") - - settings.MEETECHO_SESSION_RECORDING_URL = "http://player.example.com?{session_label}" - self.assertEqual(session.session_recording_url(), "http://player.example.com?LABEL") + self.assertEqual( + session.session_recording_url(), "http://player.example.com" + ) + + settings.MEETECHO_SESSION_RECORDING_URL = ( + "http://player.example.com?{session_label}" + ) + self.assertEqual( + session.session_recording_url(), "http://player.example.com?LABEL" + ) - session.meetecho_recording_name="actualname" + session.meetecho_recording_name = "actualname" session.save() - self.assertEqual(session.session_recording_url(), "http://player.example.com?actualname") + self.assertEqual( + session.session_recording_url(), + "http://player.example.com?actualname", + ) def test_session_recording_url_label_ietf(self): session = SessionFactory( - meeting__type_id='ietf', + meeting__type_id="ietf", meeting__date=date_today(), meeting__number="123", group__acronym="acro", @@ -186,15 +309,17 @@ def test_session_recording_url_label_ietf(self): session_time = session.official_timeslotassignment().timeslot.time self.assertEqual( f"IETF123-ACRO-{session_time:%Y%m%d-%H%M}", # n.b., time in label is UTC - session._session_recording_url_label()) + session._session_recording_url_label(), + ) def test_session_recording_url_label_interim(self): session = SessionFactory( - meeting__type_id='interim', + meeting__type_id="interim", meeting__date=date_today(), group__acronym="acro", ) session_time = session.official_timeslotassignment().timeslot.time self.assertEqual( f"IETF-ACRO-{session_time:%Y%m%d-%H%M}", # n.b., time in label is UTC - session._session_recording_url_label()) + session._session_recording_url_label(), + ) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 17988e50be4..eea08be8c77 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -55,7 +55,7 @@ generate_proceedings_content, diff_meeting_schedules, ) -from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting +from ietf.meeting.utils import add_event_info_to_session_qs from ietf.meeting.utils import create_recording, delete_recording, get_next_sequence, bluesheet_data from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose, generate_agenda_data @@ -9440,20 +9440,32 @@ def test_get_next_sequence(self): sequence = get_next_sequence(group,meeting,'recording') self.assertEqual(sequence,1) - def test_participants_for_meeting(self): + def test_nomcom_eligible_participants_for_meeting(self): m = MeetingFactory.create(type_id='ietf') areg = RegistrationFactory(meeting=m, checkedin=True, with_ticket={'attendance_type_id': 'onsite'}) breg = RegistrationFactory(meeting=m, checkedin=False, with_ticket={'attendance_type_id': 'onsite'}) creg = RegistrationFactory(meeting=m, with_ticket={'attendance_type_id': 'remote'}) dreg = RegistrationFactory(meeting=m, with_ticket={'attendance_type_id': 'remote'}) AttendedFactory(session__meeting=m, session__type_id='plenary', person=creg.person) - checked_in, attended = participants_for_meeting(m) - self.assertIn(areg.person.pk, checked_in) - self.assertNotIn(breg.person.pk, checked_in) - self.assertNotIn(areg.person.pk, attended) - self.assertNotIn(breg.person.pk, attended) - self.assertIn(creg.person.pk, attended) - self.assertNotIn(dreg.person.pk, attended) + onsite_pks, remote_pks = m.nomcom_eligible_participants() + self.assertIn(areg.person.pk, onsite_pks) + self.assertNotIn(breg.person.pk, onsite_pks) + self.assertNotIn(areg.person.pk, remote_pks) + self.assertNotIn(breg.person.pk, remote_pks) + self.assertIn(creg.person.pk, remote_pks) + self.assertNotIn(dreg.person.pk, remote_pks) + + def test_nomcom_eligible_participants_remote_requires_session(self): + """Remote registration.attended=True without a qualifying session does not count (>= 110 path)""" + m = MeetingFactory.create(type_id='ietf', number='119') + # Remote with attended=True but no Attended record: should not be eligible + no_session_reg = RegistrationFactory(meeting=m, attended=True, with_ticket={'attendance_type_id': 'remote'}) + # Remote with attended=True AND a qualifying Attended record: should be eligible + session_reg = RegistrationFactory(meeting=m, attended=True, with_ticket={'attendance_type_id': 'remote'}) + AttendedFactory(session__meeting=m, session__type_id='plenary', person=session_reg.person) + onsite_pks, remote_pks = m.nomcom_eligible_participants() + self.assertNotIn(no_session_reg.person.pk, remote_pks) + self.assertIn(session_reg.person.pk, remote_pks) def test_session_attendance(self): meeting = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number='118') diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 10ae0d3667d..ffd37fc363d 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -1362,17 +1362,6 @@ def post_process(doc): doc.save_with_history([e]) -def participants_for_meeting(meeting): - """ Return a tuple (checked_in, attended) - checked_in = queryset of onsite, checkedin participants values_list('person') - attended = queryset of remote participants who attended a session values_list('person') - """ - checked_in = meeting.registration_set.onsite().filter(checkedin=True).values_list('person', flat=True).distinct() - sessions = meeting.session_set.filter(Q(type='plenary') | Q(group__type__in=['wg', 'rg'])) - attended = Attended.objects.filter(session__in=sessions).values_list('person', flat=True).distinct() - return (checked_in, attended) - - def generate_proceedings_content(meeting, force_refresh=False): """Render proceedings content for a meeting and update cache diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 67a81305b49..913c8bd3021 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -107,7 +107,7 @@ from ietf.meeting.utils import preprocess_meeting_important_dates from ietf.meeting.utils import new_doc_for_session, write_doc_for_session from ietf.meeting.utils import get_activity_stats, post_process, create_recording, delete_recording -from ietf.meeting.utils import participants_for_meeting, generate_bluesheet, bluesheet_data, save_bluesheet +from ietf.meeting.utils import generate_bluesheet, bluesheet_data, save_bluesheet from ietf.message.utils import infer_message from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName, CountryName from ietf.utils import markdown @@ -4816,16 +4816,25 @@ def proceedings_attendees(request, num=None): chart_data = None if int(meeting.number) >= 118: - checked_in, attended = participants_for_meeting(meeting) - regs = list(Registration.objects.onsite().filter(meeting__number=num, checkedin=True)) - onsite_count = len(regs) - regs += [ + onsite, remote = meeting.get_attendees() + onsite_count = len(onsite) + remote_count = len(remote) + onsite_pks = frozenset(p.pk for p in onsite) + remote_pks = frozenset(p.pk for p in remote) + + regs = [ + reg + for reg in Registration.objects.onsite() + .filter(meeting__number=num) + .select_related("person") + if reg.person.pk in onsite_pks + ] + [ reg - for reg in Registration.objects.remote().filter(meeting__number=num).select_related('person') - if reg.person.pk in attended and reg.person.pk not in checked_in + for reg in Registration.objects.remote() + .filter(meeting__number=num) + .select_related("person") + if reg.person.pk in remote_pks ] - remote_count = len(regs) - onsite_count - registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name)) country_codes = [r.country_code for r in registrations if r.country_code] diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index a2ab680df6d..46418714880 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -32,7 +32,6 @@ from ietf.person.models import Email, Person from ietf.mailtrigger.utils import gather_address_lists from ietf.meeting.models import Meeting -from ietf.meeting.utils import participants_for_meeting from ietf.utils.pipe import pipe from ietf.utils.mail import send_mail_text, send_mail, get_payload_text from ietf.utils.log import log @@ -743,8 +742,8 @@ def three_of_five_eligible_9389(previous_five, queryset=None): counts = defaultdict(lambda: 0) for meeting in previous_five: - checked_in, attended = participants_for_meeting(meeting) - for id in set(checked_in) | set(attended): + onsite_pks, remote_pks = meeting.nomcom_eligible_participants() + for id in onsite_pks | remote_pks: counts[id] += 1 return queryset.filter(pk__in=[id for id, count in counts.items() if count >= 3]) From eba657a9d04aee815a14d07a6706ffbbdfa0b207 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 30 Jun 2026 15:01:39 -0500 Subject: [PATCH 296/313] fix: Use primary address for ballot email from for non ietf-stream docs (#11119) * fix: Use primary address for ballot email from for non ietf-stream docs * chore: remove unintended whitespace --- ietf/doc/tests_draft.py | 40 ++++++++++++++++++++++++++++++++++++++++ ietf/doc/views_ballot.py | 5 ++++- ietf/nomcom/tests.py | 4 ++-- ietf/person/factories.py | 4 ++-- ietf/person/tests.py | 6 ++++-- 5 files changed, 52 insertions(+), 7 deletions(-) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index 688b933aad3..7f075ee95ca 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -2836,3 +2836,43 @@ def _post_json(self, url, json_to_post): ]: self.assertIn(snippet, response["text"]) + # Non-IETF stream: from address uses balloter.formatted_email(), not role_email("ad"). + # Give the balloter a chair role (in a different RG) with a distinct email to prove + # that chair role address is not used; formatted_email() uses the primary email. + irtf_doc = RgDraftFactory() + non_ad_balloter = PersonFactory(name="Some Irsgmember") + other_rg = GroupFactory(type_id="rg") + chair_role_email = EmailFactory( + person=non_ad_balloter, address="chair-role@example.com" + ) + RoleFactory( + name_id="chair", + group=other_rg, + person=non_ad_balloter, + email=chair_role_email, + ) + response = _post_json( + self, + url, + { + "post_data": { + "discuss": "", + "comment": "cccccc", + "position": "yes", + "balloter": non_ad_balloter.pk, + "docname": irtf_doc.name, + "cc_choices": [], + "additional_cc": "", + } + }, + ) + self.assertTrue(response["success"]) + from_line = next( + line for line in response["text"].split("\n") if line.startswith("From: ") + ) + # formatted_email() uses the primary email, not the chair role email + self.assertIn(non_ad_balloter.email().address, from_line) + self.assertNotIn(chair_role_email.address, from_line) + for snippet in ["cccccc", non_ad_balloter.plain_name(), irtf_doc.name]: + self.assertIn(snippet, response["text"]) + diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py index 29aadfdb9b7..ec593934131 100644 --- a/ietf/doc/views_ballot.py +++ b/ietf/doc/views_ballot.py @@ -425,7 +425,10 @@ def build_position_email_from_dict(pos_dict): pos=pos_name, blocking_name=blocking_name, settings=settings)) - frm = balloter.role_email("ad").formatted_email() + if doc.stream_id == "ietf": + frm = balloter.role_email("ad").formatted_email() + else: + frm = balloter.formatted_email() if doc.stream_id == "irtf": addrs = gather_address_lists('irsg_ballot_saved',doc=doc) diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index 28152ef79b1..b406dd31528 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -1190,8 +1190,8 @@ def setUp(self): today = datetime_today() t_minus_3 = today - datetime.timedelta(days=3) t_minus_4 = today - datetime.timedelta(days=4) - e1 = EmailFactory(address="nominee1@example.org", person=PersonFactory(name="Nominee 1"), origin='test') - e2 = EmailFactory(address="nominee2@example.org", person=PersonFactory(name="Nominee 2"), origin='test') + e1 = EmailFactory(address="nominee1@example.org", person__name="Nominee 1", origin='test', primary=True) + e2 = EmailFactory(address="nominee2@example.org", person__name="Nominee 2", origin='test', primary=True) n = make_nomineeposition(self.nomcom,e1.person,gen,None) np = n.nomineeposition_set.get(position=gen) np.time = t_minus_3 diff --git a/ietf/person/factories.py b/ietf/person/factories.py index 98756f26c8a..655f25994b3 100644 --- a/ietf/person/factories.py +++ b/ietf/person/factories.py @@ -95,7 +95,7 @@ def default_emails(obj, create, extracted, **kwargs): # pylint: disable=no-self- extracted = True if create and extracted: make_email = getattr(EmailFactory, 'create' if create else 'build') - make_email(person=obj, address=obj.user.email) + make_email(person=obj, address=obj.user.email, primary=True, **kwargs) @factory.post_generation def default_photo(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument @@ -151,7 +151,7 @@ class Meta: django_get_or_create = ('address',) address = factory.Sequence(fake_email_address) - person = factory.SubFactory(PersonFactory) + person = factory.SubFactory(PersonFactory, default_emails=False) active = True primary = False diff --git a/ietf/person/tests.py b/ietf/person/tests.py index f55d8b8a347..42c2c1547cb 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -78,7 +78,9 @@ def test_ajax_person_email_json(self): def test_default_email(self): person = PersonFactory() - primary = EmailFactory(person=person, primary=True, active=True) + primary = person.email_set.get() + self.assertEqual(primary.primary, True) + self.assertEqual(primary.active, True) EmailFactory(person=person, primary=False, active=True) EmailFactory(person=person, primary=False, active=False) self.assertTrue(primary.address in person.formatted_email()) @@ -358,7 +360,7 @@ def test_get_extra_primary(self): source = PersonFactory() target = PersonFactory() extra = get_extra_primary(source, target) - self.assertTrue(extra == list(source.email_set.filter(primary=True))) + self.assertEqual(set(extra), set(source.email_set.filter(primary=True))) def test_dedupe_aliases(self): person = PersonFactory() From 9f6f99c5957fb4f5cc9d4ad96a39595bacd7bf6f Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 2 Jul 2026 09:32:24 -0500 Subject: [PATCH 297/313] feat: better arguments for manually updating rfc json (#11131) * feat: better arguments for manually updating rfc json * fix: sort and deduplicate generated integer list --- ietf/sync/tasks.py | 24 ++++++++++++++++++- ietf/sync/tests_tasks.py | 17 ++++++++++++++ ietf/sync/tests_utils.py | 47 +++++++++++++++++++++++++++++++++++- ietf/sync/utils.py | 51 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 2 deletions(-) diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index c61e754ed30..3af5cb8984d 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -31,7 +31,12 @@ create_std_txt_index, rfcindex_is_dirty, mark_rfcindex_as_processed, mark_rfcindex_as_dirty, ) -from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper +from ietf.sync.utils import ( + build_from_file_content, + expand_rfc_number_range_list, + load_rfcs_into_blobdb, + rsync_helper, +) from ietf.utils import log @@ -219,6 +224,23 @@ def update_errata_from_rfceditor_task(): if changed_numbers: update_rfc_json_task.delay(list(changed_numbers)) +@shared_task +def update_rfc_json_by_range_list_task(ranges: str) -> None: + """Regenerate RFC JSON for the RFCs described by a range-list string + + See expand_rfc_number_range_list() for the accepted format (e.g. + "[1,100,1000-1004]"). Invalid input is logged and otherwise ignored. + """ + try: + rfc_numbers = expand_rfc_number_range_list(ranges) + except ValueError as e: + log.log( + f"update_rfc_json_by_range_list_task: ignoring invalid input " + f"'{ranges}': {e}" + ) + return + if rfc_numbers: + update_rfc_json_task(rfc_numbers) @shared_task def update_rfc_json_task(rfc_numbers: list[int]) -> None: diff --git a/ietf/sync/tests_tasks.py b/ietf/sync/tests_tasks.py index edfd080079a..57284b72980 100644 --- a/ietf/sync/tests_tasks.py +++ b/ietf/sync/tests_tasks.py @@ -536,3 +536,20 @@ def test_json_task_not_called_when_no_changes( mock_update.return_value = set() tasks.update_errata_from_rfceditor_task() mock_json_delay.assert_not_called() + + +class UpdateRfcJsonByRangeListTaskTests(TestCase): + @mock.patch("ietf.sync.tasks.update_rfc_json_task") + def test_expands_range_list_and_calls_update(self, mock_update): + tasks.update_rfc_json_by_range_list_task("[1,100,1000-1004]") + mock_update.assert_called_once_with([1, 100, 1000, 1001, 1002, 1003]) + + @mock.patch("ietf.sync.tasks.update_rfc_json_task") + def test_invalid_input_is_ignored(self, mock_update): + tasks.update_rfc_json_by_range_list_task("[5-3]") + mock_update.assert_not_called() + + @mock.patch("ietf.sync.tasks.update_rfc_json_task") + def test_empty_input_does_not_call_update(self, mock_update): + tasks.update_rfc_json_by_range_list_task("[]") + mock_update.assert_not_called() diff --git a/ietf/sync/tests_utils.py b/ietf/sync/tests_utils.py index bb4a859e301..ed1dd4a62aa 100644 --- a/ietf/sync/tests_utils.py +++ b/ietf/sync/tests_utils.py @@ -7,10 +7,55 @@ from ietf import settings from ietf.doc.factories import RfcFactory from ietf.doc.storage_utils import exists_in_storage, retrieve_str -from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper +from ietf.sync.utils import ( + build_from_file_content, + expand_rfc_number_range_list, + load_rfcs_into_blobdb, + rsync_helper, +) from ietf.utils.test_utils import TestCase +class ExpandRfcNumberRangeListTests(TestCase): + def test_expands_bare_numbers_and_ranges(self): + # ranges follow Python range() conventions: right value is excluded + self.assertEqual( + expand_rfc_number_range_list("[1,100,1000-1004]"), + [1, 100, 1000, 1001, 1002, 1003], + ) + + def test_accepts_input_without_brackets_and_whitespace(self): + self.assertEqual( + expand_rfc_number_range_list(" 1 , 3 - 5 , 9 "), + [1, 3, 4, 9], + ) + + def test_overlapping_ranges_are_sorted_and_deduplicated(self): + self.assertEqual( + expand_rfc_number_range_list("[5,1-4,3-6,2]"), + [1, 2, 3, 4, 5], + ) + + def test_empty_input_yields_empty_list(self): + self.assertEqual(expand_rfc_number_range_list("[]"), []) + self.assertEqual(expand_rfc_number_range_list(""), []) + + def test_rejects_invalid_input(self): + for bad in [ + "[0]", # zero is not positive + "[-5]", # negatives are not accepted + "[abc]", # non-numeric + "[5-3]", # reversed range + "[5-5]", # empty range (start not less than end) + "[1-]", # missing end + "[-1]", # missing start + "[1-2-3]", # malformed range + "[1.5]", # not an integer + ]: + with self.assertRaises(ValueError, msg=f"expected ValueError for {bad!r}"): + expand_rfc_number_range_list(bad) + + class RsyncHelperTests(TestCase): def test_rsync_helper(self): with ( diff --git a/ietf/sync/utils.py b/ietf/sync/utils.py index b3bdd8d2068..b75d80e76bc 100644 --- a/ietf/sync/utils.py +++ b/ietf/sync/utils.py @@ -14,6 +14,57 @@ def rsync_helper(subprocess_arg_array: list[str]): subprocess.run(["/usr/bin/rsync"]+subprocess_arg_array) + +def _parse_positive_int(value: str) -> int: + """Parse a string as a positive (>= 1) integer, raising ValueError otherwise""" + token = value.strip() + # str.isdigit() rejects signs, whitespace, and non-digits, so anything that + # passes is a non-negative integer literal; guard against zero separately. + if not token.isdigit(): + raise ValueError(f"'{value}' is not a positive integer") + number = int(token) + if number < 1: + raise ValueError(f"'{value}' is not a positive integer") + return number + + +def expand_rfc_number_range_list(ranges: str) -> list[int]: + """Expand a range-list string into a list of RFC numbers + + The string is a comma-separated list of tokens, optionally surrounded by a + pair of square brackets. Each token is either a bare positive integer or a + pair of positive integers separated by a hyphen. A hyphenated pair is + expanded following the convention of Python's range(): the left value is + included and the right value is excluded. For example, "[1,100,1000-1004]" + expands to [1, 100, 1000, 1001, 1002, 1003]. + + The returned list is sorted and deduplicated, so overlapping ranges do not + produce repeated numbers. + + Raises ValueError if the input contains anything other than positive + integers and well-formed (non-reversed) ranges. + """ + numbers: set[int] = set() + stripped = ranges.strip() + if stripped.startswith("[") and stripped.endswith("]"): + stripped = stripped[1:-1] + for raw_token in stripped.split(","): + token = raw_token.strip() + if not token: + continue + if "-" in token: + start_str, _, end_str = token.partition("-") + start = _parse_positive_int(start_str) + end = _parse_positive_int(end_str) + if start >= end: + raise ValueError( + f"'{token}' is not a valid range (start must be less than end)" + ) + numbers.update(range(start, end)) + else: + numbers.add(_parse_positive_int(token)) + return sorted(numbers) + def build_from_file_content(rfc_numbers: list[int]) -> str: types_to_sync = settings.RFC_FILE_TYPES + ("json",) lines = [] From 629061210d9350be37500f98dab2c387d87618da Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Thu, 2 Jul 2026 16:34:50 +0200 Subject: [PATCH 298/313] Ignore the timestamp elements, just check that the data is correct (#11124) --- ietf/stats/tests.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index d8e87417026..57e3b76b39b 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -1,6 +1,5 @@ # Copyright The IETF Trust 2016-2026, All Rights Reserved -import calendar import csv import datetime import io @@ -26,7 +25,6 @@ from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory from ietf.submit.factories import SubmissionFactory -from ietf.utils.timezone import date_today class StatisticsTests(TestCase): @@ -158,16 +156,19 @@ def test_review_stats(self): self.assertTrue(q('.review-stats td:contains("1")')) # check stacked chart - expected_date = date_today().replace(day=1) - expected_js_timestamp = calendar.timegm(expected_date.timetuple()) * 1000 url = urlreverse(ietf.stats.views.review_stats, kwargs={ "stats_type": "time" }) url += "?team={}".format(review_req.team.acronym) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertEqual(json.loads(r.context['data']), [ - {"label": "in time", "color": "#3d22b3", "data": [[expected_js_timestamp, 0]]}, - {"label": "late", "color": "#b42222", "data": [[expected_js_timestamp, 0]]} - ]) + stacked_data = json.loads(r.context['data']) + # Ignore the timestamp elements, just check that the data is correct + self.assertEqual(len(stacked_data), 2) + self.assertEqual(stacked_data[0]['label'], 'in time') + self.assertEqual(stacked_data[0]['color'], '#3d22b3') + self.assertEqual(stacked_data[0]['data'], [[stacked_data[0]['data'][0][0], 0]]) + self.assertEqual(stacked_data[1]['label'], 'late') + self.assertEqual(stacked_data[1]['color'], '#b42222') + self.assertEqual(stacked_data[1]['data'], [[stacked_data[0]['data'][0][0], 0]]) q = PyQuery(r.content) self.assertTrue(q('#stats-time-graph')) @@ -177,7 +178,11 @@ def test_review_stats(self): url += "&completion=not_completed" r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertEqual(json.loads(r.context['data']), [{"color": "#3d22b3", "data": [[expected_js_timestamp, 0]]}]) + non_stacked_data = json.loads(r.context['data']) + # Ignore the timestamp elements, just check that the data is correct + self.assertEqual(len(non_stacked_data), 1) + self.assertEqual(non_stacked_data[0]['color'], '#3d22b3') + self.assertEqual(non_stacked_data[0]['data'], [[non_stacked_data[0]['data'][0][0], 0]]) q = PyQuery(r.content) self.assertTrue(q('#stats-time-graph')) From b11e7041c057c2b475729c74241f425d2813f777 Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Thu, 2 Jul 2026 16:51:23 +0200 Subject: [PATCH 299/313] feat: Add IESG decisions to the IESG group menu (#11050) * Add IESG decisions to the IESG group menu * Minor fix to template + copyright dates --- ietf/group/utils.py | 1 + ietf/iesg/views.py | 34 +++++++++++++++-------- ietf/templates/base/menu.html | 6 ++++ ietf/templates/iesg/review_decisions.html | 8 +++--- ietf/templates/iesg/working_groups.html | 8 +++--- 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/ietf/group/utils.py b/ietf/group/utils.py index 6777ed1933b..b1675de659e 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -241,6 +241,7 @@ def construct_group_menu_context(request, group, selected, group_type, others): entries.append(("Meetings", urlreverse("ietf.group.views.meetings", kwargs=kwargs))) if group.acronym in ["iesg"]: entries.append(("Working Groups", urlreverse("ietf.iesg.views.working_groups"))) + entries.append(("Decisions", urlreverse("ietf.iesg.views.review_decisions"))) if group.acronym in ["iab", "iesg"]: entries.append(("Statements", urlreverse("ietf.group.views.statements", kwargs=kwargs))) entries.append(("Appeals", urlreverse("ietf.group.views.appeals", kwargs=kwargs))) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index f03afb9fc1a..af04ad06fd5 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2020, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- # # Portion Copyright (C) 2008-2009 Nokia Corporation and/or its subsidiary(-ies). @@ -59,6 +59,7 @@ from ietf.doc.models import Document, State, LastCallDocEvent, ConsensusDocEvent, DocEvent, IESG_BALLOT_ACTIVE_STATES from ietf.doc.utils import update_telechat, augment_events_with_revision from ietf.group.models import GroupMilestone, Role +from ietf.group.utils import construct_group_menu_context, get_group_or_404 from ietf.iesg.agenda import agenda_data, agenda_sections, fill_in_agenda_docs, get_agenda_date from ietf.iesg.models import TelechatDate, TelechatAgendaContent from ietf.iesg.utils import get_wg_dashboard_info, telechat_page_count @@ -89,13 +90,15 @@ def review_decisions(request, year=None): #doc_levels = ["exp", "inf"] timeframe = "%s" % year if year else "the past 6 months" + group_type = None + group = get_group_or_404("iesg", group_type) return render(request, 'iesg/review_decisions.html', - dict(events=events, - years=years, - year=year, - timeframe=timeframe), - ) + construct_group_menu_context(request, group, "decisions", group_type, { + "events": events, + "years": years, + "year": year, + "timeframe": timeframe})) def agenda_json(request, date=None): data = agenda_data(date) @@ -632,8 +635,17 @@ def working_groups(request): area_summary, area_totals, ad_summary, noad_summary, ad_totals, noad_totals, totals, wg_summary = get_wg_dashboard_info() - return render( - request, - "iesg/working_groups.html", - dict(area_summary=area_summary, area_totals=area_totals, ad_summary=ad_summary, noad_summary=noad_summary, ad_totals=ad_totals, noad_totals=noad_totals, totals=totals, wg_summary=wg_summary), - ) + group_type = None + group = get_group_or_404("iesg", group_type) + + return render(request, 'iesg/working_groups.html', + construct_group_menu_context(request, group, "working groups", group_type, { + "area_summary": area_summary, + "area_totals": area_totals, + "ad_summary": ad_summary, + "noad_summary": noad_summary, + "ad_totals": ad_totals, + "noad_totals": noad_totals, + "totals": totals, + "wg_summary": wg_summary, + })) diff --git a/ietf/templates/base/menu.html b/ietf/templates/base/menu.html index b6b744ec21e..905c337850e 100644 --- a/ietf/templates/base/menu.html +++ b/ietf/templates/base/menu.html @@ -132,6 +132,12 @@ IESG dashboard
    • +
  • + + IESG decisions + +
  • diff --git a/ietf/templates/iesg/review_decisions.html b/ietf/templates/iesg/review_decisions.html index 68a9d7fb953..2f69c1809b4 100644 --- a/ietf/templates/iesg/review_decisions.html +++ b/ietf/templates/iesg/review_decisions.html @@ -1,8 +1,8 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{% extends "group/group_base.html" %} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin %} -{% block title %}IESG review decisions in {{ timeframe }}{% endblock %} -{% block content %} +{% block group_subtitle %}IESG review decisions in {{ timeframe }}{% endblock %} +{% block group_content %} {% origin %}

    IESG review decisions in {{ timeframe }}

    diff --git a/ietf/templates/iesg/working_groups.html b/ietf/templates/iesg/working_groups.html index b799636857f..1bde1362c50 100644 --- a/ietf/templates/iesg/working_groups.html +++ b/ietf/templates/iesg/working_groups.html @@ -1,11 +1,11 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "group/group_base.html" %} +{# Copyright The IETF Trust 2025-2026, All Rights Reserved #} {% load origin static %} {% block pagehead %} {% endblock %} -{% block title %}IESG view of working groups{% endblock %} -{% block content %} +{% block group_subtitle %}IESG view of working groups{% endblock %} +{% block group_content %} {% origin %}

    IESG view of working groups

    Area Size and Load

    From e7360821d7da140b8348db42d11a8957e0cef195 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 6 Jul 2026 15:58:50 -0300 Subject: [PATCH 300/313] fix: 405 instead of 501 on POST to v1 detail API (#11162) * fix: 405 instead of 501 on detail API POST * fix: import custom ModelResource * fix: use custom MakeResources in mgmt cmd --- ietf/api/__init__.py | 4 +++ ietf/api/management/commands/makeresources.py | 6 ++-- ietf/api/tests.py | 31 +++++++++++++++++++ ietf/review/resources.py | 4 +-- ietf/stats/resources.py | 4 +-- 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index 00733717c4f..f4bfe8330ee 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -10,6 +10,7 @@ from django.apps import apps as django_apps from django.core.exceptions import ObjectDoesNotExist +from django.http import HttpResponseNotAllowed from django.utils.module_loading import autodiscover_modules @@ -50,6 +51,9 @@ def autodiscover(): class ModelResource(tastypie.resources.ModelResource): + def post_detail(self, request, **kwargs): + return HttpResponseNotAllowed(["GET"]) + def generate_cache_key(self, *args, **kwargs): """ Creates a unique-enough cache key. diff --git a/ietf/api/management/commands/makeresources.py b/ietf/api/management/commands/makeresources.py index 889b2cdfb51..8e2c6f4b5cf 100644 --- a/ietf/api/management/commands/makeresources.py +++ b/ietf/api/management/commands/makeresources.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2014-2020, All Rights Reserved +# Copyright The IETF Trust 2014-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,7 +15,7 @@ from django.template import Template, Context from django.utils import timezone -from tastypie.resources import ModelResource +from ietf.api import ModelResource resource_head_template = """# Copyright The IETF Trust {{date}}, All Rights Reserved @@ -23,7 +23,7 @@ # Generated by the makeresources management command {{date}} -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 887969cec15..87f7d684d32 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -12,6 +12,7 @@ from importlib import import_module from pathlib import Path from random import randrange +from urllib.parse import urljoin from django.apps import apps from django.conf import settings @@ -1564,6 +1565,36 @@ def test_serializer_to_etree_handles_xml_invalid_control_chars(self): f"containing control character U+{ord(ch):04X}" ) + def test_post_detail_is_not_allowed(self): + """POST to a detail route returns 405 + + Added because default TastyPie behavior is a 500 due to a NotImplemented + exception. + """ + r = self.client.get("/api/v1", headers={"Accept": "application/json"}) + self.assertValidJSONResponse(r) + resource_list = r.json() + for name in self.apps: + r = self.client.get( + resource_list[name]["list_endpoint"], + headers={"Accept": "application/json"}, + ) + self.assertValidJSONResponse(r) + app_resources = r.json() + model_list = apps.get_app_config(name).get_models() + for model in model_list: + model_name = model._meta.model_name + assert model_name + detail_url = urljoin( + app_resources[model_name]["list_endpoint"], "some-id/" + ) + r = self.client.post(detail_url) + self.assertEqual( + r.status_code, + 405, + f"POST to {name}.{model_name} detail should return 405 status", + ) + class RfcdiffSupportTests(TestCase): diff --git a/ietf/review/resources.py b/ietf/review/resources.py index f79d6dfc221..546ab50a5ca 100644 --- a/ietf/review/resources.py +++ b/ietf/review/resources.py @@ -1,9 +1,9 @@ -# Copyright The IETF Trust 2016-2019, All Rights Reserved +# Copyright The IETF Trust 2016-2026, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2016-06-14 04:21 PDT -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache diff --git a/ietf/stats/resources.py b/ietf/stats/resources.py index 59722c505ef..21e2c171acc 100644 --- a/ietf/stats/resources.py +++ b/ietf/stats/resources.py @@ -1,9 +1,9 @@ -# Copyright The IETF Trust 2017-2019, All Rights Reserved +# Copyright The IETF Trust 2017-2026, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2017-02-15 10:10 PST -from tastypie.resources import ModelResource +from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore from tastypie.cache import SimpleCache From 1fe544a01a11feb9b13d9df5594054f1cba5b534 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 7 Jul 2026 13:20:50 -0300 Subject: [PATCH 301/313] feat: generate RFC json at publication time (#11143) * feat: generate RFC json at publication time Deprecates uploading json files and ignores the file, instead generating its own. * refactor: partial() instead of lambda in on_commit Using partial() is safer as general practice because it binds arguments at call-time. * test: json upload ignored, creation task queued --- ietf/api/serializers_rpc.py | 2 +- ietf/api/tests_views_rpc.py | 29 +++++++++++++++++++++-------- ietf/api/views_rpc.py | 14 +++++++++++--- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 0e45ee3b391..ca605fe6441 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -805,7 +805,7 @@ class RfcFileSerializer(serializers.Serializer): # works. allowed_extensions = ( ".html", - ".json", + ".json", # deprecated - accepted but ignored, datatracker generates its own ".notprepped.xml", ".pdf", ".txt", diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 72e0d197857..5748a7caa60 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -224,6 +224,7 @@ def test_notify_rfc_published(self, mock_task_delay): self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.update_rfc_json_task") @mock.patch("ietf.api.views_rpc.rebuild_reference_relations_task") @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task") @@ -232,6 +233,7 @@ def test_upload_rfc_files( mock_trigger_red_task, mock_update_searchindex_task, mock_rebuild_relations, + mock_update_rfc_json, ): def _valid_post_data(): """Generate a valid post data dict @@ -292,7 +294,7 @@ def _valid_post_data(): ContentFile(b"", "myfile.txt"), ContentFile(b"", "myfile.html"), ContentFile(b"", "myfile.pdf"), - ContentFile(b"", "myfile.json"), + ContentFile(b"", "myfile.json"), # deprecated! ContentFile(b"", "myfile.notprepped.xml"), ] }, @@ -346,18 +348,19 @@ def _valid_post_data(): # valid post mock_trigger_red_task.delay.reset_mock() - r = self.client.post( - url, - _valid_post_data(), - format="multipart", - headers={"X-Api-Key": "valid-token"}, - ) + with self.captureOnCommitCallbacks(execute=True): + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) self.assertEqual(r.status_code, 200) self.assertEqual( mock_update_searchindex_task.delay.call_args, mock.call(rfc.rfc_number), ) - for extension in ["xml", "txt", "html", "pdf", "json"]: + for extension in ["xml", "txt", "html", "pdf"]: # no "json"! filename = f"{rfc.name}.{extension}" self.assertEqual( (rfc_path / filename).read_text(), @@ -389,6 +392,12 @@ def _valid_post_data(): b"This is .notprepped.xml", ".notprepped.xml blob should contain the expected content", ) + # special case for json (deprecated and should be ignored) + self.assertFalse((rfc_path / f"{rfc.name}.json").exists()) + self.assertFalse( + Blob.objects.filter(bucket="rfc", name=f"json/{rfc.name}.json").exists() + ) + # Confirm that the red precomputer was triggered correctly self.assertTrue(mock_trigger_red_task.delay.called) _, mock_kwargs = mock_trigger_red_task.delay.call_args @@ -404,6 +413,10 @@ def _valid_post_data(): _, mock_kwargs = mock_rebuild_relations.delay.call_args self.assertIn("doc_names", mock_kwargs) self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) + # Confirm rfc JSON creation was triggered correctly + self.assertTrue(mock_update_rfc_json.delay.called) + mock_args, _ = mock_update_rfc_json.delay.call_args + self.assertEqual(mock_args[0], [rfc.rfc_number]) # re-post with replace = False should now fail mock_update_searchindex_task.reset_mock() diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 0c9e98e2dc0..21d59e3f14c 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2023-2026, All Rights Reserved import os import shutil +from functools import partial from pathlib import Path from tempfile import TemporaryDirectory @@ -463,7 +464,9 @@ def post(self, request): {d.rfc_number for d in rfc.related_that_doc(("updates", "obs"))} ) if related_numbers: - transaction.on_commit(lambda: update_rfc_json_task.delay(related_numbers)) + # Only update json for related rfcs. We can't generate json for _this_ rfc + # until we receive the files and know what formats we have. + transaction.on_commit(partial(update_rfc_json_task.delay, related_numbers)) return Response(NotificationAckSerializer().data) @@ -523,6 +526,10 @@ def post(self, request): for upfile in uploaded_files: uploaded_filename = Path(upfile.name) # name supplied by request uploaded_ext = "".join(uploaded_filename.suffixes) + # Ignore json files, which are deprecated. Remove this when purple no + # longer tries to send them. + if uploaded_ext == ".json": + continue tempfile_path = tmpfile_stem.with_suffix(uploaded_ext) with tempfile_path.open("wb") as dest: for chunk in upfile.chunks(): @@ -565,9 +572,10 @@ def post(self, request): trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating)) # Trigger search index update update_rfc_searchindex_task.delay(rfc.rfc_number) - # Trigger reference relation srebuild + # Trigger reference relations rebuild rebuild_reference_relations_task.delay(doc_names=[rfc.name]) - + # Build rfc json (json for related rfcs was updated in RfcPubNotificationView) + transaction.on_commit(partial(update_rfc_json_task.delay, [rfc.rfc_number])) return Response(NotificationAckSerializer().data) From bd24d4da64d68f81937202d80b04422c81aaf412 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 8 Jul 2026 04:36:55 +1200 Subject: [PATCH 302/313] feat: Expose idnits v3 for everyone (#11166) --- ietf/templates/doc/document_draft.html | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index 9b967c013bf..57194144989 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -666,17 +666,14 @@     Nits - {% if user|has_role:"Area Director" %} - {# IDNITS3 is an experimental service, so only show it to Area Directors #} - Nits-v3 (Experimental) + Nits v3 - {% endif %} Date: Tue, 7 Jul 2026 16:34:14 -0300 Subject: [PATCH 303/313] chore: add keywords to red-content search preset (#11159) --- ietf/utils/searchindex.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py index 4e1ee27895b..3c6e1b8f26f 100644 --- a/ietf/utils/searchindex.py +++ b/ietf/utils/searchindex.py @@ -383,9 +383,9 @@ def update_or_create_rfc_entries( }, "red-content": { "collection": "docs", - "infix": "off,always,off,off", - "query_by": "rfc,filename,authors,content", - "query_by_weights": "127,50,5,1" + "infix": "off,always,off,off,off", + "query_by": "rfc,filename,keywords,authors,content", + "query_by_weights": "127,50,20,5,1" }, } From 6e3549a0b7707ce1f3cfe70e21aafdd4e076a8d4 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 8 Jul 2026 14:22:26 -0300 Subject: [PATCH 304/313] fix: prevent overlap in html page breaks (#11172) --- ietf/static/css/document_html.scss | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ietf/static/css/document_html.scss b/ietf/static/css/document_html.scss index 47ef8d64b47..5a4282195f9 100644 --- a/ietf/static/css/document_html.scss +++ b/ietf/static/css/document_html.scss @@ -205,10 +205,6 @@ display: inline; } - .newpage { - margin-top: -1.25em; - } - } tbody.meta tr { From bd600888d0da085af2eac35d4ad780c33af0d813 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 Jul 2026 17:57:37 -0300 Subject: [PATCH 305/313] test: fix test_submit_bad_author_email() (#11186) Original test accidentally created a second primary email by trying to modify the PK of an existing Email. That led to a stochastic apparently data-dependent failure that depended on which address was returned by `first()`. --- ietf/submit/tests.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index ad361d31b26..abe23c1a643 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -40,7 +40,7 @@ from ietf.meeting.factories import MeetingFactory from ietf.name.models import DraftSubmissionStateName, FormalLanguageName from ietf.person.models import Person -from ietf.person.factories import UserFactory, PersonFactory +from ietf.person.factories import UserFactory, PersonFactory, EmailFactory from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactory from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm from ietf.submit.models import Submission, Preapproval, SubmissionExtResource @@ -1872,10 +1872,8 @@ def test_submit_bad_author_email(self): name = "draft-authorname-testing-bademail" rev = "00" - author = PersonFactory() - email = author.email_set.first() - email.address = '@bad.email' - email.save() + author = PersonFactory(default_emails=False) + EmailFactory(person=author, primary=True, address="@bad.email") status_url, _ = self.do_submission(name=name, rev=rev, author=author, formats=('xml',)) r = self.client.get(status_url) From 8d7139b0538605033a8577174ee5aefd585ad145 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 Jul 2026 18:24:35 -0300 Subject: [PATCH 306/313] fix: use RfcAuthor in bibtex (w/titlepage_name) (#10947) * test: create RfcAuthors in RfcFactory * test: update test to check authors * fix: use RfcAuthor in bibtex (w/titlepage_name) * fix: trailing / on bibtex url * fix: update URL in bibtex warning for legacy RFCs * test: fix failing tests Remove a case that tests fallback from RfcAuthor to DocumentAuthor since we now always have RfcAuthors. Remove RfcAuthor records when deleting Persons they refer to, since we don't allow cascade on this model. --- ietf/doc/factories.py | 7 +++++++ ietf/doc/templatetags/ietf_filters.py | 2 +- ietf/doc/tests.py | 23 +++++++++++++++-------- ietf/nomcom/tests.py | 18 ++++-------------- ietf/templates/doc/document_bibtex.bib | 6 +++--- 5 files changed, 30 insertions(+), 26 deletions(-) diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index 1a178c6f31d..0cf83e4525f 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -128,6 +128,13 @@ def states(obj, create, extracted, **kwargs): else: obj.set_state(State.objects.get(type_id='rfc',slug='published')) + @factory.post_generation + def authors(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument + # Override base class, creating RfcAuthor instead of DocumentAuthor + if create and extracted: + for person in extracted: + RfcAuthorFactory(document=obj, person=person) + class IndividualDraftFactory(BaseDocumentFactory): diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index ae5df641c29..e72cc04ff34 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -138,7 +138,7 @@ def prettystdname(string, space=" "): @register.filter def rfceditor_info_url(rfcnum : str): """Link to the RFC editor info page for an RFC""" - return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}') + return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}/') def doc_name(name): diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index ff4461d4666..86731000730 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -36,9 +36,9 @@ import debug # pyflakes:ignore -from ietf.doc.models import ( Document, DocRelationshipName, RelatedDocument, State, - DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType, - EditedAuthorsDocEvent, StateType) +from ietf.doc.models import (Document, DocRelationshipName, RelatedDocument, State, + DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType, + EditedAuthorsDocEvent, StateType, RfcAuthor) from ietf.doc.factories import (DocumentFactory, DocEventFactory, CharterFactory, ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, @@ -73,7 +73,7 @@ from ietf.utils.mail import get_payload_text, outbox, empty_outbox from ietf.utils.test_utils import login_testing_unauthorized, unicontent from ietf.utils.test_utils import TestCase -from ietf.utils.text import normalize_text +from ietf.utils.text import normalize_text, texescape from ietf.utils.timezone import date_today, datetime_today, DEADLINE_TZINFO, RPC_TZINFO from ietf.doc.utils_search import AD_WORKLOAD @@ -2133,9 +2133,11 @@ def test_document_bibtex(self): doc = factory() url = urlreverse("ietf.doc.views_doc.document_bibtex", kwargs=dict(name=doc.name)) r = self.client.get(url) - self.assertEqual(r.status_code, 404) + self.assertEqual(r.status_code, 404) + authors = PersonFactory.create_batch(2) rfc = WgRfcFactory.create( - time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo(settings.TIME_ZONE)) + time=datetime.datetime(2010, 10, 10, tzinfo=ZoneInfo(settings.TIME_ZONE)), + authors=authors, ) num = rfc.rfc_number DocEventFactory.create( @@ -2152,7 +2154,12 @@ def test_document_bibtex(self): self.assertEqual(entry["doi"], "10.17487/RFC%s" % num) self.assertEqual(entry["year"], "2010") self.assertEqual(entry["month"].lower()[0:3], "oct") - self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}") + self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}/") + escaped_author_names = [ + texescape(ra.titlepage_name) + for ra in RfcAuthor.objects.filter(document=rfc) + ] + self.assertEqual(entry["author"], " and ".join(escaped_author_names)) # self.assertNotIn("day", entry) @@ -2188,7 +2195,7 @@ def test_document_bibtex(self): self.assertEqual(entry["year"], "1990") self.assertEqual(entry["month"].lower()[0:3], "apr") self.assertEqual(entry["day"], "1") - self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}") + self.assertEqual(entry["url"], f"https://www.rfc-editor.ietf.org/info/rfc{num}/") draft = IndividualDraftFactory.create() docname = "%s-%s" % (draft.name, draft.rev) diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index b406dd31528..ac86d7cf4b4 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -34,6 +34,7 @@ RfcAuthorFactory, WgDraftFactory, WgRfcFactory, ) +from ietf.doc.models import RfcAuthor from ietf.group.factories import GroupFactory, GroupHistoryFactory, RoleFactory, RoleHistoryFactory from ietf.group.models import Group, Role from ietf.meeting.factories import MeetingFactory, AttendedFactory, RegistrationFactory @@ -2535,19 +2536,6 @@ def test_get_qualified_author_queryset(self): people, # does not include extra_person! ) - # Now add an RfcAuthor for only one of the two authors to the RFC. This should - # remove the other author from the eligibility list because the DocumentAuthor - # records are no longer used. - RfcAuthorFactory( - document=rfc, - person=people[0], - titlepage_name="P. Zero", - ) - self.assertCountEqual( - get_qualified_author_queryset(base_qs, now - 5 * one_year, now), - [people[0]], - ) - class rfc8713EligibilityTests(TestCase): @@ -2875,7 +2863,9 @@ def test_elig_by_author(self): self.assertFalse(is_eligible(person,nomcom)) self.assertEqual(set(list_eligible(nomcom=nomcom)),set(eligible)) - Person.objects.filter(pk__in=[p.pk for p in eligible.union(ineligible)]).delete() + people_pks_to_delete = [p.pk for p in eligible.union(ineligible)] + RfcAuthor.objects.filter(person__pk__in=people_pks_to_delete).delete() + Person.objects.filter(pk__in=people_pks_to_delete).delete() class rfc9389EligibilityTests(TestCase): diff --git a/ietf/templates/doc/document_bibtex.bib b/ietf/templates/doc/document_bibtex.bib index 5e52ec3c58d..71b7498351a 100644 --- a/ietf/templates/doc/document_bibtex.bib +++ b/ietf/templates/doc/document_bibtex.bib @@ -6,8 +6,8 @@ {% if doc.type_id == "rfc" %} {% if doc.stream|slugify == "legacy" %} % Datatracker information for RFCs on the Legacy Stream is unfortunately often -% incorrect. Please correct the bibtex below based on the information in the -% actual RFC at https://rfc-editor.org/rfc/rfc{{ doc.rfc_number }}.txt +% incorrect. Please correct the bibtex below based on the information available +% at {{ doc.rfc_number|rfceditor_info_url }} {% endif %} @misc{% templatetag openbrace %}rfc{{ doc.rfc_number }}, series = {Request for Comments}, @@ -25,7 +25,7 @@ @techreport{ publisher = {% templatetag openbrace %}Internet Engineering Task Force{% templatetag closebrace %}, note = {% templatetag openbrace %}Work in Progress{% templatetag closebrace %}, url = {% templatetag openbrace %}{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name=doc.name rev=doc.rev %}{% templatetag closebrace %},{% endif %} - author = {% templatetag openbrace %}{% for author in doc.documentauthor_set.all %}{{ author.person.name|texescape}}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %}, + author = {% templatetag openbrace %}{% for author in doc.author_persons_or_names %}{% firstof author.titlepage_name|texescape author.person.name|texescape %}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %}, title = {% templatetag openbrace %}{% templatetag openbrace %}{{doc.title|texescape}}{% templatetag closebrace %}{% templatetag closebrace %}, pagetotal = {{ doc.pages }}, year = {{ doc.pub_date.year }}, From 8ee78c401ca68c6141924700e48f21b6ddde38c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:51:02 -0300 Subject: [PATCH 307/313] chore(deps): bump codecov/codecov-action from 6 to 7 (#11000) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 6 to 7. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v6...v7) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/reusable-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml index 10d04c8265e..852e2d6fedd 100644 --- a/.github/workflows/reusable-tests.yml +++ b/.github/workflows/reusable-tests.yml @@ -75,7 +75,7 @@ jobs: path: geckodriver.log - name: Upload Coverage Results to Codecov - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7 with: disable_search: true files: coverage.xml From ce86eae4f608110c83869b5e12d9d98cd290d7b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:56:32 -0300 Subject: [PATCH 308/313] chore(deps): bump actions/dependency-review-action from 4 to 5 (#10842) Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 4 to 5. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dependency-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 7393e83b829..5bb26e4f6fa 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -17,6 +17,6 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@v7 - name: 'Dependency Review' - uses: actions/dependency-review-action@v4 + uses: actions/dependency-review-action@v5 with: vulnerability-check: false From cf139acf19ec6257033dfbfe1f2b7f91dd9b2015 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:58:08 -0300 Subject: [PATCH 309/313] chore(deps): update pydyf requirement from >=0.11.0 to >=0.12.1 (#10762) Updates the requirements on [pydyf](https://github.com/CourtBouillon/pydyf) to permit the latest version. - [Release notes](https://github.com/CourtBouillon/pydyf/releases) - [Changelog](https://github.com/CourtBouillon/pydyf/blob/main/docs/changelog.rst) - [Commits](https://github.com/CourtBouillon/pydyf/compare/v0.11.0...v0.12.1) --- updated-dependencies: - dependency-name: pydyf dependency-version: 0.12.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 31e8ea69d1e..9f3ddf53b88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -64,7 +64,7 @@ opentelemetry-exporter-otlp-proto-http>=1.38.0 pillow>=11.3.0 psycopg2>=2.9.10 pyang>=2.6.1 -pydyf>=0.11.0 +pydyf>=0.12.1 pyflakes>=3.4.0 pyopenssl>=25.1.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency pyquery>=2.0.1 From 0dcb849b372581022e630b98a344aca9c55473a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:58:53 -0300 Subject: [PATCH 310/313] chore(deps): update lxml requirement from >=6.0.0 to >=6.1.0 (#10760) Updates the requirements on [lxml](https://github.com/lxml/lxml) to permit the latest version. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-6.0.0...lxml-6.1.0) --- updated-dependencies: - dependency-name: lxml dependency-version: 6.1.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9f3ddf53b88..226c960b196 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,7 +48,7 @@ jsonfield>=3.2.0 # deprecated - need to replace with Django's JSONField jsonschema[format]>=4.25.0 jwcrypto>=1.5.6 # for signed notifications - this is aspirational, and is not really used. logging_tree>=1.10 # Used only by the showloggers management command -lxml>=6.0.0 +lxml>=6.1.0 markdown>=3.8.0 types-markdown>=3.8.0 mock>=5.2.0 # should replace with unittest.mock and remove dependency From 133a406da263e5fd73239368dfafcfc1d15adb2a Mon Sep 17 00:00:00 2001 From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:10:21 +0000 Subject: [PATCH 311/313] ci: update base image target version to 20260711T0159 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 72bdb7a7ae1..3738d2ffa8b 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260605T2314 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260711T0159 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 7ca47fd1979..6061a016b79 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260605T2314 +20260711T0159 From b88ff91551dd763ea0a45fb2c2ed25f87979f3ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:40:42 -0300 Subject: [PATCH 312/313] chore(deps): update types-zxcvbn requirement (#10701) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 226c960b196..485e4eb4a3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -92,4 +92,4 @@ weasyprint>=66.0 xml2rfc>=3.30.0 xym>=0.6,<0.10.0 zxcvbn>=4.5.0 -types-zxcvbn~=4.5.0.20250223 # match zxcvbn version +types-zxcvbn~=4.5.0.20260518 # match zxcvbn version From b30026de6b42b0b54b3310e9f8dc2dc448ee7cbb Mon Sep 17 00:00:00 2001 From: jennifer-richards <19472766+jennifer-richards@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:54:04 +0000 Subject: [PATCH 313/313] ci: update base image target version to 20260711T0341 --- dev/build/Dockerfile | 2 +- dev/build/TARGET_BASE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 3738d2ffa8b..aa1ab4a55cf 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260711T0159 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260711T0341 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 6061a016b79..2fe337efb27 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260711T0159 +20260711T0341