From 9b834176ee2b6f158a20f937a1c19f69d1f1f718 Mon Sep 17 00:00:00 2001 From: Eric Vyncke Date: Thu, 13 Aug 2026 05:28:56 +0200 Subject: [PATCH 1/2] fix: Use Django Collate() when listing all ADs (#11234) --- ietf/doc/forms.py | 12 ++++++++++-- ietf/doc/views_charter.py | 11 +++++++++-- ietf/doc/views_conflict_review.py | 11 +++++++++-- ietf/doc/views_draft.py | 4 +++- ietf/doc/views_status_change.py | 11 +++++++++-- ietf/settings.py | 8 ++++++++ 6 files changed, 48 insertions(+), 9 deletions(-) diff --git a/ietf/doc/forms.py b/ietf/doc/forms.py index 768d6f96af2..82076d31733 100644 --- a/ietf/doc/forms.py +++ b/ietf/doc/forms.py @@ -5,8 +5,10 @@ import datetime import debug #pyflakes:ignore from django import forms +from django.conf import settings from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.core.validators import validate_email +from django.db.models.functions import Collate from ietf.doc.fields import SearchableDocumentField, SearchableDocumentsField from ietf.doc.models import RelatedDocument, DocExtResource, State @@ -63,8 +65,14 @@ class DocAuthorChangeBasisForm(forms.Form): help_text='What is the source or reasoning for the changes to the author list?') class AdForm(forms.Form): - ad = forms.ModelChoiceField(Person.objects.filter(role__name="ad", role__group__state="active", role__group__type='area').order_by('name'), - label="Shepherding AD", empty_label="(None)", required=True) + ad = forms.ModelChoiceField( + Person.objects.filter( + role__name="ad", role__group__state="active", role__group__type="area" + ).order_by(Collate("name", settings.PREFERRED_COLLATION)), + label="Shepherding AD", + empty_label="(None)", + required=True, + ) def __init__(self, *args, **kwargs): super(self.__class__, self).__init__(*args, **kwargs) diff --git a/ietf/doc/views_charter.py b/ietf/doc/views_charter.py index e899f592271..b2f400d6d6f 100644 --- a/ietf/doc/views_charter.py +++ b/ietf/doc/views_charter.py @@ -20,6 +20,7 @@ from django.utils import timezone from django.utils.encoding import force_str from django.utils.html import escape +from django.db.models.functions import Collate import debug # pyflakes:ignore @@ -305,8 +306,14 @@ def change_title(request, name, option=None): )) class AdForm(forms.Form): - ad = forms.ModelChoiceField(Person.objects.filter(role__name="ad", role__group__state="active", role__group__type="area").order_by('name'), - label="Responsible AD", empty_label="(None)", required=True) + ad = forms.ModelChoiceField( + Person.objects.filter( + role__name="ad", role__group__state="active", role__group__type="area" + ).order_by(Collate("name", settings.PREFERRED_COLLATION)), + label="Responsible AD", + empty_label="(None)", + required=True, + ) def __init__(self, *args, **kwargs): super(self.__class__, self).__init__(*args, **kwargs) diff --git a/ietf/doc/views_conflict_review.py b/ietf/doc/views_conflict_review.py index 159f1340a49..495958133ad 100644 --- a/ietf/doc/views_conflict_review.py +++ b/ietf/doc/views_conflict_review.py @@ -14,6 +14,7 @@ from django.template.loader import render_to_string from django.conf import settings from django.utils.html import escape +from django.db.models.functions import Collate import debug # pyflakes:ignore @@ -404,8 +405,14 @@ class SimpleStartReviewForm(forms.Form): ) class StartReviewForm(forms.Form): - ad = forms.ModelChoiceField(Person.objects.filter(role__name="ad", role__group__state="active",role__group__type='area').order_by('name'), - label="Shepherding AD", empty_label="(None)", required=True) + ad = forms.ModelChoiceField( + Person.objects.filter( + role__name="ad", role__group__state="active", role__group__type="area" + ).order_by(Collate("name", settings.PREFERRED_COLLATION)), + label="Shepherding AD", + empty_label="(None)", + required=True, + ) create_in_state = forms.ModelChoiceField(State.objects.filter(used=True, type="conflrev", slug__in=("needshep", "adrev")), empty_label=None, required=False) notify = forms.CharField( widget=forms.Textarea, diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py index a64d0a53fe9..a0904afa0bc 100644 --- a/ietf/doc/views_draft.py +++ b/ietf/doc/views_draft.py @@ -20,6 +20,8 @@ from django.forms.utils import ErrorList from django.template.defaultfilters import pluralize from django.utils import timezone +from django.db.models.functions import Collate + import debug # pyflakes:ignore @@ -1130,7 +1132,7 @@ class AdForm(forms.Form): role__name__in=("ad", "pre-ad"), role__group__state="active", role__group__type="area", - ).order_by('name'), + ).order_by(Collate('name', settings.PREFERRED_COLLATION)), label="Shepherding AD", empty_label="(None)", required=False, diff --git a/ietf/doc/views_status_change.py b/ietf/doc/views_status_change.py index 2bccc213c40..a5db6e4bdd0 100644 --- a/ietf/doc/views_status_change.py +++ b/ietf/doc/views_status_change.py @@ -18,6 +18,7 @@ from django.conf import settings from django.utils.encoding import force_str from django.utils.html import escape +from django.db.models.functions import Collate import debug # pyflakes:ignore from ietf.doc.mails import email_ad_approved_status_change @@ -484,8 +485,14 @@ def clean(self): class StartStatusChangeForm(forms.Form): document_name = forms.CharField(max_length=255, label="Document name", help_text="A descriptive name such as status-change-md2-to-historic is better than status-change-rfc1319.", required=True) title = forms.CharField(max_length=255, label="Title", required=True) - ad = forms.ModelChoiceField(Person.objects.filter(role__name="ad", role__group__state="active",role__group__type='area').order_by('name'), - label="Shepherding AD", empty_label="(None)", required=False) + ad = forms.ModelChoiceField( + Person.objects.filter( + role__name="ad", role__group__state="active", role__group__type="area" + ).order_by(Collate("name", settings.PREFERRED_COLLATION)), + label="Shepherding AD", + empty_label="(None)", + required=False, + ) create_in_state = forms.ModelChoiceField(State.objects.filter(type="statchg", slug__in=("needshep", "adrev")), empty_label=None, required=False) notify = forms.CharField( widget=forms.Textarea, diff --git a/ietf/settings.py b/ietf/settings.py index d2a622d63e6..483b08bff47 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -121,6 +121,14 @@ } +# Collation that we wish we were using. The production database is currently using +# the C collation, which does not handle accented characters. Do not change this +# without confirming that the production and dev databases support the new collation. +# This setting and places we use it can go away if we switch the production database +# collation. That requires creating and populating a new database, it cannot be done +# on an existing one. +PREFERRED_COLLATION = "en-US-x-icu" + # Local time zone for this installation. Choices can be found here: # http://www.postgresql.org/docs/8.1/static/datetime-keywords.html#DATETIME-TIMEZONE-SET-TABLE # although not all variations may be possible on all operating systems. From 2e89adb62801a5973942892e28c6c17d83525cfc Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 13 Aug 2026 01:03:21 -0300 Subject: [PATCH 2/2] feat: include doc shepherd in survey API; rename endpoint (#11503) --- ietf/api/tests.py | 112 ++++++++++++++++++++++++++++++++++++++++++---- ietf/api/urls.py | 6 ++- ietf/api/views.py | 105 +++++++++++++++++++++++++++++++++---------- 3 files changed, 189 insertions(+), 34 deletions(-) diff --git a/ietf/api/tests.py b/ietf/api/tests.py index f7fa1aff02c..75147d679a4 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -1072,10 +1072,10 @@ def test_role_holder_addresses(self): ) @override_settings( - APP_API_TOKENS={"ietf.api.views.rfc_authors": ["valid-token"]} + APP_API_TOKENS={"ietf.api.views.rfc_author_survey_recipients": ["valid-token"]} ) - def test_rfc_authors(self): - url = urlreverse("ietf.api.views.rfc_authors") + def test_rfc_author_survey_recipients(self): + url = urlreverse("ietf.api.views.rfc_author_survey_recipients") # auth and method checks self.assertEqual( self.client.get(url, headers={}).status_code, 403, "No api token, no access" @@ -1179,18 +1179,30 @@ def test_rfc_authors(self): self.assertEqual(rows[0]["name"], "Jane Q. Author") # If in test mode and testaddr parameters are present, records for those - # addresses should be returned with a fake RFC. + # addresses should be returned with a fake RFC. Also exercise specifying + # recipient type, including that author is the default. r = self.client.get( - url + "?testing&testaddr=fake@a.example.com&testaddr=phony@b.example.com", + url + + ( + "?testing" + "&testaddr=fake@a.example.com" + "&testaddr=phony@b.example.com;shepherd" + "&testaddr=ersatz@c.example.com;shepherd,author" + ), headers={"X-Api-Key": "valid-token"}, ) self.assertEqual(r.status_code, 200) rows = json.loads(r.content) - self.assertEqual(len(rows), 3) + self.assertEqual(len(rows), 4) fake_author_addr = author.email().address.split("@", 1)[0] + "@fake.example.com" self.assertCountEqual( - [fake_author_addr, "fake@a.example.com", "phony@b.example.com"], - [r["email"] for r in rows], + [ + (fake_author_addr, "author"), + ("fake@a.example.com", "author"), + ("phony@b.example.com", "shepherd"), + ("ersatz@c.example.com", "author/shepherd"), + ], + [(r["email"], r["type"]) for r in rows], ) # Can only use testaddr when testing is also present @@ -1210,6 +1222,90 @@ def test_rfc_authors(self): ) self.assertEqual(r.status_code, 400, "out-of-order from/to parameters") + @override_settings( + APP_API_TOKENS={"ietf.api.views.rfc_author_survey_recipients": ["valid-token"]} + ) + def test_rfc_author_survey_recipients_with_shepherd(self): + url = urlreverse("ietf.api.views.rfc_author_survey_recipients") + now = timezone.now() + one_day_ago = now - datetime.timedelta(days=1) + # A recently published RFC with a known author and shepherd + author = PersonFactory(name="Jane Q. Author") + recent_rfc = WgRfcFactory(title="A Recently Published RFC") + DocEventFactory(doc=recent_rfc, type="published_rfc", time=one_day_ago) + RfcAuthorFactory(document=recent_rfc, person=author) + shepherd_email = EmailFactory(person__name="Alicia Shepherd") + shepherd = shepherd_email.person + recent_rfc_draft = WgDraftFactory(shepherd_id=shepherd_email.pk) + recent_rfc_draft.relateddocument_set.create( + relationship_id="became_rfc", target=recent_rfc + ) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + rows = json.loads(r.content) + expected_rows = [ + { + "name": shepherd.name, + "email": shepherd.email_address(), + "type": "shepherd", + "rfc_number": str(recent_rfc.rfc_number), + "rfc_name": recent_rfc.name, + "rfc_number_and_title": ( + f"RFC {recent_rfc.rfc_number}: {recent_rfc.title}" + ), + "rfc_title": recent_rfc.title, + "published_date": str(recent_rfc.pub_date()), + }, + { + "name": author.name, + "email": author.email_address(), + "type": "author", + "rfc_number": str(recent_rfc.rfc_number), + "rfc_name": recent_rfc.name, + "rfc_title": recent_rfc.title, + "rfc_number_and_title": ( + f"RFC {recent_rfc.rfc_number}: {recent_rfc.title}" + ), + "published_date": str(recent_rfc.pub_date()), + }, + ] + self.assertCountEqual( + rows, + expected_rows, + ) + + # Now give the shepherd an RFC as author + other_rfc = WgRfcFactory(title="Other Published RFC") + DocEventFactory(doc=other_rfc, type="published_rfc", time=one_day_ago) + RfcAuthorFactory(document=other_rfc, person=shepherd) + + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + rows = json.loads(r.content) + # update expected rows + expected_rows[0]["type"] = "author/shepherd" + expected_rows[0]["rfc_number"] = ( + f"{recent_rfc.rfc_number}, {other_rfc.rfc_number}" + ) + expected_rows[0]["rfc_name"] = f"{recent_rfc.name}, {other_rfc.name}" + expected_rows[0]["rfc_title"] = f"{recent_rfc.title}, {other_rfc.title}" + expected_rows[0]["rfc_number_and_title"] = ( + f"RFC {recent_rfc.rfc_number}: {recent_rfc.title}, " + f"RFC {other_rfc.rfc_number}: {other_rfc.title}" + ) + expected_rows[0]["published_date"] = ( + f"{recent_rfc.pub_date()}, {other_rfc.pub_date()}" + ) + # and compare + self.assertCountEqual( + rows, + expected_rows, + ) + + + @override_settings( APP_API_TOKENS={"ietf.api.views.ingest_email": "valid-token", "ietf.api.views.ingest_email_test": "test-token"} ) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index dc1ddb6d0ca..d2dc774efb6 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -44,8 +44,10 @@ # --- Custom API endpoints, sorted alphabetically --- # Email alias information for drafts url(r'^doc/draft-aliases/$', api_views.draft_aliases), - # Authors of recently published RFCs, as CSV - url(r'^doc/rfc-authors/$', api_views.rfc_authors), + # Recipients for author survey for recently published RFCs + url( + r'^doc/rfc-author-survey-recipients/$', api_views.rfc_author_survey_recipients + ), # email ingestor url(r'email/$', api_views.ingest_email), # email ingestor diff --git a/ietf/api/views.py b/ietf/api/views.py index 9c4743c152a..5d574ac4e60 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -566,27 +566,39 @@ def role_holder_addresses(request): @requires_api_token @csrf_exempt -def rfc_authors(request): - """Return authors of published RFCs as a JSON list of objects. +def rfc_author_survey_recipients(request): + """Return recipients of the RFC Author Survey for RFCs in a time range - Finds authors by date, though this could be extended to other selection + Returns the authors and (if present) document shepherd for RFCs as a JSON + structure. This is intended for generation of the post-publication Author + Survey and is not likely to be useful elsewhere. + + Finds RFCs by date, though this could be extended to other selection criteria in the future. Specify the range as ?from=&to=. Each is an ISO-8601 timestamp, treated as UTC if it does not include time zone information. Defaults to `to`=now, `from`=14 days before `to` Each author appears once, with their RFC numbers, names, titles, and publication dates accumulated across every RFC they published in the window. + Authors have `"type": "author"` in their records. + + If an RFC has a shepherd assigned to its originating draft, that shepherd is + included in the list of recipients alongside the authors. Shepherds have + `"type": "shepherd"` in their records. If a shepherd is also an author, then + `"type": "author/shepherd"`. When the ?testing query parameter is supplied, each author's real email address is replaced with a fake one that keeps the original mailbox but uses the "fake.example.com" domain, so the output can be shared without exposing - real addresses. + real addresses. When the ?testing query parameter is supplied, one or more testaddr=ADDRESS query parameters can also be specified. The value of each parameter is an email address. When these parameters are present, the response data will include an - entry for each address as though it belonged to the author of a recently published - RFC. + entry for each address as though it belonged to an author or shepherd of a recently + published RFC. To specify the recipient type, append ";author", ";shepherd", + or ";author,shepherd" to the end of the email address. E.g., + "?testing&testaddr=somebody@example.com;author,shepherd" """ if request.method != "GET": return HttpResponse(status=405) @@ -634,56 +646,99 @@ def rfc_authors(request): docevent__type="published_rfc", docevent__time__gte=time_range_start, docevent__time__lt=time_range_end, - ).distinct() + ).order_by("rfc_number").distinct() # Collect per-author data keyed by email so each author gets one row. # Values accumulate RFC numbers, names, titles, and dates across all RFCs. - author_data = {} + recipient_data = {} for rfc in rfcs: # RfcAuthor is the authoritative source for RFC authors. Documents of # type "rfc" always have an rfcauthor_set, so no fallback is needed. - authors = [ + recipients = [ { "name": a.person.name if a.person else a.titlepage_name, "email": a.person.email().address if (a.person and a.person.email()) else None, + "type": "author", # distinguishes authors from the shepherd entry added below } for a in rfc.rfcauthor_set.select_related("person").order_by("order") ] - for author in authors: - if not author["email"]: + # As of Aug 2026, the rfc Document doesn't carry its own shepherd - it's only + # set on the draft it came from. If the shepherd changes, the value on the draft + # will be kept up to date. + # + # came_from_draft() can return None (e.g. very old RFCs with no tracked + # originating draft, April 1 RFCs, and other special cases), so guard for that. + # Also, shepherd.person should always be set (see assertion in views_doc.py) + # but skip rather than crash if that invariant is ever violated. + originating_draft = rfc.came_from_draft() + if ( + originating_draft + and originating_draft.shepherd is not None + and originating_draft.shepherd.person is not None + ): + # Use email_address() to find an active address if this one is stale + shepherd_email = originating_draft.shepherd.email_address() + shepherd = originating_draft.shepherd.person + if shepherd_email is not None: + recipients.append({ + "name": shepherd.name, + "email": shepherd_email, + "type": "shepherd", + }) + else: + log.log( + f"rfc_authors(): shepherd for {rfc.name}, has no active email " + f"address, omitting shepherd ({shepherd.name})" + ) + + for recipient in recipients: + if not recipient["email"]: continue if testing: - mailbox = author["email"].split("@", 1)[0] + mailbox = recipient["email"].split("@", 1)[0] email = f"{mailbox}@fake.example.com" else: - email = author["email"] + email = recipient["email"] - if email not in author_data: - author_data[email] = { - "name": author["name"], + if email not in recipient_data: + recipient_data[email] = { + "name": recipient["name"], "email": email, + "types": set(), "rfc_numbers": [], "rfc_names": [], "rfc_titles": [], "published_dates": [], } - author_data[email]["rfc_numbers"].append(str(rfc.rfc_number)) - author_data[email]["rfc_names"].append(rfc.name) - author_data[email]["rfc_titles"].append(rfc.title) - author_data[email]["published_dates"].append(str(rfc.pub_date())) + recipient_data[email]["rfc_numbers"].append(str(rfc.rfc_number)) + recipient_data[email]["rfc_names"].append(rfc.name) + recipient_data[email]["rfc_titles"].append(rfc.title) + recipient_data[email]["published_dates"].append(str(rfc.pub_date())) + recipient_data[email]["types"].add(recipient["type"]) if testing: for n, email in enumerate(test_addresses): - if email in author_data: - continue # author will already be included - author_data[email] = { + if email in recipient_data: + continue # recipient will already be included + # see if a type list was included + if ";" in email: + email, types = email.rsplit(";", 1) + types = {type_.strip() for type_ in types.split(",")} + if len(types.difference({"author", "shepherd"})) != 0: + return HttpResponseBadRequest( + f"Invalid types for testaddr={email}" + ) + else: + types = {"author"} + recipient_data[email] = { "name": f"Test Author {n + 1}", "email": email, + "types": types, "rfc_numbers": ["99999"], "rfc_names": ["rfc99999"], "rfc_titles": ["A Fake RFC for Testing"], @@ -691,11 +746,13 @@ def rfc_authors(request): } rows = [] - for entry in author_data.values(): + for entry in recipient_data.values(): + entry_type = "/".join(sorted(entry["types"])) rows.append( { "name": entry["name"], "email": entry["email"], + "type": entry_type, # "author", "shepherd", or "author/shepherd" "rfc_number": ", ".join(entry["rfc_numbers"]), "rfc_name": ", ".join(entry["rfc_names"]), "rfc_title": ", ".join(entry["rfc_titles"]),