diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index d80aceaffb5..d13405fd4ba 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20260804T1752 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260827T1454 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index d40bd9e9299..d5d96a8f329 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20260804T1752 +20260827T1454 diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 94adc516a47..227da0a0ace 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -115,6 +115,8 @@ APP_API_TOKENS = { "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret "ietf.api.views_rpc" : ["devtoken"], # Not a real secret + "ietf.person.api_uuid" : ["devtoken"], # Not a real secret + "ietf.person.api_uuid_by_pk" : ["devtoken"], # Not a real secret } # Errata system api configuration diff --git a/ietf/api/urls.py b/ietf/api/urls.py index d2dc774efb6..072c913ad79 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -8,7 +8,9 @@ from ietf import api from ietf.doc import views_ballot, api as doc_api +from ietf.meeting import api as meeting_api from ietf.meeting import views as meeting_views +from ietf.person import api_uuid as person_uuid_api from ietf.submit import views as submit_views from ietf.utils.urls import url @@ -21,6 +23,14 @@ # core_router.register("email", person_api.EmailViewSet) # core_router.register("person", person_api.PersonViewSet) +# Person identity API router +person_router = PrefixedSimpleRouter( + use_regex_path=False, name_prefix="ietf.api.person_api" +) +person_router.register( + "uuid", person_uuid_api.PersonUUIDViewSet, basename="person-uuid" +) + # todo more general name for this API? red_router = PrefixedSimpleRouter(name_prefix="ietf.api.red_api") # red api router red_router.register("doc", doc_api.RfcViewSet) @@ -71,6 +81,7 @@ url(r'^meeting/(?P[A-Za-z0-9._+-]+)/agenda-data$', meeting_views.api_get_agenda_data), # Meeting session materials url(r'^meeting/session/(?P[A-Za-z0-9._+-]+)/materials$', meeting_views.api_get_session_materials), + url(r'^meeting/registration/attended/(?P[^/\x00]+)/?$', meeting_api.MeetingsAttendedByEmail.as_view(), name="ietf.api.meeting.registration.attended"), # Let MeetEcho upload bluesheets url(r'^notify/meeting/bluesheet/?$', meeting_views.api_upload_bluesheet), # Let MeetEcho tell us about session attendees @@ -88,6 +99,16 @@ url(r'^person/email/$', api_views.active_email_list), # Related Email listing url(r'^person/email/(?P[^/\x00]+)/related/$', api_views.related_email_list), + # Transitional pk-to-UUID conversion. Before the router include below so it wins + # over the router's uuid/ routes. + path( + "person/uuid/by-person-pk/", + person_uuid_api.PersonUUIDByPersonPkView.as_view(), + name="ietf.api.person_api.person-uuid-by-pk", + ), + # Person UUID resolution API. After the ^person/email/ routes above so those keep + # matching first. + path("person/", include(person_router.urls)), # Draft submission API url(r'^submit/?$', submit_views.api_submit_tombstone), # Draft upload API diff --git a/ietf/api/views.py b/ietf/api/views.py index 5d574ac4e60..0b766d22d07 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -90,8 +90,6 @@ def top_level(request): def api_help(request): key = JWK() - # import just public part here, for display in info page - key.import_from_pem(settings.API_PUBLIC_KEY_PEM) return render(request, "api/index.html", {'key': key, 'settings':settings, }) diff --git a/ietf/checks.py b/ietf/checks.py index f911d081f0b..3853e49f04e 100644 --- a/ietf/checks.py +++ b/ietf/checks.py @@ -4,7 +4,6 @@ import os import time -from textwrap import dedent from typing import List, Tuple # pyflakes:ignore import debug # pyflakes:ignore @@ -270,40 +269,3 @@ def maybe_patch_library(app_configs, **kwargs): ) pass return errors - -@checks.register('security') -def check_api_key_in_local_settings(app_configs, **kwargs): - errors = [] - import ietf.settings_local - if settings.SERVER_MODE == 'production': - if not ( hasattr(ietf.settings_local, 'API_PUBLIC_KEY_PEM') - and hasattr(ietf.settings_local, 'API_PRIVATE_KEY_PEM')): - errors.append(checks.Critical( - "There are no API key settings in your settings_local.py", - hint = dedent(""" - You are running in production mode, and need API key settings that are - different than the default settings. Please add settings for - API_PUBLIC_KEY_PEM and API_PRIVATE_KEY_PEM to your settings local. The - content should be matching public and private keys in PEM format. You - can generate a suitable keypair with 'ssh-keygen -f apikey.pem', and then - extract the public key with 'openssl rsa -in apikey.pem -pubout > apikey.pub'. - - """).replace('\n', '\n ').rstrip(), - id = "datatracker.E0020", - )) - elif not ( ietf.settings_local.API_PUBLIC_KEY_PEM == settings.API_PUBLIC_KEY_PEM - and ietf.settings_local.API_PRIVATE_KEY_PEM == settings.API_PRIVATE_KEY_PEM ): - errors.append(checks.Critical( - "Your API key settings in your settings_local.py are not picked up in settings.", - hint = dedent(""" - You are running in production mode, and need API key settings which are - different than the default settings. You seem to have API key settings - in settings_local.py, but they don't seem to propagate to django.conf.settings. - Please check if you have multiple settings_local.py files. - - """).replace('\n', '\n ').rstrip(), - id = "datatracker.E0021", - )) - - return errors - diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index 86f5ac5fda1..78e55a0add7 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -15,7 +15,7 @@ AddedMessageEvent, SubmissionDocEvent, DeletedEvent, EditedAuthorsDocEvent, DocumentURL, ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject, RfcAuthor, - EditedRfcAuthorsDocEvent, RpcAssignmentDocEvent) + EditedRfcAuthorsDocEvent, RpcAssignmentDocEvent, RpcActionHolderOpenEntry) from ietf.utils.admin import SaferTabularInline from ietf.utils.validators import validate_external_resource_value @@ -233,6 +233,26 @@ class RpcAssignmentDocEventAdmin(DocEventAdmin): search_fields = DocEventAdmin.search_fields + ["assignments"] admin.site.register(RpcAssignmentDocEvent, RpcAssignmentDocEventAdmin) +class RpcActionHolderOpenEntryAdmin(admin.ModelAdmin): + """Read-only view of the open action holder entries from the RPC tool + + The RPC tool owns these - every queue push replaces them - so editing them + here would accomplish nothing. + """ + list_display = ['id', 'purple_id', 'document', 'name', 'since_when', 'deadline', ] + search_fields = ['document__name', 'person__name', 'body', ] + raw_id_fields = ['document', 'person', ] + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False +admin.site.register(RpcActionHolderOpenEntry, RpcActionHolderOpenEntryAdmin) + class DocumentUrlAdmin(admin.ModelAdmin): list_display = ['id', 'doc', 'tag', 'url', 'desc', ] search_fields = ['doc__name', 'url', ] diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index 0cf83e4525f..aadac27c6f9 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -14,7 +14,8 @@ from ietf.doc.models import ( Document, DocEvent, NewRevisionDocEvent, State, DocumentAuthor, StateDocEvent, BallotPositionDocEvent, BallotDocEvent, BallotType, IRSGBallotDocEvent, TelechatDocEvent, - DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, DocExtResource, RfcAuthor ) + DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, DocExtResource, RfcAuthor, + RpcActionHolderOpenEntry ) from ietf.group.models import Group from ietf.person.factories import PersonFactory from ietf.group.factories import RoleFactory @@ -384,6 +385,17 @@ class Meta: document = factory.SubFactory(WgDraftFactory) person = factory.SubFactory('ietf.person.factories.PersonFactory') +class RpcActionHolderOpenEntryFactory(factory.django.DjangoModelFactory): + class Meta: + model = RpcActionHolderOpenEntry + + purple_id = factory.Sequence(lambda n: n + 1) + document = factory.SubFactory(WgDraftFactory) + person = factory.SubFactory('ietf.person.factories.PersonFactory') + display_name = factory.LazyAttribute(lambda o: o.person.plain_name() if o.person else '') + comment = factory.Faker('sentence') + since_when = factory.LazyFunction(timezone.now) + class DocumentAuthorFactory(factory.django.DjangoModelFactory): class Meta: model = DocumentAuthor diff --git a/ietf/doc/migrations/0038_rpcactionholderopenentry.py b/ietf/doc/migrations/0038_rpcactionholderopenentry.py new file mode 100644 index 00000000000..365a7f5ba63 --- /dev/null +++ b/ietf/doc/migrations/0038_rpcactionholderopenentry.py @@ -0,0 +1,69 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion +import ietf.utils.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("person", "0005_alter_historicalperson_pronouns_selectable_and_more"), + ("doc", "0037_rpcassignmentdocevent"), + ] + + operations = [ + migrations.CreateModel( + name="RpcActionHolderOpenEntry", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "purple_id", + models.PositiveIntegerField( + help_text="ID of the ActionHolder in the RPC tool", unique=True + ), + ), + ( + "body", + models.CharField( + blank=True, + default="", + help_text="Name of the body holding the action, if it is not a person", + max_length=64, + ), + ), + ( + "display_name", + models.CharField(blank=True, default="", max_length=255), + ), + ("comment", models.TextField(blank=True)), + ("rfc_number", models.PositiveIntegerField(blank=True, null=True)), + ("since_when", models.DateTimeField()), + ("deadline", models.DateTimeField(blank=True, null=True)), + ("time_captured", models.DateTimeField(auto_now=True)), + ( + "document", + ietf.utils.models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="doc.document" + ), + ), + ( + "person", + ietf.utils.models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="person.person", + ), + ), + ], + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 708bbb2ede9..8aed1726103 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -401,8 +401,33 @@ def get_state_slug(self, state_type=None): self._cached_state_slug[state_type] = s.slug if s else None return self._cached_state_slug[state_type] - def friendly_state(self): - """ Return a concise text description of the document's current state.""" + def rfc_editor_queue_status(self): + """Human-readable RPC publication queue "Status" for the document, or None. + + While a document is in the RFC Editor queue (draft-rfceditor state + "in_progress" or "blocked"), this is the status text pushed by the RFC + Production Center, matching what the queue website shows. Displays of the RFC + Editor state show it in place of the state name. It is None for a document whose + draft-rfceditor state predates the queue integration, which leaves those displays + showing the state name itself. + + Costs a query per call. Tables rendering many documents replace this with a + precomputed value; see ietf.doc.utils_search.fill_in_rfc_editor_queue_status. + """ + if self.get_state_slug("draft-rfceditor") not in ("in_progress", "blocked"): + return None + event = self.latest_event(RpcAssignmentDocEvent, type="changed_rpc_assignments") + return event.assignments if event else None + + def friendly_state(self, label_iesg_state=True): + """ Return a concise text description of the document's current state. + + For a draft in the RFC Editor queue that description is "IESG: RFC Ed Queue", + labeled because displays of it sit next to the RFC Editor's own status for the + document. Every other state stands on its own and is returned unlabeled. Callers + rendering the description somewhere already labeled as the IESG's pass + label_iesg_state=False. + """ state = self.get_state() if not state: return "Unknown state" @@ -440,7 +465,11 @@ def friendly_state(self): e = self.latest_event(LastCallDocEvent, type="sent_last_call") if e: return iesg_state_summary + " (ends %s)" % e.expires.astimezone(DEADLINE_TZINFO).date().isoformat() - + elif label_iesg_state and iesg_state.slug == "rfcqueue": + # The only state whose displays sit next to the RFC Editor's own + # status for the document, where a bare state name is ambiguous. + return "IESG: %s" % iesg_state_summary + return iesg_state_summary else: return "I-D Exists" @@ -1626,6 +1655,77 @@ class ConsensusDocEvent(DocEvent): class RpcAssignmentDocEvent(DocEvent): assignments = models.TextField(blank=True) + +class RpcActionHolderOpenEntry(models.Model): + """Open action holder entry from the RFC Production Center + + A read-only capture of the RPC tool's open ActionHolder entries, held here + so the datatracker can query them efficiently - who is the RPC waiting on, + and for which documents. The RPC tool owns the entries. It sends the full + set for every document in the publication queue on each push to + /api/purple/queue/process/, and ietf.sync.tasks.process_rpc_queue_task + reconciles this table against that push. Nothing else writes it, there is no + editing UI, and any local change is discarded by the next push. + + Only open entries are kept: an entry the RPC has completed is dropped at + ingest, and rows disappear when their document leaves the publication queue. + Every row present is therefore one the RPC is still waiting on, so readers + do not need to filter. + + Not to be confused with DocumentActionHolder, which is the datatracker's own + action holder list for documents in IESG processing. + """ + + purple_id = models.PositiveIntegerField( + unique=True, help_text="ID of the ActionHolder in the RPC tool" + ) + document = ForeignKey(Document) + # Null when a body rather than a person holds the action, when the RPC tool + # sent its system person as a placeholder, or when it named a person the + # datatracker cannot resolve. Never the "(System)" person. + person = ForeignKey(Person, blank=True, null=True) + body = models.CharField( + max_length=64, + blank=True, + default="", + help_text="Name of the body holding the action, if it is not a person", + ) + display_name = models.CharField(max_length=255, blank=True, default="") + comment = models.TextField(blank=True) + # Belongs to the document rather than to the action holder, and is null + # until the RPC assigns it. Used to build the final review link. + rfc_number = models.PositiveIntegerField(blank=True, null=True) + # Only the date components of since_when / deadline are significant - the + # RPC tool customarily sets the time to 12:00 UTC. + since_when = models.DateTimeField() + deadline = models.DateTimeField(blank=True, null=True) + time_captured = models.DateTimeField(auto_now=True) + + def __str__(self): + return "%s: open action held by %s" % (self.document.name, self.name()) + + def name(self): + """Best available label for whoever holds this action""" + if self.body: + return self.body + if self.person: + return self.person.plain_name() + return self.display_name + + def final_review_url(self): + """Link to the queue site final review page, if there is one + + A document can have an action holder before the RPC assigns it an RFC + number, and there is no final review page until it does. + """ + if self.rfc_number is None: + return None + return "%s/final-review/rfc%d/" % ( + settings.RFC_EDITOR_QUEUE_SITE_BASE_URL, + self.rfc_number, + ) + + # IESG events class BallotType(models.Model): doc_type = ForeignKey(DocTypeName, blank=True, null=True) diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index 9da7cb57d80..f7db33925cd 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -19,7 +19,7 @@ ReviewRequestDocEvent, ReviewAssignmentDocEvent, EditedAuthorsDocEvent, DocumentURL, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject, RfcAuthor, - EditedRfcAuthorsDocEvent, RpcAssignmentDocEvent) + EditedRfcAuthorsDocEvent, RpcAssignmentDocEvent, RpcActionHolderOpenEntry) from ietf.name.resources import BallotPositionNameResource, DocTypeNameResource class BallotTypeResource(ModelResource): @@ -941,3 +941,28 @@ class Meta: "docevent_ptr": ALL_WITH_RELATIONS, } api.doc.register(RpcAssignmentDocEventResource()) + + +class RpcActionHolderOpenEntryResource(ModelResource): + document = ToOneField(DocumentResource, 'document') + person = ToOneField(PersonResource, 'person', null=True) + class Meta: + queryset = RpcActionHolderOpenEntry.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'rpcactionholderopenentry' + ordering = ['id', ] + filtering = { + "id": ALL, + "purple_id": ALL, + "body": ALL, + "display_name": ALL, + "comment": ALL, + "rfc_number": ALL, + "since_when": ALL, + "deadline": ALL, + "time_captured": ALL, + "document": ALL_WITH_RELATIONS, + "person": ALL_WITH_RELATIONS, + } +api.doc.register(RpcActionHolderOpenEntryResource()) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 95bee78ae5f..83a152c148b 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -56,11 +56,13 @@ StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory, StdFactory, FyiFactory, RfcAuthorFactory, + RpcActionHolderOpenEntryFactory, TelechatDocEventFactory) from ietf.doc.forms import NotifyForm from ietf.doc.fields import SearchableDocumentsField from ietf.doc.utils import ( create_ballot_if_not_open, + save_document_in_history, external_canonical_url, investigate_fragment, uppercase_std_abbreviated_name, @@ -86,7 +88,8 @@ from ietf.utils.test_utils import TestCase from ietf.utils.text import normalize_text, texescape from ietf.utils.timezone import date_today, datetime_today, DEADLINE_TZINFO, RPC_TZINFO -from ietf.doc.utils_search import AD_WORKLOAD, fill_in_telechat_date, prepare_document_table +from ietf.doc.utils_search import (AD_WORKLOAD, fill_in_rfc_editor_queue_status, + fill_in_telechat_date, prepare_document_table) class SearchTests(TestCase): @@ -308,21 +311,38 @@ def test_search_query_count_does_not_grow_with_results(self): so doubling the number of rows must not change the number of queries. A lookup that slipped back into the per-row path shows up here as a count that grows. - Mind the blind spots: these documents have no IESG state, ballot, last call, - action holders, telechat or obsoleting RFCs, so the per-row work the columns - driven by those still do is not covered. Widen the fixtures rather than reading - a pass here as "the table does no per-row queries". + Mind the blind spots: these documents have no ballot, last call, action holders, + telechat, obsoleting RFCs, or IESG state under way, so the per-row work the + columns driven by those still do is not covered -- state_age_colored and the + action holder list each still cost a query for a document being processed by the + IESG. Widen the fixtures rather than reading a pass here as "the table does no + per-row queries". """ group = GroupFactory(type_id="wg") url = urlreverse('ietf.doc.views_search.search') + ( f"?activedrafts=on&olddrafts=on&rfcs=on&by=group&group={group.acronym}" ) + system = Person.objects.get(name="(System)") def add_documents(count): for _ in range(count): WgDraftFactory(group=group, authors=[PersonFactory()], ad=PersonFactory(), shepherd=EmailFactory()) WgRfcFactory(group=group) + # A draft in the RFC Editor queue, whose row also shows the queue status. + # Not one the IESG is processing, to keep the blind spots above out of + # the count. + queued = WgDraftFactory( + group=group, + states=[("draft", "active"), ("draft-iesg", "idexists"), + ("draft-rfceditor", "in_progress")], + ) + RpcAssignmentDocEvent.objects.create( + doc=queued, rev=queued.rev, by=system, + type="changed_rpc_assignments", + assignments="In Progress (First Edit)", + desc="RPC status changed to In Progress (First Edit)", + ) def count_queries(): with CaptureQueriesContext(connection) as context: @@ -335,7 +355,7 @@ def count_queries(): add_documents(4) doubled = count_queries() - # A per-row lookup would add at least one query for each of the 8 new documents. + # A per-row lookup would add at least one query for each of the 12 new documents. self.assertLessEqual( doubled, baseline + 2, f"query count grew from {baseline} to {doubled} when the result set tripled", @@ -387,6 +407,15 @@ def test_prepared_documents_are_picklable(self): shepherd=EmailFactory()) TelechatDocEventFactory(doc=draft) WgRfcFactory() + queued = WgDraftFactory( + states=[("draft", "active"), ("draft-iesg", "rfcqueue"), + ("draft-rfceditor", "in_progress")], + ) + RpcAssignmentDocEvent.objects.create( + doc=queued, rev=queued.rev, by=Person.objects.get(name="(System)"), + type="changed_rpc_assignments", assignments="In Progress (First Edit)", + desc="RPC status changed to In Progress (First Edit)", + ) request = RequestFactory().get("/doc/recent/") request.user = AnonymousUser() @@ -396,6 +425,9 @@ def test_prepared_documents_are_picklable(self): restored, _ = pickle.loads(pickle.dumps([results, meta])) for before, after in zip(results, restored): self.assertEqual(after.telechat_date(), before.telechat_date()) + self.assertEqual( + after.rfc_editor_queue_status(), before.rfc_editor_queue_status() + ) def test_fill_in_telechat_date_matches_the_method(self): """The precomputed value has to equal what Document.telechat_date() returns. @@ -433,6 +465,49 @@ def test_fill_in_telechat_date_matches_the_method(self): self.assertIsNotNone(expected[future.name]) self.assertIsNone(expected[past.name]) + def test_fill_in_rfc_editor_queue_status_matches_the_method(self): + """The precomputed value has to equal what Document.rfc_editor_queue_status() returns. + + The document page renders the method and the document tables render the + precomputed value, so a mismatch shows the same document two different statuses. + """ + system = Person.objects.get(name="(System)") + + def queued(state_slug, *statuses): + doc = WgDraftFactory( + states=[("draft", "active"), ("draft-iesg", "rfcqueue"), + ("draft-rfceditor", state_slug)], + ) + for status in statuses: + RpcAssignmentDocEvent.objects.create( + doc=doc, rev=doc.rev, by=system, type="changed_rpc_assignments", + assignments=status, desc=f"RPC status changed to {status}", + ) + return doc + + in_progress = queued("in_progress", "In Progress (First Edit)") + # A document that has moved on has an event for every status it has held. + moved_on = queued("in_progress", "Awaiting First editor", "In Final Review") + blocked = queued("blocked", "blocked: Manual Hold") + # No queue status: a state from before the queue integration, and no state at all. + legacy = queued("rfc-edit", "In Progress (First Edit)") + not_queued = WgDraftFactory() + + fixtures = (in_progress, moved_on, blocked, legacy, not_queued) + expected = { + d.name: Document.objects.get(pk=d.pk).rfc_editor_queue_status() + for d in fixtures + } + self.assertEqual(expected[moved_on.name], "In Final Review") + self.assertIsNone(expected[legacy.name]) + + docs = list(Document.objects.filter(name__in=[d.name for d in fixtures])) + doc_dict = {d.pk: d for d in docs} + fill_in_rfc_editor_queue_status(docs, doc_dict, list(doc_dict)) + + for doc in docs: + self.assertEqual(doc.rfc_editor_queue_status(), expected[doc.name], doc.name) + def test_search_for_name(self): draft = WgDraftFactory(name='draft-ietf-mars-test',group=GroupFactory(acronym='mars',parent=Group.objects.get(acronym='farfut')),authors=[PersonFactory()],ad=PersonFactory()) draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="pub-req")) @@ -651,6 +726,80 @@ def test_docs_for_ad(self): self.assertContains(r, discuss_other.doc.name) self.assertContains(r, block_other.doc.name) + def test_ad_workload_shows_rpc_decisions_pending(self): + """The IESG dashboard shows who the RPC is currently waiting on""" + # ad_list.html builds element ids from AD names, so start from a known + # set of ADs whose names slugify predictably. + Role.objects.filter(name_id="ad").delete() + ad = RoleFactory(name_id='ad', group__type_id='area', group__state_id='active', + person__name='Example Areadirector').person + idle_ad = RoleFactory(name_id='ad', group__type_id='area', group__state_id='active', + person__name='Other Areadirector').person + RpcActionHolderOpenEntryFactory(person=ad) + RpcActionHolderOpenEntryFactory(person=ad) + + url = urlreverse('ietf.doc.views_search.ad_workload') + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertContains(r, 'RPC decisions pending') + q = PyQuery(r.content) + rows = q('#rpc-pending').next('p').next('table').find('tbody tr') + self.assertEqual(len(rows), 1, 'only ADs with something pending get a row') + self.assertIn(ad.plain_name(), rows.text()) + self.assertNotIn(idle_ad.plain_name(), rows.text()) + self.assertIn('2', rows.text()) + + def test_ad_workload_without_rpc_decisions_pending(self): + """The section is absent when the RPC is waiting on nobody""" + Role.objects.filter(name_id="ad").delete() + RoleFactory(name_id='ad', group__type_id='area', group__state_id='active', + person__name='Example Areadirector') + r = self.client.get(urlreverse('ietf.doc.views_search.ad_workload')) + self.assertEqual(r.status_code, 200) + self.assertNotContains(r, 'RPC decisions pending') + + def test_docs_for_ad_shows_rpc_action_holders(self): + """An AD sees the decisions the RPC is waiting on them for""" + ad = RoleFactory(name_id='ad', group__type_id='area', group__state_id='active').person + other_ad = RoleFactory(name_id='ad', group__type_id='area', group__state_id='active').person + entry = RpcActionHolderOpenEntryFactory( + person=ad, comment='Confirm the change in section 4.2', rfc_number=9850 + ) + other_entry = RpcActionHolderOpenEntryFactory(person=other_ad) + + url = urlreverse('ietf.doc.views_search.docs_for_ad', + kwargs=dict(name=ad.full_name_as_key())) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertContains(r, 'RPC decisions pending') + self.assertContains(r, entry.document.name) + self.assertNotContains(r, other_entry.document.name) + # the queue site final review page for this document + self.assertContains(r, f'{settings.RFC_EDITOR_QUEUE_SITE_BASE_URL}/final-review/rfc9850/') + # the request is shown here to everyone, including the anonymous user + self.assertContains(r, 'Confirm the change in section 4.2') + + self.client.login(username=ad.user.username, password=ad.user.username + '+password') + self.assertContains(self.client.get(url), 'Confirm the change in section 4.2') + + def test_docs_for_ad_without_rpc_action_holders(self): + """The section is absent when the RPC is waiting on nothing""" + ad = RoleFactory(name_id='ad', group__type_id='area', group__state_id='active').person + r = self.client.get(urlreverse('ietf.doc.views_search.docs_for_ad', + kwargs=dict(name=ad.full_name_as_key()))) + self.assertEqual(r.status_code, 200) + self.assertNotContains(r, 'RPC decisions pending') + + def test_docs_for_ad_rpc_action_holder_without_rfc_number(self): + """A document with no rfc number yet has no final review page to link""" + ad = RoleFactory(name_id='ad', group__type_id='area', group__state_id='active').person + RpcActionHolderOpenEntryFactory(person=ad, rfc_number=None) + r = self.client.get(urlreverse('ietf.doc.views_search.docs_for_ad', + kwargs=dict(name=ad.full_name_as_key()))) + self.assertEqual(r.status_code, 200) + self.assertContains(r, 'RPC decisions pending') + self.assertNotContains(r, '/final-review/') + def test_docs_for_iesg(self): ad1 = RoleFactory(name_id='ad',group__type_id='area',group__state_id='active').person ad2 = RoleFactory(name_id='ad',group__type_id='area',group__state_id='active').person @@ -688,6 +837,90 @@ def test_auth48_doc_for_ad(self): self.assertContains(r, draft.name) self.assertContains(r, 'title="AUTH48"') # title attribute of AUTH48 badge in auth48_alert_badge filter + def _status_column_text(self, draft): + r = self.client.get( + urlreverse("ietf.doc.views_search.search") + + f"?activedrafts=on&olddrafts=on&rfcs=on&name={draft.name}" + ) + self.assertEqual(r.status_code, 200) + return PyQuery(r.content)("td.status").text() + + def test_search_labels_the_iesg_state_of_a_queued_draft(self): + """The RFC Ed Queue state is labeled as the IESG's, because the row also shows + the RFC Editor's own status. Other states are left to speak for themselves. + """ + queued = IndividualDraftFactory( + states=[("draft", "active"), ("draft-iesg", "rfcqueue"), + ("draft-rfceditor", "in_progress")] + ) + queue_state_name = queued.get_state("draft-iesg").name + self.assertIn(f"IESG: {queue_state_name}", self._status_column_text(queued)) + # The document page has a row labeled "IESG state" already, so the value there + # is the bare state name. + r = self.client.get( + urlreverse("ietf.doc.views_doc.document_main", + kwargs=dict(name=queued.name)) + ) + self.assertEqual(r.status_code, 200) + self.assertContains(r, queue_state_name) + self.assertNotContains(r, f"IESG: {queue_state_name}") + + for slug, states in ( + ("iesg-eva", [("draft", "active"), ("draft-iesg", "iesg-eva")]), + ("idexists", [("draft", "active"), ("draft-iesg", "idexists")]), + ("expired", [("draft", "expired"), ("draft-iesg", "idexists")]), + ): + with self.subTest(slug=slug): + draft = IndividualDraftFactory(states=states) + self.assertNotIn("IESG: ", self._status_column_text(draft)) + + # friendly_state labels this one itself; it must not be labeled twice. + dead = IndividualDraftFactory( + states=[("draft", "active"), ("draft-iesg", "dead")] + ) + self.assertIn("I-D Exists (IESG: Dead)", self._status_column_text(dead)) + + def test_search_shows_rfc_editor_queue_status(self): + """A queued draft's row shows the publication queue Status, as its document page does. + + The status column carries two state machines at once, so each state is labeled + with the body it belongs to. + """ + draft = IndividualDraftFactory( + states=[ + ("draft", "active"), + ("draft-iesg", "rfcqueue"), + ("draft-rfceditor", "in_progress"), + ] + ) + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=Person.objects.get(name="(System)"), + type="changed_rpc_assignments", + assignments="In Progress (First Edit)", + desc="RPC status changed to In Progress (First Edit)", + ) + status = self._status_column_text(draft) + self.assertIn(f"IESG: {draft.get_state('draft-iesg').name}", status) + self.assertIn("RFC Editor: In Progress (First Edit)", status) + + def test_search_falls_back_to_rfc_editor_state_name(self): + """A document with no queue status shows its draft-rfceditor state name. + + Documents that went through the RFC Editor before the publication queue + integration have a draft-rfceditor state but no RpcAssignmentDocEvent. + """ + draft = IndividualDraftFactory( + states=[ + ("draft", "active"), + ("draft-iesg", "rfcqueue"), + ("draft-rfceditor", "rfc-edit"), + ] + ) + status = self._status_column_text(draft) + self.assertIn(f"RFC Editor: {draft.get_state('draft-rfceditor').name}", status) + def test_drafts_in_last_call(self): draft = IndividualDraftFactory(pages=1) draft.action_holders.set([PersonFactory()]) @@ -945,6 +1178,47 @@ def setUp(self): with (Path(dir) / 'draft-ietf-mars-test-01.txt').open('w') as f: f.write(self.draft_text) + def test_document_draft_rpc_action_holders(self): + """A draft in the RFC Editor queue shows who the RPC is waiting on""" + draft = WgDraftFactory(states=[('draft-iesg', 'rfcqueue'), + ('draft-rfceditor', 'in_progress')], + rev='00') + holder = PersonFactory() + RpcActionHolderOpenEntryFactory( + document=draft, person=holder, comment='Confirm the change in section 4.2' + ) + # An action held by a body is not shown here - the queue status covers it + RpcActionHolderOpenEntryFactory( + document=draft, person=None, body='Registry Of Xyzzy', + display_name='Registry Of Xyzzy' + ) + url = urlreverse('ietf.doc.views_doc.document_main', kwargs=dict(name=draft.name)) + + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertContains(r, escape(holder.name)) + self.assertNotContains(r, 'Registry Of Xyzzy') + self.assertNotContains(r, 'Confirm the change in section 4.2') + + # the person being asked can see the request even though they are not an AD + self.client.login(username=holder.user.username, + password=holder.user.username + '+password') + self.assertContains(self.client.get(url), 'Confirm the change in section 4.2') + + # An earlier revision is a snapshot of the past and reports none of it. + # The snapshot is taken while the draft is in the queue, so its own + # RFC Editor state is set and that block of the page does render. + save_document_in_history(draft) + draft.rev = '01' + draft.save() + snapshot_url = urlreverse('ietf.doc.views_doc.document_main', + kwargs=dict(name=draft.name, rev='00')) + r = self.client.get(snapshot_url) + self.assertEqual(r.status_code, 200) + self.assertContains(r, 'RFC Editor status') # the block is there ... + self.assertNotContains(r, escape(holder.name)) # ... without the holders + self.assertNotContains(r, 'Confirm the change in section 4.2') + def test_document_draft(self): draft = WgDraftFactory(name='draft-ietf-mars-test',rev='01', create_revisions=range(0,2)) @@ -3374,8 +3648,23 @@ def test_obsoleted(self): def test_generate_idnits2_rfc_status(self): for slug in ('bcp', 'ds', 'exp', 'hist', 'inf', 'std', 'ps', 'unkn'): WgRfcFactory(std_level_id=slug) + WgRfcFactory(rfc_number=10001, std_level_id='ps') + WgRfcFactory(rfc_number=10002, std_level_id=None) + blob = generate_idnits2_rfc_status().replace("\n", "") + self.assertEqual(blob[6312-1], "O") + # idnits2 discards the whole file if this is not "O" - see generate_idnits2_rfc_status + self.assertEqual(blob[16-1], "O") + self.assertEqual(blob[10001-1], "P") + self.assertEqual(blob[10002-1], "U") + + def test_generate_idnits2_rfc_status_low_numbers_only(self): + # The workarounds write fixed offsets, so the blob has to reach 6312 even when + # no RFC does. + WgRfcFactory(rfc_number=1001, std_level_id="ps") blob = generate_idnits2_rfc_status().replace("\n", "") self.assertEqual(blob[6312-1], "O") + self.assertEqual(blob[200-1], "O") + self.assertEqual(blob[16-1], "O") def test_rfc_status(self): url = urlreverse('ietf.doc.views_doc.idnits2_rfc_status') diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index c371ba6f2e4..301c352a0de 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -21,7 +21,7 @@ from django.conf import settings from django.contrib import messages from django.core.cache import caches -from django.db.models import OuterRef +from django.db.models import Max, OuterRef from django.forms import ValidationError from django.http import Http404 from django.template.loader import render_to_string @@ -456,6 +456,24 @@ def add_state_change_event(doc, by, prev_state, new_state, prev_tags=None, new_t return e +def show_rpc_action_holder_comments(user, entries): + """Should this user be shown the RPC's requests to these action holders? + + Controls whether a view renders the comment the RPC attached to an action + holder. Only the document main page views ask: they show it to the IESG, the + Secretariat and the RPC, and to whoever is being asked for the decision. + Views that do not call this show the comments to everyone, as the AD + document list does. The text is not confidential either way - the queue site + publishes it on its public final-review pages - so this is a choice about + what belongs on a given page. + """ + if has_role(user, ["Area Director", "Secretariat", "RFC Editor"]): + return True + if not (user.is_authenticated and hasattr(user, "person")): + return False + return any(entry.person_id == user.person.pk for entry in entries) + + def add_action_holder_change_event(doc, by, prev_set, reason=None): set_changed = False if doc.documentactionholder_set.exclude(person__in=prev_set).exists(): @@ -1368,8 +1386,6 @@ def update_doc_extresources(doc, new_resources, by): def generate_idnits2_rfc_status(): - blob=['N']*10000 - symbols={ 'ps': 'P', 'inf': 'I', @@ -1381,10 +1397,17 @@ def generate_idnits2_rfc_status(): 'unkn': 'U', } - rfcs = Document.objects.filter(type_id='rfc') + rfcs = Document.objects.filter(type_id='rfc').exclude(rfc_number=None) + + # One character per RFC number, indexed by that number, so the array has to reach the + # highest RFC published. The floor keeps the fixed offsets in the workarounds below in + # range when only a few RFCs exist, as is the case under test. + highest = rfcs.aggregate(Max('rfc_number'))['rfc_number__max'] or 0 + blob=['N']*max(highest, 6312) + for rfc in rfcs: offset = int(rfc.rfc_number)-1 - blob[offset] = symbols[rfc.std_level_id] + blob[offset] = symbols.get(rfc.std_level_id, 'U') if rfc.related_that('obs'): blob[offset] = 'O' @@ -1404,6 +1427,17 @@ def generate_idnits2_rfc_status(): # RFC200 is an old RFC List by Number blob[200 -1] = 'O' + # !! Do not remove: idnits2 rejects this entire file if RFC16 is not 'O' !! + # + # This deliberately contradicts both the datatracker and the RFC Editor, which + # record RFC16 as updated rather than obsoleted. idnits2 validates its download + # of this file by matching the first 64 characters against a literal pattern that + # asserts 'O' here, inherited from a tools.ietf.org curation that disagreed with + # the RFC Editor. A mismatch makes idnits2 discard the file as corrupt and fall + # back to whatever stale copy it has, silently performing no RFC status checks at + # all. Removing this line therefore breaks every idnits2 client, not just RFC16. + blob[16 - 1] = 'O' + # End Workarounds blob = re.sub('N*$','',''.join(blob)) diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py index 267c51aeee7..45108debb18 100644 --- a/ietf/doc/utils_search.py +++ b/ietf/doc/utils_search.py @@ -10,7 +10,8 @@ from django.conf import settings -from ietf.doc.models import Document, RelatedDocument, DocEvent, TelechatDocEvent, BallotDocEvent, DocTypeName +from ietf.doc.models import (Document, RelatedDocument, DocEvent, TelechatDocEvent, BallotDocEvent, + DocTypeName, RpcAssignmentDocEvent) from ietf.doc.expire import expirable_drafts from ietf.doc.utils import augment_docs_and_person_with_person_info from ietf.meeting.models import SessionPresentation, Meeting, Session @@ -159,6 +160,39 @@ def reachable_from(start): d.related_ipr = wrap_value(sorted(related)) +def fill_in_rfc_editor_queue_status(docs, doc_dict, doc_ids): + """Attach each document's RFC Editor publication queue status. + + The status column shows it for every document sitting in the RFC Editor queue, and + Document.rfc_editor_queue_status() costs a query per such row to find the latest + RpcAssignmentDocEvent. Here they take one query between them. + + Finding which documents are queued reads each document's states, which this assumes + is free -- either prefetched by the caller, as prepare_document_table() does, or + already cached on the instances by an earlier get_state(). Absent both, that read + costs a query per document, which is the sort of per-row cost this exists to avoid. + """ + queued_ids = [ + d.pk + for d in docs + if d.get_state_slug("draft-rfceditor") in ("in_progress", "blocked") + ] + # Wrapped rather than assigned bare so the attribute stays callable, like the + # Document.rfc_editor_queue_status method it shadows. + for d in docs: + d.rfc_editor_queue_status = wrap_value(None) + if not queued_ids: + return + + # DISTINCT ON fetches only the newest event per document; a document that has moved + # through the queue has one for every status it has held. + for e in (RpcAssignmentDocEvent.objects + .filter(doc_id__in=queued_ids, type="changed_rpc_assignments") + .order_by("doc_id", "-time", "-id") + .distinct("doc_id")): + doc_dict[e.doc_id].rfc_editor_queue_status = wrap_value(e.assignments) + + def fill_in_person_caches(docs): """Seed the per-instance caches person_link and email_person_link read. @@ -288,6 +322,7 @@ def fill_in_document_table_attributes(docs, have_telechat_date=False): fill_in_document_relations(docs, doc_dict, doc_ids) fill_in_related_ipr(docs, doc_dict, doc_ids) + fill_in_rfc_editor_queue_status(docs, doc_dict, doc_ids) fill_in_person_caches(docs) if not have_telechat_date: diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 1472c808eb0..6ae100e6cb0 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -60,9 +60,9 @@ from ietf.doc.models import ( Document, DocHistory, DocEvent, BallotDocEvent, BallotType, ConsensusDocEvent, NewRevisionDocEvent, StoredObject, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent, IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS, DocumentActionHolder, DocumentAuthor, - RelatedDocument, RelatedDocHistory, RpcAssignmentDocEvent) + RelatedDocument, RelatedDocHistory, RpcActionHolderOpenEntry, RpcAssignmentDocEvent) from ietf.doc.tasks import investigate_fragment_task -from ietf.doc.utils import (augment_events_with_revision, +from ietf.doc.utils import (augment_events_with_revision, show_rpc_action_holder_comments, can_adopt_draft, can_unadopt_draft, get_chartering_type, get_tags_for_stream_id, needed_ballot_positions, nice_consensus, update_telechat, has_same_ballot, get_initial_notify, make_notify_changed_event, make_rev_history, default_consensus, @@ -213,6 +213,20 @@ def rfc_editor_queue_status(doc): event = doc.latest_event(RpcAssignmentDocEvent, type="changed_rpc_assignments") return event.assignments if event else None +def rpc_action_holders(doc): + """Open action holder entries the RPC has for this document + + Only entries naming a person the datatracker knows: an action held by a + body, or by the RPC tool's own placeholder person, is not something to show + here - the queue status already reports that the RPC is waiting on someone. + """ + return list( + RpcActionHolderOpenEntry.objects.filter( + document=doc, person__isnull=False + ).select_related("person").order_by("since_when") + ) + + def document_main(request, name, rev=None, document_html=False): doc = get_object_or_404(Document.objects.select_related(), name=name) @@ -380,7 +394,7 @@ def document_main(request, name, rev=None, document_html=False): has_errata=doc.pk and doc.tags.filter(slug="errata"), # doc.pk == None if using a fake_history_obj file_urls=file_urls, rfc_editor_state=doc.get_state("draft-rfceditor"), - rfc_editor_queue_status=rfc_editor_queue_status(doc), + rfc_editor_queue_status=doc.rfc_editor_queue_status(), iana_review_state=doc.get_state("draft-iana-review"), iana_action_state=doc.get_state("draft-iana-action"), iana_experts_state=doc.get_state("draft-iana-experts"), @@ -406,7 +420,8 @@ def document_main(request, name, rev=None, document_html=False): if isinstance(doc, Document): log.assertion('iesg_state', note="A document's 'draft-iesg' state should never be unset'. Failed for %s"%doc.name) iesg_state_slug = iesg_state.slug if iesg_state else None - iesg_state_summary = doc.friendly_state() + # Rendered in a row of its own labeled "IESG state", so not labeled again here. + iesg_state_summary = doc.friendly_state(label_iesg_state=False) irsg_state = doc.get_state("draft-stream-irtf") can_edit = has_role(request.user, ("Area Director", "Secretariat")) @@ -664,6 +679,11 @@ def document_main(request, name, rev=None, document_html=False): if html: css += Path(finders.find("ietf/css/document_html_txt.css")).read_text() + # Actions the RPC is waiting on, while the document is in its queue. + # The entries describe the document as it stands now, so a view of an + # earlier revision has none. + doc_rpc_action_holders = [] if snapshot else rpc_action_holders(doc) + return render(request, "doc/document_draft.html" if document_html is False else "doc/document_html.html", dict(doc=doc, document_html=document_html, @@ -724,8 +744,11 @@ def document_main(request, name, rev=None, document_html=False): iesg_state=iesg_state, iesg_state_summary=iesg_state_summary, rfc_editor_state=doc.get_state("draft-rfceditor"), - rfc_editor_queue_status=rfc_editor_queue_status(doc), + rfc_editor_queue_status=doc.rfc_editor_queue_status(), rfc_editor_auth48_url=auth48_url, + rpc_action_holders=doc_rpc_action_holders, + show_rpc_action_holder_comments=show_rpc_action_holder_comments( + request.user, doc_rpc_action_holders), iana_review_state=doc.get_state("draft-iana-review"), iana_action_state=doc.get_state("draft-iana-action"), iana_experts_state=doc.get_state("draft-iana-experts"), diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index 1968b133ce1..17eb4d40b47 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -249,7 +249,8 @@ def review_request(request, name, request_id): if review_req.doc.group: wg_chairs = [role.person for role in review_req.doc.group.role_set.filter(name__slug='chair')] - iesg_state_summary = review_req.doc.friendly_state() + # Rendered in a row of its own labeled "IESG document state", so not labeled again here. + iesg_state_summary = review_req.doc.friendly_state(label_iesg_state=False) history = list(review_req.history.all()) history += itertools.chain(*[list(r.history.all()) for r in review_req.reviewassignment_set.all()]) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 7cf84b52768..9da1aa04529 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -48,7 +48,7 @@ from django.conf import settings from django.core.cache import cache, caches from django.urls import reverse as urlreverse -from django.db.models import Model, Q +from django.db.models import Count, Model, Q from django.http import Http404, HttpResponseBadRequest, HttpResponse, HttpResponseRedirect, QueryDict from django.shortcuts import render from django.utils import timezone @@ -59,11 +59,14 @@ import debug # pyflakes:ignore from ietf.doc.models import ( Document, DocHistory, DocumentAuthor, RelatedDocument, - RfcAuthor, State, NewRevisionDocEvent, IESG_SUBSTATE_TAGS, + RfcAuthor, RpcActionHolderOpenEntry, State, NewRevisionDocEvent, IESG_SUBSTATE_TAGS, IESG_BALLOT_ACTIVE_STATES, IESG_STATCHG_CONFLREV_ACTIVE_STATES, IESG_CHARTER_ACTIVE_STATES ) from ietf.doc.fields import select2_id_doc_name_json -from ietf.doc.utils import augment_events_with_revision, needed_ballot_positions +from ietf.doc.utils import ( + augment_events_with_revision, + needed_ballot_positions, +) from ietf.group.models import Group from ietf.idindex.index import active_drafts_index_by_group from ietf.name.models import DocTagName, DocTypeName, StreamName @@ -703,10 +706,28 @@ def _state_to_doc_type(state): for dt in AD_WORKLOAD } + # Current state, not a trend: the open entries the RPC pushes are a + # snapshot, so there is no history to bucket the way the tables above do. + rpc_pending_counts = dict( + RpcActionHolderOpenEntry.objects.filter(person__in=ads) + .values_list("person") + .annotate(Count("id")) + ) + rpc_pending = [ + {"ad": ad, "count": rpc_pending_counts[ad.pk]} + for ad in ads + if rpc_pending_counts.get(ad.pk) + ] + return render( request, "doc/ad_list.html", - {"metadata": metadata, "data": data, "delta": days}, + { + "metadata": metadata, + "data": data, + "delta": days, + "rpc_pending": rpc_pending, + }, ) @@ -831,6 +852,13 @@ def sort_key(doc): if re.search(r"\bNeeds\s+\d+", iesg_ballot_summary): not_balloted_docs.append(doc) + # Actions the RPC is waiting on this AD for, from the publication queue. + rpc_action_holders = list( + RpcActionHolderOpenEntry.objects.filter(person=ad) + .select_related("document") + .order_by("since_when") + ) + return render( request, "doc/drafts_for_ad.html", @@ -840,6 +868,7 @@ def sort_key(doc): "ad": ad, "blocked_docs": blocked_docs, "not_balloted_docs": not_balloted_docs, + "rpc_action_holders": rpc_action_holders, }, ) diff --git a/ietf/group/models.py b/ietf/group/models.py index a7e3c6616e7..dfa710627e7 100644 --- a/ietf/group/models.py +++ b/ietf/group/models.py @@ -511,35 +511,3 @@ def notify_rfceditor_of_group_name_change(sender, instance=None, **kwargs): """ % (current.acronym, current.name, instance.name, ) send_mail_text(None, to=addr, frm=None, subject="Group '%s' name change"%instance.acronym, txt=msg) log.log("Sent notification email: %s: '%s' --> '%s' to %s" % (current.acronym, current.name, instance.name, addr)) - - -## Keep this code as a worked and tested example of sending signed notifies -## by HTTP POST. (superseded for this use case by email notification) -# url = settings.RFC_EDITOR_GROUP_NOTIFICATION_URL -# if url and instance.name != current.name: -# data = { -# 'acronym': current.acronym, -# 'old_name': current.name, -# 'name': instance.name, -# } -# # Build signed data -# key = jwk.JWK() -# key.import_from_pem(settings.API_PRIVATE_KEY_PEM) -# payload = json.dumps(data) -# jwstoken = jws.JWS(payload.encode('utf-8')) -# jwstoken.add_signature(key, None, -# json_encode({"alg": settings.API_KEY_TYPE}), -# json_encode({"kid": key.thumbprint()})) -# sig = jwstoken.serialize() -# # Send signed data -# response = requests.post(url, data = { 'jws': sig, }) -# log.log("Sent notify: %s: '%s' --> '%s' to %s, result code %s" % -# (current.acronym, current.name, instance.name, url, response.status_code)) -# # Verify locally, to make sure we've got things right -# key = jwk.JWK() -# key.import_from_pem(settings.API_PUBLIC_KEY_PEM) -# jwstoken = jws.JWS() -# jwstoken.deserialize(sig) -# jwstoken.verify(key) -# log.assertion('payload == jwstoken.payload') - diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 4e0096b1859..c93525bd7e4 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -2147,7 +2147,86 @@ def test_meeting_info(self): self.assertEqual(response.status_code, 200) q = PyQuery(response.content) self.assertFalse(q('#inprogressmeets')) - + + +class PendingInterimMeetingTests(TestCase): + """Tests for the pending-interim warning on a group's meetings list. + + The meetings page shows a ``#pending_warning`` banner when the group has an + interim meeting that is either awaiting approval (session status ``apprw``) + or approved but not yet announced (session status ``scheda``). See + ietf.meeting.helpers.has_pending_interim. + """ + + def _meetings_page(self, group): + url = urlreverse('ietf.group.views.meetings', kwargs={'acronym': group.acronym}) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + return PyQuery(response.content) + + def test_pending_approval_interim_shows_warning(self): + """An interim awaiting approval (apprw) triggers the warning.""" + group = GroupFactory.create(type_id='wg') + SessionFactory.create( + meeting__type_id="interim", + meeting__date=date_today() + datetime.timedelta(days=30), + group=group, + status_id="apprw", + ) + q = self._meetings_page(group) + warning = q('#pending_warning') + self.assertTrue(warning) + # The warning links to both the pending-approval and to-be-announced views + # and names the group. + self.assertIn(urlreverse('ietf.meeting.views.interim_pending'), + [a.attrib['href'] for a in warning.find('a')]) + self.assertIn(urlreverse('ietf.meeting.views.interim_announce'), + [a.attrib['href'] for a in warning.find('a')]) + self.assertIn(group.acronym, warning.text()) + + def test_to_be_announced_interim_shows_warning(self): + """An approved-but-unannounced interim (scheda) triggers the warning.""" + group = GroupFactory.create(type_id='wg') + SessionFactory.create( + meeting__type_id="interim", + meeting__date=date_today() + datetime.timedelta(days=30), + group=group, + status_id="scheda", + ) + q = self._meetings_page(group) + self.assertTrue(q('#pending_warning')) + + def test_scheduled_interim_shows_no_warning(self): + """A fully scheduled interim (sched) does not trigger the warning.""" + group = GroupFactory.create(type_id='wg') + SessionFactory.create( + meeting__type_id='interim', + meeting__date=date_today() + datetime.timedelta(days=30), + group=group, + status_id='sched', + ) + q = self._meetings_page(group) + self.assertFalse(q('#pending_warning')) + + def test_no_interim_meetings_shows_no_warning(self): + """A group with no interim meetings does not trigger the warning.""" + group = GroupFactory.create(type_id='wg') + q = self._meetings_page(group) + self.assertFalse(q('#pending_warning')) + + def test_pending_interim_for_other_group_not_shown(self): + """A pending interim belonging to another group must not warn on this group.""" + group = GroupFactory.create(type_id='wg') + other = GroupFactory.create(type_id='wg') + SessionFactory.create( + meeting__type_id='interim', + meeting__date=date_today() + datetime.timedelta(days=30), + group=other, + status_id='apprw', + ) + q = self._meetings_page(group) + self.assertFalse(q('#pending_warning')) + class StatusUpdateTests(TestCase): diff --git a/ietf/group/views.py b/ietf/group/views.py index 8561a5059fc..eddd2499e06 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -92,7 +92,7 @@ # from ietf.ietfauth.utils import has_role, is_authorized_in_group from ietf.mailtrigger.utils import gather_relevant_expansions -from ietf.meeting.helpers import get_meeting +from ietf.meeting.helpers import get_meeting, has_pending_interim from ietf.meeting.models import ImportantDate, SchedTimeSessAssignment, SchedulingEvent from ietf.meeting.utils import group_sessions from ietf.name.models import GroupTypeName, StreamName @@ -953,6 +953,8 @@ def meetings(request, acronym, group_type=None): future, in_progress, recent, past = group_sessions(sessions) + pending_interims_flag = has_pending_interim(group.acronym) + can_edit = group.has_role(request.user, group.features.groupman_roles) can_always_edit = has_role(request.user, ["Secretariat", "Area Director"]) @@ -996,6 +998,7 @@ def meetings(request, acronym, group_type=None): "can_edit": can_edit, "can_always_edit": can_always_edit, "cal_actions": cal_actions, + "pending_interims": pending_interims_flag, }, ), ) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index af04ad06fd5..d65d923ad9c 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -383,7 +383,9 @@ def agenda_documents(request): docs_by_date = dict((d, []) for d in dates) docs = Document.objects.filter(docevent__telechatdocevent__telechat_date__in=dates).distinct() docs = docs.select_related("ad", "std_level", "intended_std_level", "group", "stream", "shepherd", ) - # No prefetch-related -- turns out not to be worth it + # The states and tags the status column of every row reads, each of which otherwise + # costs a query per document. + docs = docs.prefetch_related("states__type", "tags") fill_in_telechat_date(docs) for doc in docs: diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py index a77e5bd5d58..9b793bf6ccb 100644 --- a/ietf/ietfauth/tests.py +++ b/ietf/ietfauth/tests.py @@ -34,7 +34,13 @@ from ietf.ietfauth.utils import has_role from ietf.meeting.factories import MeetingFactory, RegistrationFactory, RegistrationTicketFactory from ietf.nomcom.factories import NomComFactory -from ietf.person.factories import PersonFactory, EmailFactory, UserFactory, PersonalApiKeyFactory +from ietf.person.factories import ( + PersonFactory, + EmailFactory, + UserFactory, + PersonalApiKeyFactory, + PersonUUIDFactory, +) from ietf.person.models import Person, Email from ietf.person.tasks import send_apikey_usage_emails_task from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory @@ -657,7 +663,9 @@ def test_change_password(self): ) user.set_password(VALID_PASSWORD) user.save() - p = Person.objects.create(name="Some One", ascii="Some One", user=user) + p = PersonFactory( + user=user, name="Some One", ascii="Some One", default_emails=False + ) Email.objects.create(address=user.username, person=p, origin=user.username) # log in @@ -758,7 +766,9 @@ def test_change_username(self): ) user.set_password(VALID_PASSWORD) user.save() - p = Person.objects.create(name="Some One", ascii="Some One", user=user) + p = PersonFactory( + user=user, name="Some One", ascii="Some One", default_emails=False + ) Email.objects.create(address=user.username, person=p, origin=user.username) Email.objects.create( address="othername@example.org", person=p, origin=user.username @@ -1162,7 +1172,16 @@ def test_oidc_code_auth(self): session["nonce"] = rndstr() args = { "response_type": "code", - "scope": ['openid', 'profile', 'email', 'roles', 'registration', 'dots', 'pronouns' ], + "scope": [ + "openid", + "profile", + "email", + "roles", + "registration", + "dots", + "pronouns", + "datatracker_uuid", + ], "nonce": session["nonce"], "redirect_uri": redirect_uris[0], "state": session["state"] @@ -1207,6 +1226,10 @@ def test_oidc_code_auth(self): self.assertIn(key, access_token_info) for key in ['iss', 'sub', 'aud', 'exp', 'iat', 'auth_time', 'nonce', 'at_hash']: self.assertIn(key, access_token_info['id_token']) + # Custom claims are served from userinfo, not the id_token. This guards + # against an accidental OIDC_IDTOKEN_INCLUDE_CLAIMS flip. + for key in ["datatracker_uuid", "datatracker_prior_uuids"]: + self.assertNotIn(key, access_token_info["id_token"]) # Get userinfo, check keys present, most common scenario userinfo = client.do_user_info_request(state=params["state"], scope=args['scope']) @@ -1218,6 +1241,18 @@ def test_oidc_code_auth(self): self.assertNotIn('hackathon_onsite', set(userinfo['reg_type'].split())) self.assertIn(active_group.acronym, [i[1] for i in userinfo['roles']]) self.assertNotIn(closed_group.acronym, [i[1] for i in userinfo['roles']]) + self.assertEqual(userinfo['datatracker_uuid'], str(person.primary_uuid)) + # Present and empty, not absent, for a Person that has never been merged + self.assertIn("datatracker_prior_uuids", userinfo) + self.assertEqual(userinfo["datatracker_prior_uuids"], []) + + # A UUID absorbed by a merge shows up in the prior list + absorbed = PersonUUIDFactory(person=person) + userinfo = client.do_user_info_request( + state=params["state"], scope=args["scope"] + ) + self.assertEqual(userinfo["datatracker_uuid"], str(person.primary_uuid)) + self.assertEqual(userinfo["datatracker_prior_uuids"], [str(absorbed.uuid)]) # Create a registration, with only email, no person (rare if at all) reg_person.delete() diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index 30d51cddd09..fcce43dc2dc 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -341,6 +341,29 @@ def scope_dots(self): dots = get_dots(self.user.person) return { 'dots': dots } + info_datatracker_uuid = ( + "Datatracker person identifiers", + ( + "Access to the stable identifier the datatracker uses for you when " + "telling other systems who you are, and to any identifiers it used for " + "you before they were superseded." + ), + ) + + def scope_datatracker_uuid(self): + # One scope for both claims: there is no case for granting the current + # identifier without the superseded ones that resolve to it. + person = self.user.person + return { + # An empty string is dropped by ScopeClaims._clean_dic, so an inconsistent + # Person yields an absent claim rather than a bogus identifier. + "datatracker_uuid": str(person.primary_uuid or ""), + # An empty list survives _clean_dic, so this claim is present-and-empty + # rather than absent for a Person that has never been merged. It holds only + # superseded identifiers - the current one is datatracker_uuid. + "datatracker_prior_uuids": [str(u) for u in person.prior_uuids], + } + def scope_pronouns(self): return { 'pronouns': self.user.person.pronouns() } diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py index b5256b14f8b..2e32dfd5d3c 100644 --- a/ietf/ietfauth/views.py +++ b/ietf/ietfauth/views.py @@ -53,7 +53,7 @@ from django.contrib.auth.views import LoginView from django.contrib.sites.models import Site from django.core.exceptions import ObjectDoesNotExist, ValidationError -from django.db import IntegrityError +from django.db import IntegrityError, transaction from django.urls import reverse as urlreverse from django.http import Http404, HttpResponseRedirect, HttpResponseForbidden from django.shortcuts import render, redirect, get_object_or_404 @@ -69,6 +69,7 @@ from ietf.name.models import ExtResourceName from ietf.nomcom.models import NomCom from ietf.person.models import Person, Email, Alias, PersonalApiKey, PERSON_API_KEY_VALUES +from ietf.person.utils import assign_primary_uuid from ietf.review.models import ReviewerSettings, ReviewWish, ReviewAssignment from ietf.review.utils import unavailable_periods_to_list, get_default_filter_re from ietf.doc.fields import SearchableDocumentField @@ -232,12 +233,15 @@ def confirm_account(request, auth): if not person: name = form.cleaned_data["name"] ascii = form.cleaned_data["ascii"] - person = Person.objects.create(user=user, - name=name, - ascii=ascii) - for name in set([ person.name, person.ascii, person.plain_name(), person.plain_ascii(), ]): - Alias.objects.create(person=person, name=name) + # Atomic so a Person is never left without the primary UUID that + # external systems need to name them by. + with transaction.atomic(): + person = Person.objects.create(user=user, name=name, ascii=ascii) + assign_primary_uuid(person) + + for name in set([ person.name, person.ascii, person.plain_name(), person.plain_ascii(), ]): + Alias.objects.create(person=person, name=name) if not email_obj: email_obj = Email.objects.create(address=email, person=person, origin=user.username) @@ -756,9 +760,9 @@ def clean(self): except ValueError: self.add_error( "password", - 'Your password has been cleared because of possible password leakage. ' - 'Please use the "Forgot your password?" button below to set a new password ' - 'for your account.', + 'Your password has been reset due to inactivity. Please use the ' + '"Forgot your password?" button below to set a new password for ' + 'your account.', ) return super().clean() diff --git a/ietf/meeting/api.py b/ietf/meeting/api.py new file mode 100644 index 00000000000..31042620913 --- /dev/null +++ b/ietf/meeting/api.py @@ -0,0 +1,61 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from django.db.models import IntegerField +from django.db.models.functions import Cast +from rest_framework import generics + +from ietf.meeting.models import Meeting +from ietf.meeting.serializers import PersonAttendedMeetingsSerializer +from ietf.person.models import Person + + +class MeetingsAttendedByEmail(generics.RetrieveAPIView): + """List meetings attended by a person identified by email address + + Requires API key authentication + """ + + queryset = Person.objects.all() + lookup_field = "email__address" + lookup_url_kwarg = "email" + serializer_class = PersonAttendedMeetingsSerializer + api_key_endpoint = "ietf.api.meeting.registration.attended" + + def get_object(self): + person = super().get_object() + assert isinstance(person, Person) + person.attended_registrations = self._attended_registrations(person) + return person + + @staticmethod + def _attended_registrations(person: Person): + meetings_with_data = ( + Meeting.objects.filter(type="ietf") + .annotate(number_as_int=Cast("number", output_field=IntegerField())) + .exclude(number_as_int__lt=110) + .values_list("pk") + ) + has_attended_record = ( + person.attended_set.filter( + session__meeting_id__in=meetings_with_data, + session__meeting__type="ietf", + ) + .values_list("session__meeting__id", flat=True) + .distinct() + ) + return sorted( + [ + reg + for reg in ( + person.registration_set.onsite_or_remote() + .with_plenary_ticket_details() + .filter(meeting_id__in=meetings_with_data) + .select_related("meeting") + ) + if ( + reg.attended + or reg.checkedin + or reg.meeting_id in has_attended_record + ) + ], + key=lambda reg: reg.meeting.date, + ) diff --git a/ietf/meeting/helpers.py b/ietf/meeting/helpers.py index 39d271ae6b9..568af7422c2 100644 --- a/ietf/meeting/helpers.py +++ b/ietf/meeting/helpers.py @@ -1,7 +1,4 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved -# -*- coding: utf-8 -*- - - +# Copyright The IETF Trust 2013-2026, All Rights Reserved from collections import defaultdict import datetime import io @@ -28,12 +25,20 @@ from ietf.mailtrigger.utils import gather_address_lists from ietf.person.models import Person from ietf.meeting.models import Meeting, Schedule, TimeSlot, SchedTimeSessAssignment, ImportantDate, SchedulingEvent, Session -from ietf.meeting.utils import session_requested_by, add_event_info_to_session_qs +from ietf.meeting.utils import session_requested_by, add_event_info_to_session_qs, data_for_meetings_overview from ietf.name.models import ImportantDateName, SessionPurposeName from ietf.utils import log, meetecho from ietf.utils.mail import send_mail from ietf.utils.pipe import pipe from ietf.utils.text import xslugify +from ietf.utils.timezone import date_today + +# Ignore meetings older than this when querying for pending interims. It's expected that +# every interim should be out of the pending (apprw / scheda) states well before the +# scheduled date. Look back a few weeks so that a meeting that is somehow left in a +# pending state does not fall off the interface until people have had time to notice. +# +PENDING_INTERIM_MAX_LOOKBACK = datetime.timedelta(days=28) def get_meeting(num=None, type_in=('ietf',), days=28): @@ -773,6 +778,7 @@ def can_edit_interim_request(meeting, user): def can_request_interim_meeting(user): return can_manage_some_groups(user) + def can_view_interim_request(meeting, user): '''Returns True if the user can see the pending interim request in the pending interim view''' if meeting.type.slug != 'interim': @@ -855,6 +861,35 @@ def get_earliest_session_date(formset): def is_interim_meeting_approved(meeting): return add_event_info_to_session_qs(meeting.session_set.all()).first().current_status == 'apprw' + +def has_pending_interim(acronym): + """Check whether group identified by acronym has a pending interim + + This function takes a group acronym and returns True if that group has at least + one pending interim meeting request or a to-be-announced interim request. + """ + rv = False + possible_meetings = Meeting.objects.filter( + type="interim", date__gte=date_today() - PENDING_INTERIM_MAX_LOOKBACK + ) + pending = data_for_meetings_overview(possible_meetings, interim_status="apprw") + for m in pending: + if m.responsible_group.acronym == acronym: + rv = True + break + + if not rv: + to_be_announced = data_for_meetings_overview( + possible_meetings, interim_status="scheda" + ) + for m in to_be_announced: + if m.responsible_group.acronym == acronym: + rv = True + break + + return rv + + def get_next_interim_number(acronym,date): ''' This function takes a group acronym and date object and returns the next number @@ -869,6 +904,7 @@ def get_next_interim_number(acronym,date): serial = 0 return "%s%02d" % (base, serial+1) + def get_next_agenda_name(meeting): """Returns the next name to use for an agenda document for *meeting*""" group = meeting.session_set.first().group @@ -941,6 +977,7 @@ def send_interim_approval_request(meetings): context, cc=cc_list) + def send_interim_approval(user, meeting): """Send an email to chairs and whoever initiated the action that resulted in approval that an interim is approved""" first_session = meeting.session_set.first() @@ -961,6 +998,7 @@ def send_interim_approval(user, meeting): context, cc=cc_list) + def send_interim_announcement_request(meeting): """Sends an email to the secretariat that an interim meeting is ready for announcement, includes the link to send the official announcement""" @@ -981,6 +1019,7 @@ def send_interim_announcement_request(meeting): context, cc_list) + def send_interim_meeting_cancellation_notice(meeting): """Sends an email that a scheduled interim meeting has been cancelled.""" session = meeting.session_set.first() @@ -1182,12 +1221,14 @@ def update_interim_session_assignment(form): session=session, schedule=meeting.schedule) + def populate_important_dates(meeting): assert ImportantDate.objects.filter(meeting=meeting).exists() is False assert meeting.type_id=='ietf' for datename in ImportantDateName.objects.filter(used=True): ImportantDate.objects.create(meeting=meeting,name=datename,date=meeting.date+datetime.timedelta(days=datename.default_offset_days)) + def update_important_dates(meeting): assert meeting.type_id=='ietf' for datename in ImportantDateName.objects.filter(used=True): diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index bf0c4dabd12..42b81b6a973 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -6,6 +6,8 @@ import datetime import io import os +from functools import cached_property + import pytz import random import re @@ -19,7 +21,7 @@ from django.core.validators import MinValueValidator, RegexValidator from django.db import models -from django.db.models import Max, Subquery, OuterRef, TextField, Value, Q +from django.db.models import Max, Subquery, OuterRef, TextField, Value, Q, Case, When from django.db.models.functions import Coalesce from django.conf import settings from django.urls import reverse as urlreverse @@ -1564,15 +1566,53 @@ def __str__(self): return f'{self.person} at {self.session}' -class RegistrationManager(models.Manager): +class RegistrationQuerySet(models.QuerySet): def onsite(self): - return self.get_queryset().filter(tickets__attendance_type__slug='onsite') + """Only registrations with at least one `onsite` ticket + + Includes any ticket type. In particular, may be a hackathon-only registration. + """ + return self.filter(tickets__attendance_type__slug="onsite") def remote(self): - return self.get_queryset().filter(tickets__attendance_type__slug='remote').exclude(tickets__attendance_type__slug='onsite') + """Only registrations with no `onsite` and at least one `remote` ticket + + Includes any ticket type. In particular, may be a hackathon-only registration. + """ + return ( + self.filter(tickets__attendance_type__slug="remote") + .exclude(tickets__attendance_type__slug="onsite") + ) + + def onsite_or_remote(self): + """Registrations that were onsite or remote + + I.e., was a registration for the plenary meeting, not e.g. hackathon-only. + """ + return self.filter( + tickets__attendance_type__slug__in=["onsite", "remote"] + ).distinct() + + def with_plenary_ticket_details(self): + """Annotate with ticket details + + Adds private annotations accessible via @properties + """ + most_representative = RegistrationTicket.objects.filter( + registration=OuterRef("pk") + ).order_by_most_representative() + return self.annotate( + _attendance_type=Subquery( + most_representative.values("attendance_type")[:1] + ), + _ticket_type=Subquery(most_representative.values("ticket_type")[:1]), + ) + class Registration(models.Model): """Registration attendee records from the IETF registration system""" + objects = RegistrationQuerySet.as_manager() # custom manager + meeting = ForeignKey(Meeting) first_name = models.CharField(max_length=255) last_name = models.CharField(max_length=255) @@ -1586,21 +1626,74 @@ class Registration(models.Model): # checkedin indicates that the badge was picked up checkedin = models.BooleanField(default=False) - # custom manager - objects = RegistrationManager() - def __str__(self): return "{} {}".format(self.first_name, self.last_name) + @cached_property + def _plenary_ticket(self): + return self.tickets.order_by_most_representative().first() + @property - def attendance_type(self): - if self.tickets.filter(attendance_type__slug='onsite').exists(): - return 'onsite' - elif self.tickets.filter(attendance_type__slug='remote').exists(): - return 'remote' - return None + def plenary_attendance_type(self): + """Attendance type for the plenary meeting + + Attendance type for the plenary meeting. Ignores hackathon/anrw or any other + types of registration that are tracked through tickets. + """ + if hasattr(self, "_attendance_type"): + return self._attendance_type # added via with_plenary_ticket_details() + return ( + self._plenary_ticket.attendance_type_id if self._plenary_ticket else None + ) + + @property + def plenary_ticket_type(self): + """Ticket type for the plenary meeting + + Ticket type for the plenary meeting. Ignores hackathon/anrw or any other types + of registration that are tracked through tickets. + """ + if hasattr(self, "_ticket_type"): + return self._ticket_type # added via with_plenary_ticket_details() + return ( + self._plenary_ticket.ticket_type_id if self._plenary_ticket else None + ) + + +class RegistrationTicketQuerySet(models.QuerySet): + def order_by_most_representative(self): + """Order-by clause for representative tickets for plenary attendance + + Filters out tickets not applicable to the plenary IETF meeting + """ + # ordered lists of interesting types + interesting_attendance_types = ["onsite", "remote"] + interesting_ticket_types = ["student", "week_pass", "one_day", "unknown"] + case_ranking_attendance_types = Case( + *[ + When(attendance_type=att_type, then=index) + for index, att_type in enumerate(interesting_attendance_types) + ] + ) + case_ranking_ticket_types = Case( + *[ + When(ticket_type=tkt_type, then=index) + for index, tkt_type in enumerate(interesting_ticket_types) + ] + ) + return self.filter( + attendance_type__in=interesting_attendance_types, + ticket_type__in=interesting_ticket_types, + ).order_by( + case_ranking_attendance_types, + case_ranking_ticket_types, + "pk", # deterministic tie-break + ) + class RegistrationTicket(models.Model): + objects = RegistrationTicketQuerySet.as_manager() # custom manager + registration = ForeignKey(Registration, related_name='tickets') attendance_type = ForeignKey(AttendanceTypeName, on_delete=models.PROTECT) ticket_type = ForeignKey(RegistrationTicketTypeName, on_delete=models.PROTECT) diff --git a/ietf/meeting/serializers.py b/ietf/meeting/serializers.py new file mode 100644 index 00000000000..70b9ee12cae --- /dev/null +++ b/ietf/meeting/serializers.py @@ -0,0 +1,28 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from rest_framework import serializers + +from ietf.meeting.models import Registration + + +class AttendedMeetingSerializer(serializers.ModelSerializer): + """Serialize a plenary meeting attendance record""" + meeting = serializers.SlugRelatedField(slug_field="number", read_only=True) + attendance_type = serializers.CharField( + source="plenary_attendance_type", read_only=True + ) + ticket_type = serializers.CharField( + source="plenary_ticket_type", read_only=True + ) + + class Meta: + model = Registration + fields = [ + "meeting", + "attendance_type", + "ticket_type", + ] + + +class PersonAttendedMeetingsSerializer(serializers.Serializer): + attended = AttendedMeetingSerializer(source="attended_registrations", many=True) diff --git a/ietf/meeting/tests_api.py b/ietf/meeting/tests_api.py new file mode 100644 index 00000000000..062ab07e4d1 --- /dev/null +++ b/ietf/meeting/tests_api.py @@ -0,0 +1,166 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from unittest.mock import PropertyMock, patch + +from django.test import override_settings +from django.urls import reverse as urlreverse + +from ietf.meeting.factories import ( + AttendedFactory, + MeetingFactory, + RegistrationFactory, +) +from ietf.meeting.models import Registration +from ietf.person.factories import EmailFactory, PersonFactory +from ietf.utils.test_utils import TestCase + + +@override_settings( + APP_API_TOKENS={"ietf.api.meeting.registration.attended": "valid-token"} +) +class MeetingsAttendedByEmailTests(TestCase): + VIEWNAME = "ietf.api.meeting.registration.attended" + + def setUp(self): + super().setUp() + self.person = PersonFactory() + + def attended_for(self, email=None): + """Retrieve the "attended" list for an email address""" + url = urlreverse( + self.VIEWNAME, kwargs={"email": email or self.person.email_address()} + ) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + return r.json()["attended"] + + @staticmethod + def ietf_meeting(number): + return MeetingFactory(type_id="ietf", number=number, populate_schedule=False) + + def test_endpoint_is_plumbed(self): + url = urlreverse(self.VIEWNAME, kwargs={"email": self.person.email_address()}) + # bad/missing API keys + r = self.client.get(url) + self.assertEqual(r.status_code, 403, "should require api key") + r = self.client.get(url, headers={"X-Api-Key": "invalid-token"}) + self.assertEqual(r.status_code, 403, "should require valid api key") + + # valid request + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200, "should accept valid api key") + self.assertEqual(r.json(), {"attended": []}) + + # nonexistent person + self.person.email_set.update(person=None) # detach email + self.person.delete() + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 404, "404 for no such Person") + + def test_lists_registrations_with_attendance_evidence(self): + """Registrations for IETF 110+ that were attended are listed + + Attendance is indicated by the attended flag, the checkedin flag, or an Attended + record. These records only had their modern form starting at IETF 110. + """ + attended = self.ietf_meeting("118") + RegistrationFactory(meeting=attended, person=self.person, attended=True) + checkedin = self.ietf_meeting("119") + RegistrationFactory(meeting=checkedin, person=self.person, checkedin=True) + session_attended = self.ietf_meeting("120") + RegistrationFactory(meeting=session_attended, person=self.person) + AttendedFactory( + session__meeting=session_attended, + session__add_to_schedule=False, # these meetings have no timeslots + person=self.person, + ) + + # another person's registrations for the same meetings must not appear + other_person = PersonFactory() + for meeting in [attended, checkedin, session_attended]: + RegistrationFactory(meeting=meeting, person=other_person, attended=True) + + self.assertCountEqual( + [entry["meeting"] for entry in self.attended_for()], ["118", "119", "120"] + ) + + def test_excludes_registrations_without_attendance_evidence(self): + """Only attended IETF meetings from 110 onwards are reported""" + RegistrationFactory(meeting=self.ietf_meeting("118"), person=self.person) + RegistrationFactory( + meeting=self.ietf_meeting("109"), person=self.person, attended=True + ) + RegistrationFactory( + meeting=MeetingFactory(type_id="interim", populate_schedule=False), + person=self.person, + attended=True, + ) + + self.assertEqual(self.attended_for(), []) + + def test_reports_attendance_and_ticket_type(self): + """Ticket details are taken from the Registration's plenary_* properties""" + RegistrationFactory( + meeting=self.ietf_meeting("118"), person=self.person, attended=True + ) + + with ( + patch.object( + Registration, "plenary_attendance_type", new_callable=PropertyMock + ) as attendance_type, + patch.object( + Registration, "plenary_ticket_type", new_callable=PropertyMock + ) as ticket_type, + ): + attendance_type.return_value = "onsite" + ticket_type.return_value = "week_pass" + self.assertEqual( + self.attended_for(), + [ + { + "meeting": "118", + "attendance_type": "onsite", + "ticket_type": "week_pass", + } + ], + ) + + # absent ticket details are reported as null + attendance_type.return_value = None + ticket_type.return_value = None + self.assertEqual( + self.attended_for(), + [{"meeting": "118", "attendance_type": None, "ticket_type": None}], + ) + + def test_finds_person_by_any_email_address(self): + """Any of the Person's email addresses identifies them""" + RegistrationFactory( + meeting=self.ietf_meeting("118"), person=self.person, attended=True + ) + secondary_email = EmailFactory(person=self.person) + + by_secondary = self.attended_for(email=secondary_email.address) + self.assertEqual([entry["meeting"] for entry in by_secondary], ["118"]) + self.assertEqual(by_secondary, self.attended_for()) + + def test_excludes_non_plenary_registrations(self): + """Registrations without an onsite or remote plenary ticket are not reported""" + RegistrationFactory( + meeting=self.ietf_meeting("118"), + person=self.person, + attended=True, + with_ticket={"attendance_type_id": "onsite", "ticket_type_id": "week_pass"}, + ) + RegistrationFactory( + meeting=self.ietf_meeting("119"), + person=self.person, + attended=True, + with_ticket={ + "attendance_type_id": "hackathon_remote", + "ticket_type_id": "unknown", + }, + ) + + attended = self.attended_for() + self.assertEqual([entry["meeting"] for entry in attended], ["118"]) + self.assertEqual({entry["attendance_type"] for entry in attended}, {"onsite"}) diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py index 0b9ce3f607f..4c18b3633c8 100644 --- a/ietf/meeting/tests_models.py +++ b/ietf/meeting/tests_models.py @@ -17,8 +17,9 @@ AttendedFactory, SessionPresentationFactory, ) -from ietf.meeting.factories import RegistrationFactory -from ietf.meeting.models import Session +from ietf.meeting.factories import RegistrationFactory, RegistrationTicketFactory +from ietf.meeting.models import Registration, RegistrationTicket, Session +from ietf.name.models import RegistrationTicketTypeName, AttendanceTypeName from ietf.utils.test_utils import TestCase from ietf.utils.timezone import date_today, datetime_today @@ -253,7 +254,7 @@ def test_chat_archive_url(self): def test_chat_room_name(self): session = SessionFactory(group__acronym="xyzzy") - self.assertEqual(session.chat_room_name(), "xyzzy") + self.assertEqual(session.chat_room_name(), "xyzzy") session.type_id = "plenary" self.assertEqual(session.chat_room_name(), "plenary") session.chat_room = "fnord" @@ -323,3 +324,231 @@ def test_session_recording_url_label_interim(self): f"IETF-ACRO-{session_time:%Y%m%d-%H%M}", # n.b., time in label is UTC session._session_recording_url_label(), ) + + +class RegistrationTests(TestCase): + def setUp(self): + super().setUp() + self.meeting = MeetingFactory(type_id="ietf") + + def create_registration(self, tickets): + """Create a Registration with the given (attendance, ticket) type pairs""" + registration = RegistrationFactory(meeting=self.meeting, with_ticket=False) + for attendance_type_id, ticket_type_id in tickets: + RegistrationTicketFactory( + registration=registration, + attendance_type_id=attendance_type_id, + ticket_type_id=ticket_type_id, + ) + return registration + + def test_onsite(self): + expected_onsite_pks = [ + self.create_registration([("onsite", ticket_type.pk)]).pk + for ticket_type in RegistrationTicketTypeName.objects.filter(used=True) + ] + for attendance_type in AttendanceTypeName.objects.filter(used=True): + if attendance_type.pk == "onsite": + continue # we already have one + for ticket_type in RegistrationTicketTypeName.objects.filter(used=True): + self.create_registration([(attendance_type.pk, ticket_type.pk)]) + + self.assertCountEqual( + Registration.objects.onsite().values_list("pk", flat=True), + expected_onsite_pks, + ) + + def test_remote(self): + expected_remote_pks = [ + self.create_registration([("remote", ticket_type.pk)]).pk + for ticket_type in RegistrationTicketTypeName.objects.filter(used=True) + ] + for attendance_type in AttendanceTypeName.objects.filter(used=True): + if attendance_type.pk == "remote": + continue # we already have one + for ticket_type in RegistrationTicketTypeName.objects.filter(used=True): + self.create_registration([(attendance_type.pk, ticket_type.pk)]) + + self.assertCountEqual( + Registration.objects.remote().values_list("pk", flat=True), + expected_remote_pks, + ) + + def test_onsite_or_remote(self): + expected_onsite_pks = [ + self.create_registration([("onsite", ticket_type.pk)]).pk + for ticket_type in RegistrationTicketTypeName.objects.filter(used=True) + ] + expected_remote_pks = [ + self.create_registration([("remote", ticket_type.pk)]).pk + for ticket_type in RegistrationTicketTypeName.objects.filter(used=True) + ] + for attendance_type in AttendanceTypeName.objects.filter(used=True): + if attendance_type.pk in ["onsite", "remote"]: + continue # we already have one + for ticket_type in RegistrationTicketTypeName.objects.filter(used=True): + self.create_registration([(attendance_type.pk, ticket_type.pk)]) + # create a remote ticket for an onsite registration to probe whether this + # leads to duplicate records + RegistrationTicketFactory( + # arbitrary registration + registration=(Registration.objects.get(pk=expected_onsite_pks[0])), + attendance_type_id="remote", + ticket_type_id="week_pass", + ) + self.assertCountEqual( + Registration.objects.onsite_or_remote().values_list("pk", flat=True), + expected_onsite_pks + expected_remote_pks, + ) + + # A ticket counts toward the plenary meeting only if both its attendance type + # and its ticket type are applicable. Applicable tickets are ranked by + # attendance type (onsite, then remote), then by ticket type (student, + # week_pass, one_day, then unknown), then by pk. + # + # Each case is a label, the tickets to create as (attendance_type_id, + # ticket_type_id) pairs, and the expected plenary_attendance_type and + # plenary_ticket_type. + PLENARY_TICKET_CASES = [ + ("no tickets at all", [], None, None), + ("a single plenary ticket", [("onsite", "week_pass")], "onsite", "week_pass"), + ( + "onsite outranks remote", + [("remote", "week_pass"), ("onsite", "week_pass")], + "onsite", + "week_pass", + ), + ( + "attendance type outranks ticket type", + [("remote", "student"), ("onsite", "one_day")], + "onsite", + "one_day", + ), + ( + "student outranks one_day", + [("onsite", "one_day"), ("onsite", "student")], + "onsite", + "student", + ), + ( + "unknown ticket type ranks last", + [("onsite", "unknown"), ("onsite", "one_day")], + "onsite", + "one_day", + ), + ( + "non-plenary attendance type is ignored", + [("hackathon_onsite", "hackathon_only")], + None, + None, + ), + ( + "non-plenary ticket type disqualifies its ticket entirely", + [("onsite", "hackathon_combo"), ("remote", "week_pass")], + "remote", + "week_pass", + ), + ( + "unknown is a plenary ticket type but not an attendance type", + [("unknown", "week_pass")], + None, + None, + ), + ] + + def test_plenary_ticket_details(self): + """The plenary_* properties report the most representative ticket""" + for label, tickets, attendance_type, ticket_type in self.PLENARY_TICKET_CASES: + with self.subTest(label): + registration = self.create_registration(tickets) + # refetch so the properties cannot see state left by the factories + registration = Registration.objects.get(pk=registration.pk) + self.assertEqual(registration.plenary_attendance_type, attendance_type) + self.assertEqual(registration.plenary_ticket_type, ticket_type) + + def test_plenary_ticket_details_annotated(self): + """The annotations agree with the properties + + The properties return the annotations when they are present, so the two + must rank tickets identically. + """ + for label, tickets, attendance_type, ticket_type in self.PLENARY_TICKET_CASES: + with self.subTest(label): + registration = self.create_registration(tickets) + annotated = Registration.objects.with_plenary_ticket_details().get( + pk=registration.pk + ) + self.assertEqual(annotated._attendance_type, attendance_type) + self.assertEqual(annotated._ticket_type, ticket_type) + self.assertEqual(annotated.plenary_attendance_type, attendance_type) + self.assertEqual(annotated.plenary_ticket_type, ticket_type) + + def test_plenary_ticket_details_tie_break(self): + """Tickets that rank equally are resolved by pk + + The annotations cannot distinguish these tickets - they expose only the + chosen ticket's type slugs, which are identical when tickets tie. + """ + registration = self.create_registration( + [("onsite", "week_pass"), ("onsite", "week_pass")] + ) + first, second = registration.tickets.order_by("pk") + self.assertEqual( + list(registration.tickets.order_by_most_representative()), [first, second] + ) + self.assertEqual( + Registration.objects.get(pk=registration.pk)._plenary_ticket, first + ) + + def test_plenary_ticket_details_query_counts(self): + """The annotation replaces the per-registration ticket queries""" + for tickets in [ + [("onsite", "week_pass")], + [("remote", "student")], + [("hackathon_onsite", "hackathon_only")], + ]: + self.create_registration(tickets) + expected = [("onsite", "week_pass"), ("remote", "student"), (None, None)] + + annotated = Registration.objects.with_plenary_ticket_details().order_by("pk") + with self.assertNumQueries(1): + self.assertEqual( + [(r.plenary_attendance_type, r.plenary_ticket_type) for r in annotated], + expected, + ) + + # Without the annotation, one query for the registrations plus one per + # registration. Both properties share the _plenary_ticket cache, so + # reading them together does not double the count. + plain = Registration.objects.order_by("pk") + with self.assertNumQueries(4): + self.assertEqual( + [(r.plenary_attendance_type, r.plenary_ticket_type) for r in plain], + expected, + ) + + def test_order_by_most_representative(self): + """Non-plenary tickets are dropped and the rest are ranked""" + registration = self.create_registration( + [ + ("remote", "one_day"), + ("hackathon_onsite", "hackathon_only"), + ("onsite", "unknown"), + ("onsite", "student"), + ("remote", "week_pass"), + ("unknown", "week_pass"), + ] + ) + tickets = { + (ticket.attendance_type_id, ticket.ticket_type_id): ticket + for ticket in registration.tickets.all() + } + self.assertEqual( + list(RegistrationTicket.objects.order_by_most_representative()), + [ + tickets[("onsite", "student")], + tickets[("onsite", "unknown")], + tickets[("remote", "week_pass")], + tickets[("remote", "one_day")], + ], + ) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index ffd37fc363d..a25998dac91 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -20,7 +20,7 @@ from django.core.cache import caches from django.core.files.base import ContentFile from django.db import IntegrityError -from django.db.models import OuterRef, Subquery, TextField, Q, Value, Max +from django.db.models import Exists, OuterRef, Subquery, TextField, Q, Value, Max from django.db.models.functions import Coalesce from django.template.loader import render_to_string from django.utils import timezone @@ -351,49 +351,60 @@ def data_for_meetings_overview(meetings, interim_status=None): """Return filtered meetings with sessions and group hierarchy (for the interim menu).""" + # filter + if interim_status == "apprw": + session_status_condition = Q(current_status="apprw") + elif interim_status == "scheda": + session_status_condition = Q(current_status="scheda") + else: + session_status_condition = ~Q( + current_status__in=["apprw", "scheda", "canceledpa"] + ) + + meetings = meetings.filter( + ~Q(type_id="interim") + | Exists( + Session.objects.filter(meeting=OuterRef("pk")) + .with_current_status() + .filter(session_status_condition) + ) + ) + # extract sessions for m in meetings: m.sessions = [] - sessions = Session.objects.filter( - meeting__in=meetings - ).order_by( - 'meeting', 'pk' - ).with_current_status( - ).select_related( - 'group', 'group__parent' + sessions = ( + Session.objects.filter( + meeting__in=meetings, + meeting__type_id="interim", + ) + .order_by("meeting", "pk") + .with_current_status() + .select_related("group", "group__parent") ) meeting_dict = {m.pk: m for m in meetings} for s in sessions.iterator(): meeting_dict[s.meeting_id].sessions.append(s) - # filter - if interim_status == 'apprw': - meetings = [ - m for m in meetings - if not m.type_id == 'interim' or any(s.current_status == 'apprw' for s in m.sessions) - ] - - elif interim_status == 'scheda': - meetings = [ - m for m in meetings - if not m.type_id == 'interim' or any(s.current_status == 'scheda' for s in m.sessions) - ] - - else: - meetings = [ - m for m in meetings - if not m.type_id == 'interim' or not all(s.current_status in ['apprw', 'scheda', 'canceledpa'] for s in m.sessions) - ] - - ietf_group = Group.objects.get(acronym='ietf') + ietf_group = ( + Group.objects.get(acronym="ietf") + if any(m.type_id != "interim" for m in meetings) + else None + ) # set some useful attributes for m in meetings: m.end = m.date + datetime.timedelta(days=m.days) - m.responsible_group = (m.sessions[0].group if m.sessions else None) if m.type_id == 'interim' else ietf_group - m.interim_meeting_cancelled = m.type_id == 'interim' and all(s.current_status == 'canceled' for s in m.sessions) + m.responsible_group = ( + (m.sessions[0].group if m.sessions else None) + if m.type_id == "interim" + else ietf_group + ) + m.interim_meeting_cancelled = m.type_id == "interim" and all( + s.current_status == "canceled" for s in m.sessions + ) return meetings diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 3e5002a466f..763ea528720 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -1,6 +1,4 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved -# -*- coding: utf-8 -*- - +# Copyright The IETF Trust 2007-2026, All Rights Reserved import csv import datetime @@ -20,6 +18,7 @@ from collections import OrderedDict, Counter, deque, defaultdict, namedtuple from functools import partialmethod import jsonschema +from icalendar import Calendar, Event from pathlib import Path from urllib.parse import parse_qs, unquote, urlencode, urlsplit, urlunsplit, urlparse from tempfile import mkstemp @@ -30,10 +29,18 @@ from django.core.cache import caches from django.core.files.storage import storages from django.shortcuts import render, redirect, get_object_or_404 -from django.http import (HttpResponse, HttpResponseRedirect, HttpResponseForbidden, - HttpResponseNotFound, Http404, HttpResponseBadRequest, - JsonResponse, HttpResponseGone, HttpResponseNotAllowed, - FileResponse) +from django.http import ( + HttpResponse, + HttpResponseRedirect, + HttpResponseForbidden, + HttpResponseNotFound, + Http404, + HttpResponseBadRequest, + JsonResponse, + HttpResponseGone, + HttpResponseNotAllowed, + FileResponse, +) from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required @@ -55,6 +62,7 @@ import debug # pyflakes:ignore +from ietf.api.ietf_utils import requires_api_token from ietf.doc.fields import SearchableDocumentsField from ietf.doc.models import Document, State, DocEvent, NewRevisionDocEvent from ietf.doc.storage_utils import ( @@ -62,54 +70,130 @@ retrieve_bytes, store_file, ) +from ietf.doc.templatetags.ietf_filters import absurl from ietf.group.models import Group -from ietf.group.utils import can_manage_session_materials, can_manage_some_groups, can_manage_group +from ietf.group.utils import ( + can_manage_session_materials, + can_manage_some_groups, + can_manage_group, +) from ietf.person.models import Person, User from ietf.ietfauth.utils import role_required, has_role, user_is_person from ietf.mailtrigger.utils import gather_address_lists -from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, \ - SessionPresentation, TimeSlot, SlideSubmission, Attended -from ..blobdb.models import ResolvedMaterial -from ietf.meeting.models import ImportantDate, SessionStatusName, SchedulingEvent, SchedTimeSessAssignment, Room, TimeSlotTypeName -from ietf.meeting.models import Registration -from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm, ImportMinutesForm, - TimeSlotCreateForm, TimeSlotEditForm, SessionCancelForm, SessionEditForm ) -from ietf.meeting.helpers import get_person_by_email, get_schedule_by_name -from ietf.meeting.helpers import get_meeting, get_ietf_meeting, get_current_ietf_meeting_num -from ietf.meeting.helpers import get_schedule, schedule_permissions -from ietf.meeting.helpers import preprocess_assignments_for_agenda, read_agenda_file -from ietf.meeting.helpers import AgendaFilterOrganizer, AgendaKeywordTagger -from ietf.meeting.helpers import convert_draft_to_pdf, get_earliest_session_date -from ietf.meeting.helpers import can_view_interim_request, can_approve_interim_request -from ietf.meeting.helpers import can_edit_interim_request -from ietf.meeting.helpers import can_request_interim_meeting, get_announcement_initial -from ietf.meeting.helpers import sessions_post_save, is_interim_meeting_approved -from ietf.meeting.helpers import send_interim_meeting_cancellation_notice, send_interim_session_cancellation_notice -from ietf.meeting.helpers import send_interim_approval -from ietf.meeting.helpers import send_interim_approval_request -from ietf.meeting.helpers import send_interim_announcement_request, sessions_post_cancel +from ietf.meeting.models import ( + Meeting, + Session, + Schedule, + FloorPlan, + SessionPresentation, + TimeSlot, + SlideSubmission, + Attended, +) +from ietf.blobdb.models import ResolvedMaterial +from ietf.blobdb.storage import BlobdbStorage, BlobFile +from ietf.meeting.models import ( + ImportantDate, + SessionStatusName, + SchedulingEvent, + SchedTimeSessAssignment, + Room, + TimeSlotTypeName, + Registration, +) +from ietf.meeting.forms import ( + CustomDurationField, + SwapDaysForm, + SwapTimeslotsForm, + ImportMinutesForm, + TimeSlotCreateForm, + TimeSlotEditForm, + SessionCancelForm, + SessionEditForm, + InterimMeetingModelForm, + InterimAnnounceForm, + InterimSessionModelForm, + InterimCancelForm, + InterimSessionInlineFormSet, + RequestMinutesForm, + UploadAgendaForm, + UploadBlueSheetForm, + UploadMinutesForm, + UploadSlidesForm, + UploadNarrativeMinutesForm, +) +from ietf.meeting.helpers import ( + get_person_by_email, + get_schedule_by_name, + get_meeting, + get_ietf_meeting, + get_current_ietf_meeting_num, + get_schedule, + schedule_permissions, + preprocess_assignments_for_agenda, + read_agenda_file, + AgendaFilterOrganizer, + AgendaKeywordTagger, + convert_draft_to_pdf, + get_earliest_session_date, + can_view_interim_request, + can_approve_interim_request, + can_edit_interim_request, + can_request_interim_meeting, + get_announcement_initial, + sessions_post_save, + is_interim_meeting_approved, + send_interim_meeting_cancellation_notice, + send_interim_session_cancellation_notice, + send_interim_approval, + send_interim_approval_request, + send_interim_announcement_request, + sessions_post_cancel, + PENDING_INTERIM_MAX_LOOKBACK, +) from ietf.meeting.utils import ( condition_slide_order, finalize, generate_proceedings_content, organize_proceedings_sessions, resolve_uploaded_material, - sort_accept_tuple, store_blobs_for_one_material_doc, + sort_accept_tuple, + store_blobs_for_one_material_doc, +) +from ietf.meeting.utils import ( + add_event_info_to_session_qs, + session_time_for_sorting, + session_requested_by, + SaveMaterialsError, + current_session_status, + get_meeting_sessions, + SessionNotScheduledError, + data_for_meetings_overview, + handle_upload_file, + save_session_minutes_revision, + preprocess_constraints_for_meeting_schedule_editor, + diff_meeting_schedules, + prefetch_schedule_diff_objects, + swap_meeting_schedule_timeslot_assignments, + bulk_create_timeslots, + preprocess_meeting_important_dates, + new_doc_for_session, + write_doc_for_session, + get_activity_stats, + post_process, + create_recording, + delete_recording, + generate_bluesheet, + bluesheet_data, + save_bluesheet, ) -from ietf.meeting.utils import add_event_info_to_session_qs -from ietf.meeting.utils import session_time_for_sorting -from ietf.meeting.utils import session_requested_by, SaveMaterialsError -from ietf.meeting.utils import current_session_status, get_meeting_sessions, SessionNotScheduledError -from ietf.meeting.utils import data_for_meetings_overview, handle_upload_file, save_session_minutes_revision -from ietf.meeting.utils import preprocess_constraints_for_meeting_schedule_editor -from ietf.meeting.utils import diff_meeting_schedules, prefetch_schedule_diff_objects -from ietf.meeting.utils import swap_meeting_schedule_timeslot_assignments, bulk_create_timeslots -from ietf.meeting.utils import preprocess_meeting_important_dates -from ietf.meeting.utils import new_doc_for_session, write_doc_for_session -from ietf.meeting.utils import get_activity_stats, post_process, create_recording, delete_recording -from ietf.meeting.utils import generate_bluesheet, bluesheet_data, save_bluesheet from ietf.message.utils import infer_message -from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName, CountryName +from ietf.name.models import ( + SlideSubmissionStatusName, + ProceedingsMaterialTypeName, + SessionPurposeName, + CountryName, +) from ietf.utils import markdown from ietf.utils.decorators import require_api_key from ietf.utils.hedgedoc import Note, NoteError @@ -124,15 +208,6 @@ from ietf.utils.timezone import datetime_today, date_today from ietf.settings import YOUTUBE_DOMAINS -from .forms import (InterimMeetingModelForm, InterimAnnounceForm, InterimSessionModelForm, - InterimCancelForm, InterimSessionInlineFormSet, RequestMinutesForm, - UploadAgendaForm, UploadBlueSheetForm, UploadMinutesForm, UploadSlidesForm, - UploadNarrativeMinutesForm) - -from icalendar import Calendar, Event -from ietf.doc.templatetags.ietf_filters import absurl -from ..api.ietf_utils import requires_api_token -from ..blobdb.storage import BlobdbStorage, BlobFile request_summary_exclude_group_types = ['team'] @@ -4110,15 +4185,25 @@ def delete_schedule(request, num, owner, name): # Interim Views # ------------------------------------------------- def interim_announce(request): - '''View which shows interim meeting requests awaiting announcement''' - meetings = data_for_meetings_overview(Meeting.objects.filter(type='interim').order_by('date'), interim_status='scheda') + """View which shows interim meeting requests awaiting announcement""" + meetings = data_for_meetings_overview( + Meeting.objects.filter( + type="interim", date__gte=date_today() - PENDING_INTERIM_MAX_LOOKBACK + ).order_by("date"), + interim_status="scheda", + ) menu_entries = get_interim_menu_entries(request) - selected_menu_entry = 'announce' + selected_menu_entry = "announce" - return render(request, "meeting/interim_announce.html", { - 'menu_entries': menu_entries, - 'selected_menu_entry': selected_menu_entry, - 'meetings': meetings}) + return render( + request, + "meeting/interim_announce.html", + { + "menu_entries": menu_entries, + "selected_menu_entry": selected_menu_entry, + "meetings": meetings, + }, + ) @role_required('Secretariat',) @@ -4173,21 +4258,30 @@ def interim_skip_announcement(request, number): def interim_pending(request): - - '''View which shows interim meeting requests pending approval''' - meetings = data_for_meetings_overview(Meeting.objects.filter(type='interim').order_by('date'), interim_status='apprw') + """View which shows interim meeting requests pending approval""" + meetings = data_for_meetings_overview( + Meeting.objects.filter( + type="interim", date__gte=date_today() - PENDING_INTERIM_MAX_LOOKBACK + ).order_by("date"), + interim_status="apprw", + ) menu_entries = get_interim_menu_entries(request) - selected_menu_entry = 'pending' + selected_menu_entry = "pending" for meeting in meetings: if can_approve_interim_request(meeting, request.user): meeting.can_approve = True - return render(request, "meeting/interim_pending.html", { - 'menu_entries': menu_entries, - 'selected_menu_entry': selected_menu_entry, - 'meetings': meetings}) + return render( + request, + "meeting/interim_pending.html", + { + "menu_entries": menu_entries, + "selected_menu_entry": selected_menu_entry, + "meetings": meetings, + }, + ) @login_required @@ -4838,19 +4932,14 @@ def proceedings_attendees(request, num=None): onsite_pks = frozenset(p.pk for p in onsite) remote_pks = frozenset(p.pk for p in remote) - regs = [ - reg - for reg in Registration.objects.onsite() - .filter(meeting__number=num) + regs_to_consider = ( + Registration.objects.filter(meeting__number=num) + .with_plenary_ticket_details() .select_related("person") - if reg.person.pk in onsite_pks - ] + [ - reg - for reg in Registration.objects.remote() - .filter(meeting__number=num) - .select_related("person") - if reg.person.pk in remote_pks - ] + ) + regs = [ + reg for reg in regs_to_consider.onsite() if reg.person.pk in onsite_pks + ] + [reg for reg in regs_to_consider.remote() if reg.person.pk in remote_pks] registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name)) country_codes = [r.country_code for r in registrations if r.country_code] diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index 46418714880..0edf4498fc0 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -18,6 +18,7 @@ from email.utils import parseaddr from textwrap import dedent +from django.db import transaction from django.db.models import Q, Count, F, QuerySet from django.conf import settings from django.contrib.sites.models import Site @@ -36,6 +37,7 @@ from ietf.utils.mail import send_mail_text, send_mail, get_payload_text from ietf.utils.log import log from ietf.person.name import unidecode_name +from ietf.person.utils import assign_primary_uuid from ietf.utils.timezone import date_today, datetime_from_date, DEADLINE_TZINFO import debug # pyflakes:ignore @@ -416,13 +418,17 @@ def make_nomineeposition(nomcom, candidate, position, author): def make_nomineeposition_for_newperson(nomcom, candidate_name, candidate_email, position, author): - # This is expected to fail if called with an existing email address - email = Email.objects.create(address=candidate_email, origin="nominee: %s" % nomcom.group.acronym) - person = Person.objects.create(name=candidate_name, - ascii=unidecode_name(candidate_name), - ) - email.person = person - email.save() + # This is expected to fail if called with an existing email address. + # Atomic so a Person is never left without the primary UUID that external systems + # need to name them by, and so a failure part way leaves no half-built nominee. + with transaction.atomic(): + email = Email.objects.create(address=candidate_email, origin="nominee: %s" % nomcom.group.acronym) + person = Person.objects.create(name=candidate_name, + ascii=unidecode_name(candidate_name), + ) + assign_primary_uuid(person) + email.person = person + email.save() # send email to secretariat and nomcomchair to warn about the new person subject = 'New person is created' diff --git a/ietf/person/admin.py b/ietf/person/admin.py index f46edcf8aeb..4d3ad745e63 100644 --- a/ietf/person/admin.py +++ b/ietf/person/admin.py @@ -3,9 +3,13 @@ import simple_history from django import forms +from django.contrib import messages +from django.db import transaction -from ietf.person.models import Email, Alias, Person, PersonalApiKey, PersonEvent, PersonApiKeyEvent, PersonExtResource +from ietf.person.models import Email, Alias, Person, PersonalApiKey, PersonEvent, \ + PersonApiKeyEvent, PersonExtResource, PersonUUID from ietf.person.name import name_parts +from ietf.person.utils import queue_person_uuid_push from ietf.utils.admin import SaferStackedInline, SaferTabularInline from ietf.utils.validators import validate_external_resource_value @@ -29,6 +33,53 @@ class AliasAdmin(admin.ModelAdmin): class AliasInline(SaferStackedInline): model = Alias + +@admin.action(description="Make this the person's primary UUID") +def set_primary(modeladmin, request, queryset): + """Re-designate a Person's primary UUID + + Acts on exactly one UUID at a time: promoting two at once would either violate the + one-primary-per-person constraint or silently ignore one of them. + """ + if queryset.count() != 1: + modeladmin.message_user( + request, "Select exactly one UUID.", level=messages.ERROR + ) + return + new_primary = queryset.first() + if new_primary.primary: + modeladmin.message_user(request, "That UUID is already primary.") + return + person = new_primary.person + with transaction.atomic(): + person.uuids.filter(primary=True).update(primary=False) + new_primary.primary = True + new_primary.save(update_fields=["primary"]) + queue_person_uuid_push(person) + modeladmin.message_user( + request, f"{new_primary.uuid} is now the primary UUID for {person}." + ) + + +class PersonUUIDAdmin(admin.ModelAdmin): + list_display = ["uuid", "person", "primary", "time"] # noqa: RUF012 + list_filter = ["primary"] # noqa: RUF012 + search_fields = ["uuid", "person__name"] # noqa: RUF012 + raw_id_fields = ["person"] # noqa: RUF012 + readonly_fields = ["uuid", "primary", "time"] # noqa: RUF012 + actions = [set_primary] # noqa: RUF012 +admin.site.register(PersonUUID, PersonUUIDAdmin) + + +class PersonUUIDInline(SaferStackedInline): + model = PersonUUID + extra = 0 + # primary is changed through the PersonUUID admin's set_primary action, which demotes + # the old primary first. Editing it here would trip the uniqueness constraint. + readonly_fields = ["uuid", "primary", "time"] # noqa: RUF012 + can_delete = False + + class PersonAdmin(simple_history.admin.SimpleHistoryAdmin): def plain_name(self, obj): if obj.plain: @@ -41,7 +92,7 @@ def plain_name(self, obj): readonly_fields = ("name_from_draft", ) search_fields = ["name", "ascii"] raw_id_fields = ["user"] - inlines = [ EmailInline, AliasInline, ] + inlines = [ EmailInline, AliasInline, PersonUUIDInline] # actions = None admin.site.register(Person, PersonAdmin) diff --git a/ietf/person/api_uuid.py b/ietf/person/api_uuid.py new file mode 100644 index 00000000000..0fbf452566e --- /dev/null +++ b/ietf/person/api_uuid.py @@ -0,0 +1,290 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +"""Person UUID resolution API + +Lets an authorized application ask about a Person UUID it holds and learn the Person's +current identifier set. Responses carry identifiers only - no name, address or database +key. +""" + +from drf_spectacular.utils import ( + OpenApiExample, + extend_schema, + extend_schema_view, +) +from rest_framework import mixins, serializers, viewsets +from rest_framework.decorators import action +from rest_framework.response import Response +from rest_framework.views import APIView + +from ietf.person.models import Person, PersonUUID + +MAX_BATCH = 500 + +# A batch entry either resolved to a Person or it did not. Both outcomes use the same +# entry shape, discriminated by this field, so a consumer switches on it instead of +# inspecting which fields came back. +ENTRY_STATUSES = ("resolved", "unknown") + + +def uuid_sets_for(person_ids): + """Map each person_id to its (primary_uuid, [prior_uuids]) in a single query + + Keeps the batch endpoints' query count independent of the batch size: reading + Person.primary_uuid and Person.prior_uuids per row would be two queries per Person. + Ordering matches Person.prior_uuids. + """ + sets = {pid: (None, []) for pid in person_ids} + rows = ( + PersonUUID.objects.filter(person_id__in=person_ids) + .order_by("time", "uuid") + .values_list("person_id", "uuid", "primary") + ) + for pid, value, is_primary in rows: + _, priors = sets[pid] + if is_primary: + sets[pid] = (value, priors) + else: + priors.append(value) + return sets + + +class PersonUUIDResolutionSerializer(serializers.Serializer): + """A PersonUUID together with its Person's whole identifier set""" + + uuid = serializers.UUIDField(read_only=True) + is_primary = serializers.BooleanField(source="primary", read_only=True) + primary_uuid = serializers.UUIDField(source="person.primary_uuid", read_only=True) + prior_uuids = serializers.ListField( + source="person.prior_uuids", child=serializers.UUIDField(), read_only=True + ) + + +class PersonUUIDBatchRequestSerializer(serializers.Serializer): + uuids = serializers.ListField( + child=serializers.UUIDField(), allow_empty=False, max_length=MAX_BATCH + ) + + +class PersonUUIDBatchEntrySerializer(serializers.Serializer): + """One requested UUID and, when it resolved, its Person's identifier set + + Every field is always present. The identifier fields are null - prior_uuids empty - + when status is unknown. + + Output only, and deliberately not read_only=True field by field: read_only implies + required=False, which would leave a generated client treating even status as optional. + """ + + uuid = serializers.UUIDField() + status = serializers.ChoiceField(choices=ENTRY_STATUSES) + is_primary = serializers.BooleanField(allow_null=True) + primary_uuid = serializers.UUIDField(allow_null=True) + prior_uuids = serializers.ListField(child=serializers.UUIDField()) + + +class PersonUUIDBatchResponseSerializer(serializers.Serializer): + results = PersonUUIDBatchEntrySerializer(many=True) + + +class PersonPkBatchRequestSerializer(serializers.Serializer): + person_pks = serializers.ListField( + child=serializers.IntegerField(), allow_empty=False, max_length=MAX_BATCH + ) + + +class PersonPkBatchEntrySerializer(serializers.Serializer): + """One requested Person.pk and, when it resolved, that Person's identifier set + + Same one-shape-for-both-outcomes and output-only rules as + PersonUUIDBatchEntrySerializer. + """ + + person_pk = serializers.IntegerField() + status = serializers.ChoiceField(choices=ENTRY_STATUSES) + primary_uuid = serializers.UUIDField(allow_null=True) + prior_uuids = serializers.ListField(child=serializers.UUIDField()) + + +class PersonPkBatchResponseSerializer(serializers.Serializer): + results = PersonPkBatchEntrySerializer(many=True) + + +@extend_schema(tags=["person"]) +@extend_schema_view( + retrieve=extend_schema( + operation_id="person_uuid_retrieve", + summary="Resolve a Person UUID", + description=( + "Resolve any UUID the datatracker has issued for a Person to that Person's " + "current identifier set. A UUID that stopped being primary because of a " + 'merge still resolves, and the response carries the current primary. A 200 ' + 'whose primary_uuid differs from the requested uuid means "same person, new ' + 'identifier" - it is not an error.\n\n' + "A 404 means no Person has this UUID. It does not distinguish a UUID the " + "datatracker never issued from one it issued to a Person that has since been " + "deleted, because deleting a Person deletes its UUIDs.\n\n" + "primary_uuid is the person's current primary. Re-read it; do not assume it " + "is unchanged from a previous response, and do not assume a UUID you hold " + "remains primary.\n\n" + "Responses contain identifiers only. No name, address or database key is " + "returned." + ), + responses=PersonUUIDResolutionSerializer, + ) +) +class PersonUUIDViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): + """Resolve Person UUIDs to their Person's current identifier set""" + + api_key_endpoint = "ietf.person.api_uuid" + queryset = PersonUUID.objects.select_related("person") + serializer_class = PersonUUIDResolutionSerializer + lookup_field = "uuid" + lookup_url_kwarg = "uuid" + lookup_value_converter = "anycase_uuid" + + @extend_schema( + operation_id="person_uuid_lookup", + summary="Resolve a batch of Person UUIDs", + description=( + f"Resolve up to {MAX_BATCH} UUIDs in one call. Always returns 200 with one " + "results entry per distinct requested UUID - no entry is ever omitted and no " + "unresolvable UUID fails the request.\n\n" + "Each entry carries a status of resolved or unknown, corresponding to the " + "200 and 404 outcomes of the single-UUID endpoint. Every entry has the same " + "fields either way, with the identifier fields null when status is unknown, " + "so switch on status rather than on which fields are present. Duplicate " + "inputs produce one entry. Entry order is not significant - match on uuid." + ), + request=PersonUUIDBatchRequestSerializer, + responses=PersonUUIDBatchResponseSerializer, + ) + @action(detail=False, methods=["post"]) + def lookup(self, request): + requested_serializer = PersonUUIDBatchRequestSerializer(data=request.data) + requested_serializer.is_valid(raise_exception=True) + requested = list(dict.fromkeys(requested_serializer.validated_data["uuids"])) + + found = {row.uuid: row for row in PersonUUID.objects.filter(uuid__in=requested)} + sets = uuid_sets_for({row.person_id for row in found.values()}) + + results = [] + for value in requested: + row = found.get(value) + if row is None: + results.append( + { + "uuid": value, + "status": "unknown", + "is_primary": None, + "primary_uuid": None, + "prior_uuids": [], + } + ) + continue + primary, priors = sets[row.person_id] + results.append( + { + "uuid": value, + "status": "resolved", + "is_primary": row.primary, + "primary_uuid": primary, + "prior_uuids": priors, + } + ) + return Response( + PersonUUIDBatchResponseSerializer({"results": results}).data + ) + + +@extend_schema(tags=["person"]) +class PersonUUIDByPersonPkView(APIView): + """Transitional pk-to-UUID conversion, for consumers migrating off Person.pk + + Batch-only on purpose: the one legitimate use is a single bulk conversion, and a + convenient per-request lookup would become a permanent pk-to-UUID service. Its own + api_key_endpoint so its tokens can be withdrawn without touching the resolution API. + + A plain APIView rather than a ViewSet: nothing here is a resource, and routing a + lookup through a ViewSet meant calling it "create", which made the schema claim a 201 + for a request that creates nothing and returns 200. + """ + + api_key_endpoint = "ietf.person.api_uuid_by_pk" + + @extend_schema( + deprecated=True, + operation_id="person_uuid_by_person_pk", + summary="TRANSITIONAL: resolve Person database keys to UUIDs", + description=( + "DEPRECATED FROM FIRST RELEASE, AND WILL BE WITHDRAWN. Resolves up to " + f"{MAX_BATCH} Person.pk values to their UUID sets, so an application that " + "keyed its records on the datatracker database key can convert them to " + "UUIDs once and stop using the key. Person.pk is the identifier this whole " + "feature exists to stop exporting: it does not survive a Person merge, " + "which is why holding it is unsafe.\n\n" + "Intended to be called once per consuming application, to migrate a table. " + "Do not call it at request time and do not build anything that keeps " + "needing it.\n\n" + "One results entry per distinct requested pk, with the same fields either " + "way and the identifier fields null when status is unknown. Switch on " + "status." + ), + request=PersonPkBatchRequestSerializer, + responses={200: PersonPkBatchResponseSerializer}, + examples=[ + OpenApiExample( + "Two pks, one of them unknown", + value={ + "results": [ + { + "person_pk": 12345, + "status": "resolved", + "primary_uuid": "6f9a1c30-6c7e-4f0a-9a3f-2f1d0b8a4e11", + "prior_uuids": ["0b21f8d4-1a55-4c9e-8f77-9c2b4a6e3d02"], + }, + { + "person_pk": 999999, + "status": "unknown", + "primary_uuid": None, + "prior_uuids": [], + }, + ] + }, + response_only=True, + ) + ], + ) + def post(self, request): + requested_serializer = PersonPkBatchRequestSerializer(data=request.data) + requested_serializer.is_valid(raise_exception=True) + requested = list( + dict.fromkeys(requested_serializer.validated_data["person_pks"]) + ) + + existing = set( + Person.objects.filter(pk__in=requested).values_list("pk", flat=True) + ) + sets = uuid_sets_for(existing) + + results = [] + for pk in requested: + if pk not in existing: + results.append( + { + "person_pk": pk, + "status": "unknown", + "primary_uuid": None, + "prior_uuids": [], + } + ) + continue + primary, priors = sets[pk] + results.append( + { + "person_pk": pk, + "status": "resolved", + "primary_uuid": primary, + "prior_uuids": priors, + } + ) + return Response(PersonPkBatchResponseSerializer({"results": results}).data) diff --git a/ietf/person/factories.py b/ietf/person/factories.py index 655f25994b3..110542e0796 100644 --- a/ietf/person/factories.py +++ b/ietf/person/factories.py @@ -20,7 +20,8 @@ import debug # pyflakes:ignore -from ietf.person.models import Person, Alias, Email, PersonalApiKey, PersonApiKeyEvent, PERSON_API_KEY_ENDPOINTS +from ietf.person.models import Person, Alias, Email, PersonalApiKey, PersonApiKeyEvent, \ + PERSON_API_KEY_ENDPOINTS, PersonUUID from ietf.person.name import normalize_name, unidecode_name @@ -64,6 +65,21 @@ def set_password(obj, create, extracted, **kwargs): # pylint: disable=no-self-ar obj.set_password( '%s+password' % obj.username ) # pylint: disable=no-value-for-parameter obj.save() + +class PersonUUIDFactory(factory.django.DjangoModelFactory): + """A UUID for a Person + + Defaults to a superseded, non-primary UUID, which is what a test asking for an extra + UUID wants. PersonFactory uses this with primary=True to make each Person's primary; + creating a second primary for the same Person violates a uniqueness constraint. + """ + person = factory.SubFactory("ietf.person.factories.PersonFactory") + primary = False + + class Meta: + model = PersonUUID + + class PersonFactory(factory.django.DjangoModelFactory): class Meta: model = Person @@ -79,6 +95,16 @@ class Meta: class Params: with_bio = factory.Trait(biography = "\n\n".join(fake.paragraphs())) # type: ignore + @factory.post_generation + def primary_uuid(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument + """Give the Person the primary UUID every Person is supposed to have + + Pass primary_uuid=False for a Person with no UUIDs at all, which is otherwise + not reachable through any production path. + """ + if create and extracted is not False: + PersonUUIDFactory(person=obj, primary=True) + @factory.post_generation def default_aliases(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument make_alias = getattr(AliasFactory, 'create' if create else 'build') diff --git a/ietf/person/migrations/0006_personuuid.py b/ietf/person/migrations/0006_personuuid.py new file mode 100644 index 00000000000..17a56808c6d --- /dev/null +++ b/ietf/person/migrations/0006_personuuid.py @@ -0,0 +1,67 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import ietf.person.models + + +def forward(apps, schema_editor): + Person = apps.get_model("person", "Person") + PersonUUID = apps.get_model("person", "PersonUUID") + # uuid and time come from the field defaults + PersonUUID.objects.bulk_create( + [PersonUUID(person=person, primary=True) for person in Person.objects.all()], + batch_size=1000, + ) + + +def reverse(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + dependencies = [ + ("person", "0005_alter_historicalperson_pronouns_selectable_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="PersonUUID", + fields=[ + ( + "uuid", + models.UUIDField( + default=ietf.person.models.unused_person_uuid, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("primary", models.BooleanField(default=False)), + ( + "time", + models.DateTimeField( + default=django.utils.timezone.now, editable=False + ), + ), + ( + "person", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="uuids", + to="person.person", + ), + ), + ], + ), + migrations.AddConstraint( + model_name="personuuid", + constraint=models.UniqueConstraint( + condition=models.Q(("primary", True)), + fields=("person",), + name="unique_primary_uuid_per_person", + ), + ), + migrations.RunPython(forward, reverse), + ] diff --git a/ietf/person/models.py b/ietf/person/models.py index bb6b4ffc737..13c24bd1058 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -44,6 +44,48 @@ def name_character_validator(value): ) +def unused_person_uuid(): + MAX_ATTEMPTS = 50 # ludicrously large + for _ in range(MAX_ATTEMPTS): + candidate = uuid.uuid4() + if not PersonUUID.objects.filter(uuid=candidate).exists(): + return candidate + raise RuntimeError("Unable to generate unused UUID") + + +class PersonUUID(models.Model): + """Surrogate key for a Person + + A Person has one or more UUIDs. Exactly one is primary: it is the identifier handed + to external systems. Merging Persons unions the sets and keeps a single primary, so a + UUID an external system holds keeps resolving to the surviving Person even after it + stops being primary. + + Deleting a Person deletes its UUIDs. Their values return to the pool + unused_person_uuid() draws from, so a deleted UUID could in principle be issued again + to a different Person. That requires uuid4() to reproduce a specific 122-bit value, + so the risk is accepted. + """ + uuid = models.UUIDField(primary_key=True, editable=False, default=unused_person_uuid) + person = models.ForeignKey( + "person.Person", related_name="uuids", on_delete=models.CASCADE + ) + primary = models.BooleanField(default=False) + time = models.DateTimeField(default=timezone.now, editable=False) + + class Meta: + constraints = [ # noqa: RUF012 + models.UniqueConstraint( + fields=["person"], + condition=models.Q(primary=True), + name="unique_primary_uuid_per_person", + ), + ] + + def __str__(self): + return str(self.uuid) + + class Person(models.Model): history = HistoricalRecords() user = OneToOneField(User, blank=True, null=True, on_delete=models.SET_NULL) @@ -201,6 +243,27 @@ def full_name_as_key(self): # this is mostly a remnant from the old views, needed in the menu return self.plain_name().lower().replace(" ", ".") + @property + def primary_uuid(self): + """The UUID to hand to external systems, or None if data is inconsistent""" + row = self.uuids.filter(primary=True).first() + return row.uuid if row else None + + @property + def prior_uuids(self): + """UUIDs that still resolve to this Person but are no longer primary + + Oldest first. Ties are broken on the UUID itself so the order is total rather + than left to the database, and so it matches uuid_sets_for() in + ietf.person.api_uuid. Nothing in the current code produces a tie: the timestamp + defaults per row and a merge moves UUIDs without rewriting it. + """ + return list( + self.uuids.filter(primary=False) + .order_by("time", "uuid") + .values_list("uuid", flat=True) + ) + def photo_name(self,thumb=False): hasher = Hashids(salt='Person photo name salt',min_length=5) _, first, _, last, _ = name_parts(self.ascii) @@ -215,9 +278,21 @@ def rfcs(self): # When RfcAuthors are populated, this may over-return if an author is dropped # from the author list between the final draft and the published RFC. Should # ignore DocumentAuthors when an RfcAuthor exists for a draft. - rfcs = list(Document.objects.filter(type="rfc").filter(models.Q(documentauthor__person=self)|models.Q(rfcauthor__person=self)).distinct()) - rfcs.sort(key=lambda d: d.name ) - return rfcs + # + # The two authorship tables are queried separately and combined here. As a + # single ORed queryset, neither person_id index is usable and the join has to + # be materialized in full before being deduplicated. + ids = set( + Document.objects.filter( + type="rfc", documentauthor__person=self + ).values_list("pk", flat=True) + ) + ids.update( + Document.objects.filter( + type="rfc", rfcauthor__person=self + ).values_list("pk", flat=True) + ) + return sorted(Document.objects.filter(pk__in=ids), key=lambda d: d.name) def active_drafts(self): from ietf.doc.models import Document diff --git a/ietf/person/resources.py b/ietf/person/resources.py index 42bb9f7732c..c8a8bf8af35 100644 --- a/ietf/person/resources.py +++ b/ietf/person/resources.py @@ -10,7 +10,18 @@ from ietf import api -from ietf.person.models import (Person, Email, Alias, PersonalApiKey, PersonEvent, PersonApiKeyEvent, HistoricalPerson, HistoricalEmail, PersonExtResource) # type: ignore +from ietf.person.models import ( # type: ignore + Person, + Email, + Alias, + PersonalApiKey, + PersonEvent, + PersonApiKeyEvent, + HistoricalPerson, + HistoricalEmail, + PersonExtResource, + PersonUUID, +) from ietf.utils.resources import UserResource @@ -35,6 +46,26 @@ class Meta: } api.person.register(PersonResource()) + +class PersonUUIDResource(ModelResource): + person = ToOneField(PersonResource, "person") + + class Meta: + cache = SimpleCache() + queryset = PersonUUID.objects.all() + serializer = api.Serializer() + # resource_name = 'personuuid' + ordering = ['uuid', ] # noqa: RUF012 + filtering = { # noqa: RUF012 + "uuid": ALL, + "primary": ALL, + "time": ALL, + "person": ALL_WITH_RELATIONS, + } + + +api.person.register(PersonUUIDResource()) + class EmailResource(ModelResource): person = ToOneField(PersonResource, 'person', null=True) class Meta: diff --git a/ietf/person/tasks.py b/ietf/person/tasks.py index f0c979fa267..4e0a34942df 100644 --- a/ietf/person/tasks.py +++ b/ietf/person/tasks.py @@ -7,11 +7,13 @@ from celery import shared_task from django.conf import settings +from django.db.models import Count, Q from django.utils import timezone from ietf.utils import log from ietf.utils.mail import send_mail -from .models import PersonalApiKey, PersonApiKeyEvent +from .models import Person, PersonalApiKey, PersonApiKeyEvent +from .utils import ensure_primary_uuid @shared_task @@ -57,3 +59,72 @@ def purge_personal_api_key_events_task(keep_days): count = len(old_events) old_events.delete() log.log(f"Deleted {count} PersonApiKeyEvents older than {keep_since}") + + +@shared_task +def check_person_uuids_task(fix=False): + """Report - and optionally repair - Persons whose UUID set is inconsistent + + Every Person is supposed to have exactly one primary UUID. Assignment is an explicit + assign_primary_uuid() call at each site that creates a Person, so a site added + without one leaves Persons that cannot be named to any external system. This finds + them. + + Checks for exactly one rather than at least one. A partial unique constraint should + make more than one impossible, so finding one means something is grossly wrong with + the data and worth saying out loud - and this is the job that claims the condition + holds, so it may as well test it. + """ + broken = ( + Person.objects.annotate( + uuid_count=Count("uuids", distinct=True), + primary_count=Count("uuids", filter=Q(uuids__primary=True), distinct=True), + ) + .exclude(primary_count=1) + .order_by("pk") + ) + + count = 0 + for person in broken: + count += 1 + if person.primary_count > 1: + # ensure_primary_uuid() cannot resolve this - it would pick one of the + # primaries arbitrarily. Needs a human to decide which one survives. + log.log( + f"Person {person.pk} ({person.name}): {person.primary_count} primary " + f"UUIDs, which the unique constraint should have prevented - not " + f"repairing automatically" + ) + continue + problem = "no UUIDs at all" if person.uuid_count == 0 else "no primary UUID" + log.log(f"Person {person.pk} ({person.name}): {problem}") + if fix: + row = ensure_primary_uuid(person) + log.log(f"Person {person.pk}: primary UUID is now {row.uuid}") + + if count == 0: + log.log("check_person_uuids: every Person has exactly one primary UUID") + else: + log.log( + f"check_person_uuids: {count} Person(s) " + f"{'repaired' if fix else 'need attention'}" + ) + return count + + +@shared_task +def push_person_uuids_task(person_pk): + """Push a Person's UUID set to Authentik + + Enqueued by ietf.person.utils.queue_person_uuid_push whenever the set changes. The + Authentik client does not exist yet, so for now this records the desired state that + will be pushed. + """ + person = Person.objects.filter(pk=person_pk).first() + if person is None: + log.log(f"Not pushing UUIDs for Person {person_pk}: no such Person") + return + log.log( + f"Person {person_pk} UUIDs: primary={person.primary_uuid} " + f"prior={[str(u) for u in person.prior_uuids]}" + ) diff --git a/ietf/person/tests.py b/ietf/person/tests.py index 42c2c1547cb..848152183cf 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -4,34 +4,52 @@ import datetime import json +import uuid from unittest import mock from io import StringIO, BytesIO from PIL import Image from pyquery import PyQuery +import django.core.signing from django.core.exceptions import ValidationError +from django.db import connection, transaction +from django.db.utils import IntegrityError from django.http import HttpRequest from django.test import override_settings +from django.test.utils import CaptureQueriesContext from django.urls import reverse as urlreverse from django.utils import timezone from django.utils.encoding import iri_to_uri +import yaml + import debug # pyflakes:ignore from ietf.community.models import CommunityList +from ietf.doc.factories import WgDraftFactory, WgRfcFactory from ietf.group.factories import RoleFactory from ietf.group.models import Group from ietf.message.models import Message from ietf.nomcom.models import NomCom from ietf.nomcom.test_data import nomcom_test_data from ietf.nomcom.factories import NomComFactory, NomineeFactory, NominationFactory, FeedbackFactory, PositionFactory -from ietf.person.factories import EmailFactory, PersonFactory, PersonApiKeyEventFactory -from ietf.person.models import Person, Alias, PersonApiKeyEvent -from ietf.person.tasks import purge_personal_api_key_events_task +from ietf.nomcom.utils import make_nomineeposition_for_newperson +from ietf.person.factories import ( + EmailFactory, + PersonFactory, + PersonApiKeyEventFactory, + PersonUUIDFactory, +) +from ietf.person.models import Person, Alias, PersonApiKeyEvent, PersonUUID +from ietf.person.tasks import (purge_personal_api_key_events_task, push_person_uuids_task, + check_person_uuids_task) from ietf.person.utils import (merge_persons, determine_merge_order, send_merge_notification, handle_users, get_extra_primary, dedupe_aliases, move_related_objects, merge_nominees, - handle_reviewer_settings, get_dots) + handle_reviewer_settings, get_dots, assign_primary_uuid, ensure_primary_uuid, + get_person_uuid_object) +from ietf.submit.utils import ensure_person_email_info_exists +from kombu.exceptions import OperationalError as KombuOperationalError from ietf.review.models import ReviewerSettings from ietf.utils.test_utils import TestCase, login_testing_unauthorized from ietf.utils.mail import outbox, empty_outbox @@ -108,6 +126,61 @@ def test_person_profile(self): r = self.client.get(photo_url) self.assertEqual(r.status_code, 200) + def test_person_profile_query_count(self): + """The page's cost must not scale with how much a person has written""" + + def profile_queries(rfcs, active, expired): + person = PersonFactory() + RoleFactory(person=person, name_id="chair") + WgRfcFactory.create_batch(rfcs, authors=[person]) + WgDraftFactory.create_batch(active, authors=[person]) + WgDraftFactory.create_batch( + expired, authors=[person], states=[("draft", "expired")] + ) + url = urlreverse( + "ietf.person.views.profile", + kwargs={"email_or_name": person.plain_name()}, + ) + with CaptureQueriesContext(connection) as context: + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + return len(context.captured_queries) + + few = profile_queries(1, 1, 1) + many = profile_queries(6, 4, 5) + self.assertEqual( + many, + few, + f"{many} queries for 15 documents vs {few} for 3 - a query per row crept in", + ) + + @override_settings( + CACHES={ + "default": {"BACKEND": "django.core.cache.backends.dummy.DummyCache"}, + "slowpages": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "test-person-profile", + }, + } + ) + def test_person_profile_sections_cached(self): + person = PersonFactory() + WgRfcFactory(authors=[person]) + WgDraftFactory(authors=[person]) + url = urlreverse( + "ietf.person.views.profile", kwargs={"email_or_name": person.plain_name()} + ) + + first = self.client.get(url) + self.assertEqual(first.status_code, 200) + with CaptureQueriesContext(connection) as context: + second = self.client.get(url) + self.assertEqual(second.status_code, 200) + # The cached section is HTML, not text to be escaped again. + self.assertEqual(first.content, second.content) + self.assertContains(second, person.name) + self.assertLess(len(context.captured_queries), 5) + def test_person_profile_without_email(self): person = PersonFactory(name="foobar@example.com") # delete Email record @@ -116,6 +189,54 @@ def test_person_profile_without_email(self): r = self.client.get(url) self.assertContains(r, person.name, status_code=200) + def test_person_profile_by_uuid(self): + person_a = PersonFactory(name="A Fine Person") + url_a = urlreverse( + "ietf.person.views.profile_by_uuid", + kwargs={"uuid": person_a.primary_uuid}, + ) + + person_b = PersonFactory(name="Brilliant Person") + url_b = urlreverse( + "ietf.person.views.profile_by_uuid", + kwargs={"uuid": person_b.primary_uuid}, + ) + + r = self.client.get(url_a) + self.assertContains(r, person_a.name) + self.assertNotContains(r, person_b.name) + + r = self.client.get(url_b) + self.assertNotContains(r, person_a.name) + self.assertContains(r, person_b.name) + + # Move b's UUID to a as a prior UUID, as a merge would... + person_b.uuids.update(person=person_a, primary=False) + # ... and the old address redirects to a's canonical one + r = self.client.get(url_b) + self.assertRedirects(r, url_a) + r = self.client.get(url_b, follow=True) + self.assertContains(r, person_a.name) + self.assertNotContains(r, person_b.name) + + def test_person_profile_by_uuid_upper_case(self): + person = PersonFactory() + uuid_value = person.primary_uuid + url = urlreverse( + "ietf.person.views.profile_by_uuid", kwargs={"uuid": uuid_value} + ) + upper = url.replace(str(uuid_value), str(uuid_value).upper()) + self.assertNotEqual(url, upper) + r = self.client.get(upper) + self.assertContains(r, person.name, status_code=200) + + def test_person_profile_by_uuid_unknown(self): + url = urlreverse( + "ietf.person.views.profile_by_uuid", kwargs={"uuid": uuid.uuid4()} + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + def test_case_insensitive(self): # Case insensitive seach person = PersonFactory(name="Test Person") @@ -394,9 +515,12 @@ def test_merge_persons(self): request = HttpRequest() request.user = user source = PersonFactory() + PersonUUIDFactory(person=source) # give them an extra target = PersonFactory() mars = RoleFactory(name_id='chair',group__acronym='mars').group source_id = source.pk + source_uuids = {source.primary_uuid, *source.prior_uuids} + target_uuids = {target.primary_uuid, *target.prior_uuids} source_email = source.email_set.first() source_alias = source.alias_set.first() source_user = source.user @@ -415,6 +539,14 @@ def test_merge_persons(self): self.assertIn(nomination, target.nomination_set.all()) self.assertFalse(Person.objects.filter(id=source_id)) self.assertFalse(source_user.is_active) + self.assertEqual( + {target.primary_uuid, *target.prior_uuids}, + source_uuids | target_uuids, + ) + # The survivor keeps its own primary; the source's UUIDs become priors + self.assertIsNotNone(target.primary_uuid) + self.assertIn(target.primary_uuid, target_uuids) + self.assertEqual(set(target.prior_uuids), source_uuids) def test_merge_persons_reviewer_settings(self): secretariat_role = RoleFactory(group__acronym='secretariat', name_id='secr') @@ -478,6 +610,506 @@ def test_send_merge_request(self): self.assertIn(source.user.username, message.to) +class PersonUUIDTests(TestCase): + def test_every_creation_path_assigns_a_primary(self): + """Each production route that creates a Person gives it one primary UUID""" + # ietf.ietfauth.views.confirm_account + confirm_url = urlreverse( + "ietf.ietfauth.views.confirm_account", + kwargs={ + "auth": django.core.signing.dumps( + "uuidtest@example.com", salt="create_account" + ) + }, + ) + self.client.post( + confirm_url, + { + "name": "UUID Test", + "ascii": "UUID Test", + "password": "secret+password", + "password_confirmation": "secret+password", + }, + ) + created = Person.objects.get(name="UUID Test") + self.assertIsNotNone(created.primary_uuid) + self.assertEqual(created.prior_uuids, []) + + # ietf.nomcom.utils.make_nomineeposition_for_newperson + nomcom = NomComFactory(group__acronym="nomcom2021") + position = PositionFactory(nomcom=nomcom) + make_nomineeposition_for_newperson( + nomcom, + "New Nominee", + "newnominee@example.com", + position, + PersonFactory().email(), + ) + nominee_person = Person.objects.get(name="New Nominee") + self.assertIsNotNone(nominee_person.primary_uuid) + self.assertEqual(nominee_person.prior_uuids, []) + + # ietf.submit.utils.ensure_person_email_info_exists + ensure_person_email_info_exists( + "Draft Author", "draftauthor@example.com", "draft-uuid-test" + ) + author = Person.objects.get(name="Draft Author") + self.assertIsNotNone(author.primary_uuid) + self.assertEqual(author.prior_uuids, []) + + def test_factory_assigns_a_primary(self): + person = PersonFactory() + # Reads the rows directly: this is what proves the accessors below agree with + # the model, so it must not go through them + self.assertEqual(person.uuids.filter(primary=True).count(), 1) + self.assertIsNotNone(person.primary_uuid) + self.assertEqual(person.prior_uuids, []) + + def test_factory_can_skip_the_primary(self): + person = PersonFactory(primary_uuid=False) + self.assertEqual(person.uuids.count(), 0) + self.assertIsNone(person.primary_uuid) + self.assertEqual(person.prior_uuids, []) + + def test_prior_uuids_holds_the_superseded_ones_in_order(self): + person = PersonFactory() + primary = person.primary_uuid + older = PersonUUIDFactory( + person=person, time=timezone.now() - datetime.timedelta(days=2) + ) + newer = PersonUUIDFactory( + person=person, time=timezone.now() - datetime.timedelta(days=1) + ) + # The primary is not in the prior list, and the priors are oldest-first + self.assertEqual(person.primary_uuid, primary) + self.assertEqual(person.prior_uuids, [older.uuid, newer.uuid]) + + def test_assign_primary_uuid_is_idempotent(self): + person = PersonFactory() + first = person.primary_uuid + assign_primary_uuid(person) + self.assertEqual(person.uuids.count(), 1) + self.assertEqual(person.primary_uuid, first) + + def test_only_one_primary_per_person(self): + person = PersonFactory() + with self.assertRaises(IntegrityError), transaction.atomic(): + PersonUUID.objects.create(person=person, primary=True) + + def test_ensure_primary_uuid_promotes_earliest(self): + person = PersonFactory() + oldest = person.uuids.get() + PersonUUIDFactory(person=person) + person.uuids.update(primary=False) # deliberately inconsistent state + promoted = ensure_primary_uuid(person) + self.assertEqual(promoted.uuid, oldest.uuid) + self.assertEqual(person.uuids.filter(primary=True).count(), 1) + + def test_ensure_primary_uuid_creates_when_none(self): + person = PersonFactory(primary_uuid=False) + created = ensure_primary_uuid(person) + self.assertTrue(created.primary) + self.assertEqual(person.uuids.count(), 1) + + def test_get_person_uuid_object(self): + person = PersonFactory() + prior = PersonUUIDFactory(person=person) + self.assertIsNone(get_person_uuid_object(uuid.uuid4())) + primary_obj = get_person_uuid_object(person.primary_uuid) + self.assertEqual(primary_obj.person, person) + self.assertTrue(primary_obj.primary) + prior_obj = get_person_uuid_object(prior.uuid) + self.assertEqual(prior_obj.person, person) + self.assertFalse(prior_obj.primary) + + def test_deleting_a_person_deletes_its_uuids(self): + person = PersonFactory() + values = [person.primary_uuid, *person.prior_uuids] + person.delete() + self.assertEqual(PersonUUID.objects.filter(uuid__in=values).count(), 0) + for value in values: + self.assertIsNone(get_person_uuid_object(value)) + + def test_merge_chain_keeps_one_primary(self): + secretariat_role = RoleFactory(group__acronym="secretariat", name_id="secr") + request = HttpRequest() + request.user = secretariat_role.person.user + a, b, c = PersonFactory.create_batch(3) + a_uuids = {a.primary_uuid, *a.prior_uuids} + b_uuids = {b.primary_uuid, *b.prior_uuids} + c_primary = c.primary_uuid + merge_persons(request, a, b, file=StringIO()) + merge_persons(request, b, c, file=StringIO()) + c.refresh_from_db() + self.assertIsNotNone(c.primary_uuid) + self.assertEqual(c.primary_uuid, c_primary) + self.assertEqual(set(c.prior_uuids), a_uuids | b_uuids) + for value in a_uuids | b_uuids: + self.assertEqual(get_person_uuid_object(value).person, c) + + def test_merge_promotes_a_primary_for_a_target_without_one(self): + secretariat_role = RoleFactory(group__acronym="secretariat", name_id="secr") + request = HttpRequest() + request.user = secretariat_role.person.user + source = PersonFactory() + target = PersonFactory() + target.uuids.update(primary=False) # deliberately inconsistent state + merge_persons(request, source, target, file=StringIO()) + target.refresh_from_db() + self.assertIsNotNone(target.primary_uuid) + + @mock.patch("ietf.person.utils.transaction.on_commit", side_effect=lambda f: f()) + @mock.patch("ietf.person.tasks.push_person_uuids_task.apply_async") + def test_creating_a_person_does_not_push(self, mock_apply, mock_on_commit): + # A brand-new Person has no Authentik account, so there is nothing to push to. + person = PersonFactory() + self.assertFalse(mock_apply.called) + + person.name = person.name + " Jr" + person.save() + self.assertFalse(mock_apply.called) # nor does an unrelated save + + @mock.patch("ietf.person.utils.transaction.on_commit", side_effect=lambda f: f()) + @mock.patch("ietf.person.tasks.push_person_uuids_task.apply_async") + def test_push_is_dispatched_when_the_set_changes(self, mock_apply, mock_on_commit): + secretariat_role = RoleFactory(group__acronym="secretariat", name_id="secr") + request = HttpRequest() + request.user = secretariat_role.person.user + source = PersonFactory() + target = PersonFactory() + mock_apply.reset_mock() + + merge_persons(request, source, target, file=StringIO()) + self.assertTrue(mock_apply.called) + self.assertEqual( + mock_apply.call_args.kwargs["kwargs"], {"person_pk": target.pk} + ) + # Celery's default retry policy applies - short enough for the request path, + # and enough to ride out a broker blip. See queue_person_uuid_push(). + self.assertNotIn("retry", mock_apply.call_args.kwargs) + + # Promoting a primary for a Person that already existed does push + mock_apply.reset_mock() + other = PersonFactory() + other.uuids.update(primary=False) # deliberately inconsistent state + ensure_primary_uuid(other) + self.assertEqual(mock_apply.call_args.kwargs["kwargs"], {"person_pk": other.pk}) + + @mock.patch("ietf.person.utils.transaction.on_commit", side_effect=lambda f: f()) + @mock.patch("ietf.person.tasks.push_person_uuids_task.apply_async") + @mock.patch("ietf.person.utils.log.log") + def test_unreachable_broker_does_not_break_the_caller( + self, mock_log, mock_apply, mock_on_commit + ): + mock_apply.side_effect = KombuOperationalError("no broker here") + person = PersonFactory() + person.uuids.update(primary=False) # deliberately inconsistent state + ensure_primary_uuid(person) # must not raise + self.assertIsNotNone(person.primary_uuid) + self.assertIn("Could not queue UUID push", mock_log.call_args[0][0]) + + @mock.patch("ietf.person.tasks.log.log") + def test_push_person_uuids_task(self, mock_log): + person = PersonFactory() + prior = PersonUUIDFactory(person=person) + push_person_uuids_task(person_pk=person.pk) + message = mock_log.call_args[0][0] + self.assertIn(str(person.primary_uuid), message) + self.assertIn(str(prior.uuid), message) + + mock_log.reset_mock() + push_person_uuids_task(person_pk=person.pk + 10000) + self.assertIn("no such Person", mock_log.call_args[0][0]) + + @mock.patch("ietf.person.tasks.log.log") + def test_check_person_uuids_task(self, mock_log): + good = PersonFactory() + broken = PersonFactory(primary_uuid=False) + demoted = PersonFactory() + demoted.uuids.update(primary=False) # deliberately inconsistent state + + def logged(): + return "\n".join(call[0][0] for call in mock_log.call_args_list) + + self.assertEqual(check_person_uuids_task(), 2) + report = logged() + self.assertIn(f"Person {broken.pk}", report) + self.assertIn("no UUIDs at all", report) + self.assertIn(f"Person {demoted.pk}", report) + self.assertIn("no primary UUID", report) + self.assertNotIn(f"Person {good.pk} ", report) + self.assertIn("2 Person(s) need attention", report) + + mock_log.reset_mock() + self.assertEqual(check_person_uuids_task(fix=True), 2) + self.assertIn("2 Person(s) repaired", logged()) + for person in (broken, demoted): + self.assertIsNotNone(person.primary_uuid) + + mock_log.reset_mock() + self.assertEqual(check_person_uuids_task(), 0) + self.assertIn("every Person has exactly one primary UUID", logged()) + + +@override_settings( + APP_API_TOKENS={ + "ietf.person.api_uuid": ["uuid-api-token"], + "ietf.person.api_uuid_by_pk": ["by-pk-token"], + } +) +class PersonUUIDApiTests(TestCase): + def retrieve_url(self, uuid_value): + return urlreverse( + "ietf.api.person_api.person-uuid-detail", kwargs={"uuid": uuid_value} + ) + + @property + def lookup_url(self): + return urlreverse("ietf.api.person_api.person-uuid-lookup") + + @property + def by_pk_url(self): + return urlreverse("ietf.api.person_api.person-uuid-by-pk") + + def test_requires_a_valid_api_key(self): + person = PersonFactory() + url = self.retrieve_url(person.primary_uuid) + self.assertEqual(self.client.get(url).status_code, 403) + self.assertEqual( + self.client.get(url, headers={"X-Api-Key": "nope"}).status_code, 403 + ) + self.assertEqual( + self.client.get(url, headers={"X-Api-Key": "by-pk-token"}).status_code, 403 + ) + self.assertEqual( + self.client.get(url, headers={"X-Api-Key": "uuid-api-token"}).status_code, + 200, + ) + + def test_retrieve_primary(self): + person = PersonFactory() + r = self.client.get( + self.retrieve_url(person.primary_uuid), + headers={"X-Api-Key": "uuid-api-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual( + r.json(), + { + "uuid": str(person.primary_uuid), + "is_primary": True, + "primary_uuid": str(person.primary_uuid), + "prior_uuids": [], + }, + ) + + def test_retrieve_superseded(self): + person = PersonFactory() + prior = PersonUUIDFactory(person=person) + r = self.client.get( + self.retrieve_url(prior.uuid), headers={"X-Api-Key": "uuid-api-token"} + ) + self.assertEqual(r.status_code, 200) + self.assertEqual( + r.json(), + { + "uuid": str(prior.uuid), + "is_primary": False, + "primary_uuid": str(person.primary_uuid), + "prior_uuids": [str(prior.uuid)], + }, + ) + + def test_response_carries_identifiers_only(self): + person = PersonFactory() + r = self.client.get( + self.retrieve_url(person.primary_uuid), + headers={"X-Api-Key": "uuid-api-token"}, + ) + self.assertEqual( + set(r.json().keys()), + {"uuid", "is_primary", "primary_uuid", "prior_uuids"}, + ) + + def test_retrieve_unknown(self): + r = self.client.get( + self.retrieve_url(uuid.uuid4()), headers={"X-Api-Key": "uuid-api-token"} + ) + self.assertEqual(r.status_code, 404) + + def test_retrieve_upper_case(self): + person = PersonFactory() + url = self.retrieve_url(person.primary_uuid) + upper = url.replace(str(person.primary_uuid), str(person.primary_uuid).upper()) + self.assertNotEqual(url, upper) + r = self.client.get(upper, headers={"X-Api-Key": "uuid-api-token"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.json()["uuid"], str(person.primary_uuid)) + + def test_retrieve_malformed(self): + r = self.client.get( + "/api/person/uuid/not-a-uuid/", headers={"X-Api-Key": "uuid-api-token"} + ) + self.assertEqual(r.status_code, 404) + + def test_batch(self): + person = PersonFactory() + prior = PersonUUIDFactory(person=person) + missing = uuid.uuid4() + r = self.client.post( + self.lookup_url, + { + "uuids": [ + str(prior.uuid), + str(person.primary_uuid), + str(missing), + str(prior.uuid), + ] + }, + content_type="application/json", + headers={"X-Api-Key": "uuid-api-token"}, + ) + self.assertEqual(r.status_code, 200) + results = r.json()["results"] + # One entry per distinct requested UUID, duplicates collapsed + self.assertEqual(len(results), 3) + by_uuid = {entry["uuid"]: entry for entry in results} + # Same shape as a resolved entry, with the identifiers nulled out + self.assertEqual( + by_uuid[str(missing)], + { + "uuid": str(missing), + "status": "unknown", + "is_primary": None, + "primary_uuid": None, + "prior_uuids": [], + }, + ) + self.assertEqual(by_uuid[str(prior.uuid)]["status"], "resolved") + self.assertFalse(by_uuid[str(prior.uuid)]["is_primary"]) + self.assertEqual( + by_uuid[str(prior.uuid)]["primary_uuid"], str(person.primary_uuid) + ) + self.assertTrue(by_uuid[str(person.primary_uuid)]["is_primary"]) + + def test_batch_query_count_is_independent_of_size(self): + people = PersonFactory.create_batch(6) + # Resolve the UUIDs up front so the capture below sees only the API's queries + values = [str(p.primary_uuid) for p in people] + + def post(subset): + return self.client.post( + self.lookup_url, + {"uuids": subset}, + content_type="application/json", + headers={"X-Api-Key": "uuid-api-token"}, + ) + + with CaptureQueriesContext(connection) as small: + self.assertEqual(post(values[:2]).status_code, 200) + with CaptureQueriesContext(connection) as large: + self.assertEqual(post(values).status_code, 200) + self.assertEqual(len(small.captured_queries), len(large.captured_queries)) + + def test_batch_rejects_bad_input(self): + for payload in ( + {"uuids": []}, + {"uuids": ["not-a-uuid"]}, + {"uuids": [str(uuid.uuid4()) for _ in range(501)]}, + {}, + ): + r = self.client.post( + self.lookup_url, + payload, + content_type="application/json", + headers={"X-Api-Key": "uuid-api-token"}, + ) + self.assertEqual(r.status_code, 400, payload) + + def test_by_person_pk(self): + person = PersonFactory() + prior = PersonUUIDFactory(person=person) + r = self.client.post( + self.by_pk_url, + {"person_pks": [person.pk, person.pk + 10000]}, + content_type="application/json", + headers={"X-Api-Key": "by-pk-token"}, + ) + self.assertEqual(r.status_code, 200) + results = r.json()["results"] + self.assertEqual(len(results), 2) + self.assertEqual( + results[0], + { + "person_pk": person.pk, + "status": "resolved", + "primary_uuid": str(person.primary_uuid), + "prior_uuids": [str(prior.uuid)], + }, + ) + self.assertEqual( + results[1], + { + "person_pk": person.pk + 10000, + "status": "unknown", + "primary_uuid": None, + "prior_uuids": [], + }, + ) + + def test_by_person_pk_has_its_own_token(self): + person = PersonFactory() + r = self.client.post( + self.by_pk_url, + {"person_pks": [person.pk]}, + content_type="application/json", + headers={"X-Api-Key": "uuid-api-token"}, + ) + self.assertEqual(r.status_code, 403) + + def test_by_person_pk_rejects_over_cap(self): + r = self.client.post( + self.by_pk_url, + {"person_pks": list(range(501))}, + content_type="application/json", + headers={"X-Api-Key": "by-pk-token"}, + ) + self.assertEqual(r.status_code, 400) + + def test_schema(self): + r = self.client.get("/api/schema/") + self.assertEqual(r.status_code, 200) + schema = yaml.safe_load(r.content) + paths = schema["paths"] + self.assertIn("/api/person/uuid/{uuid}/", paths) + self.assertIn("/api/person/uuid/lookup/", paths) + self.assertIn("/api/person/uuid/by-person-pk/", paths) + self.assertEqual( + paths["/api/person/uuid/{uuid}/"]["get"]["operationId"], + "person_uuid_retrieve", + ) + self.assertEqual( + paths["/api/person/uuid/lookup/"]["post"]["operationId"], + "person_uuid_lookup", + ) + by_pk = paths["/api/person/uuid/by-person-pk/"]["post"] + self.assertEqual(by_pk["operationId"], "person_uuid_by_person_pk") + self.assertTrue(by_pk["deprecated"]) + self.assertIn("PersonUUIDResolution", schema["components"]["schemas"]) + for path, method in ( + ("/api/person/uuid/{uuid}/", "get"), + ("/api/person/uuid/lookup/", "post"), + ("/api/person/uuid/by-person-pk/", "post"), + ): + responses = paths[path][method]["responses"] + self.assertIn("200", responses, path) + # Consumers switch on status, so it has to be a declared, required field + for component in ("PersonUUIDBatchEntry", "PersonPkBatchEntry"): + entry = schema["components"]["schemas"][component] + self.assertIn("status", entry["required"], component) + self.assertTrue(entry["properties"]["primary_uuid"]["nullable"], component) + + class TaskTests(TestCase): @mock.patch("ietf.person.tasks.log.log") def test_purge_personal_api_key_events_task(self, mock_log): diff --git a/ietf/person/urls.py b/ietf/person/urls.py index f3eccd04b73..15338d48595 100644 --- a/ietf/person/urls.py +++ b/ietf/person/urls.py @@ -1,6 +1,8 @@ # Copyright The IETF Trust 2009-2025, All Rights Reserved # -*- coding: utf-8 -*- -from ietf.person import views, ajax +from django.urls import path + +from ietf.person import ajax, views from ietf.utils.urls import url urlpatterns = [ @@ -9,6 +11,8 @@ url(r'^merge/send_request/?$', views.send_merge_request), url(r'^search/(?P(person|email))/$', views.ajax_select2_search), url(r'^(?P[0-9]+)/email.json$', ajax.person_email_json), + path('', views.profile_by_uuid, + name='ietf.person.views.profile_by_uuid'), url(r'^(?P[^/]+)$', views.profile), url(r'^(?P[^/]+)/photo/?$', views.photo), ] diff --git a/ietf/person/utils.py b/ietf/person/utils.py index 5ed90591f9a..84b98dba2f1 100755 --- a/ietf/person/utils.py +++ b/ietf/person/utils.py @@ -9,15 +9,92 @@ from django.contrib import admin from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist +from django.db import transaction from django.db.models import Q from django.http import Http404 +from kombu.exceptions import OperationalError as KombuOperationalError import debug # pyflakes:ignore -from ietf.person.models import Person, Alias, Email +from ietf.person.models import Person, Alias, Email, PersonUUID from ietf.utils import log from ietf.utils.mail import send_mail + +def get_person_uuid_object(uuid_value): + """Look up a UUID currently issued for a Person + + Returns the PersonUUID object, whose `person` is the Person it identifies and whose + `primary` says whether it is that Person's current identifier, or None if no such + UUID exists. + """ + return PersonUUID.objects.select_related("person").filter(uuid=uuid_value).first() + + +def queue_person_uuid_push(person): + """Enqueue an Authentik attribute push for this Person's UUID set + + Called explicitly by every site that changes the set for a Person that may already + have an Authentik account. Deferred to commit so a rolled-back transaction never + pushes state that does not exist. + + Queueing is best-effort: an unreachable broker is logged and ignored rather than + failing the datatracker operation that changed the UUID set. Celery's default retry + policy applies - three attempts over well under a second, enough to ride out a blip + or a broker failover without meaningfully delaying the caller. An outright outage + still cannot fail the operation, and the reconcile job is the backstop for a push + that never got queued. + """ + from ietf.person.tasks import push_person_uuids_task # avoid a circular import + + person_pk = person.pk + + def enqueue(): + try: + push_person_uuids_task.apply_async(kwargs={"person_pk": person_pk}) + except (KombuOperationalError, OSError) as err: + log.log(f"Could not queue UUID push for Person {person_pk}: {err}") + + transaction.on_commit(enqueue) + + +def assign_primary_uuid(person): + """Give a newly created Person its primary UUID + + Idempotent: returns the existing primary if the Person already has one, so a caller + that is unsure whether an earlier step already ran can call it safely. + + Deliberately does not queue an Authentik push. A Person that has just been created + has no Authentik account, so there would be nothing to push to; its UUID reaches + Authentik when the account is linked. + """ + existing = person.uuids.filter(primary=True).first() + if existing is not None: + return existing + return PersonUUID.objects.create(person=person, primary=True) + + +def ensure_primary_uuid(person): + """Make sure a Person has exactly one primary UUID + + Promotes the earliest existing UUID if there are any but none is primary, otherwise + creates one. Returns the primary PersonUUID. + """ + existing = person.uuids.filter(primary=True).first() + if existing is not None: + return existing + oldest = person.uuids.order_by("time", "uuid").first() + if oldest is None: + row = assign_primary_uuid(person) + else: + oldest.primary = True + oldest.save(update_fields=["primary"]) + row = oldest + # Unlike a brand-new Person, this one already existed and may be linked. + queue_person_uuid_push(person) + return row + + def merge_persons(request, source, target, file=sys.stdout, verbose=False): changes = [] @@ -49,6 +126,15 @@ def merge_persons(request, source, target, file=sys.stdout, verbose=False): if reviewer_changes: changes.extend(reviewer_changes) merge_nominees(source, target) + + # Move the source's UUIDs to the target, demoting the source's primary. The target's + # primary survives; the source's identifiers keep resolving, to the target. This runs + # before move_related_objects(), which would otherwise carry the UUIDs across still + # flagged primary and trip the one-primary-per-person constraint. + ensure_primary_uuid(target) + source.uuids.update(person=target, primary=False) + queue_person_uuid_push(target) + move_related_objects(source, target, file=file, verbose=verbose) dedupe_aliases(target) @@ -131,6 +217,10 @@ def move_related_objects(source, target, file, verbose=False): and f.auto_created and not f.concrete ] for related_object in related_objects: accessor = related_object.get_accessor_name() + if accessor == "uuids": + # PersonUUIDs move by their own rule - the source's primary has to be + # demoted on the way, or the target ends up with two. See merge_persons(). + continue field_name = related_object.field.name queryset = getattr(source, accessor).all() if verbose: @@ -247,11 +337,14 @@ def get_dots(person): return dots def lookup_persons(email_or_name): - aliases = Alias.objects.filter(name__iexact=email_or_name) + aliases = Alias.objects.filter(name__iexact=email_or_name).select_related("person") persons = set(a.person for a in aliases) if '@' in email_or_name: - emails = Email.objects.filter(address__iexact=email_or_name) + # Email.address is a citext column, so an exact match is already + # case-insensitive and can use the index on it. Asking for iexact wraps the + # column in UPPER() and costs a scan of the table. + emails = Email.objects.filter(address=email_or_name).select_related("person") persons.update(e.person for e in emails) persons = [p for p in persons if p and p.id] diff --git a/ietf/person/views.py b/ietf/person/views.py index d0b5912431e..4c03f138dcc 100644 --- a/ietf/person/views.py +++ b/ietf/person/views.py @@ -7,20 +7,29 @@ from django.conf import settings from django.contrib import messages -from django.db.models import Q +from django.core.cache import caches +from django.db.models import Count, Q from django.http import HttpResponse, Http404 from django.shortcuts import render, redirect from django.template.loader import render_to_string -from django.utils import timezone import debug # pyflakes:ignore +from ietf.doc.models import DocEvent, RelatedDocument from ietf.ietfauth.utils import role_required from ietf.person.models import Email, Person from ietf.person.fields import select2_id_name_json from ietf.person.forms import MergeForm, MergeRequestForm -from ietf.person.utils import handle_users, merge_persons, lookup_persons +from ietf.person.utils import ( + get_person_uuid_object, + handle_users, + lookup_persons, + merge_persons, +) from ietf.utils.mail import send_mail_text +from ietf.utils.timezone import RPC_TZINFO + +REFERENCE_RELATIONSHIPS = ("refnorm", "refinfo", "refunk", "refold") def ajax_select2_search(request, model_name): @@ -72,9 +81,153 @@ def ajax_select2_search(request, model_name): return HttpResponse(select2_id_name_json(objs), content_type='application/json') +def rfc_rows(persons): + """Build the RFC table rows for each person + + Returns a dict keyed on person pk. The columns are gathered for every person at + once - read one at a time off the Document, each row costs a query per column. + """ + rfcs = {p.pk: p.rfcs() for p in persons} + rfc_ids = {d.pk for docs in rfcs.values() for d in docs} + + # The references of the draft an RFC was published from count as the RFC's own. + draft_of = dict( + RelatedDocument.objects.filter( + target_id__in=rfc_ids, relationship="became_rfc" + ).values_list("target_id", "source_id") + ) + referenced_by = dict( + RelatedDocument.objects.filter( + target_id__in=rfc_ids | set(draft_of.values()), + relationship__in=REFERENCE_RELATIONSHIPS, + source__type__slug="rfc", + ) + .values("target_id") + .annotate(count=Count("id")) + .values_list("target_id", "count") + ) + + # Matches Document.latest_event ordering, so the first row seen for a document + # is the one its pub_date would have reported. + published = {} + for doc_id, time in ( + DocEvent.objects.filter(doc_id__in=rfc_ids, type="published_rfc") + .order_by("-time", "-id") + .values_list("doc_id", "time") + ): + published.setdefault(doc_id, time) + + return { + pk: [ + { + "doc": doc, + "pub_date": ( + published[doc.pk].astimezone(RPC_TZINFO).date() + if doc.pk in published + else None + ), + "referenced_by": referenced_by.get(doc.pk, 0) + + referenced_by.get(draft_of.get(doc.pk), 0), + } + for doc in docs + ] + for pk, docs in rfcs.items() + } + + +def profile_data(persons): + """Build everything person/profile.html renders for each of persons""" + rfcs = rfc_rows(persons) + expired = {p.pk: list(p.expired_drafts().prefetch_related("states")) for p in persons} + replaced = set( + RelatedDocument.objects.filter( + target_id__in={d.pk for docs in expired.values() for d in docs}, + relationship="replaces", + ).values_list("target_id", flat=True) + ) + + profiles = [] + for person in persons: + # Role.Meta orders by name_id alone, which leaves ties to the query plan. + roles = sorted( + person.role_set.select_related("name", "group", "email"), + key=lambda r: (r.name_id, r.group.acronym), + ) + profiles.append( + { + "person": person, + "has_roles": bool(roles), + "roles": [ + r + for r in roles + if r.group.state_id in ["active", "bof"] + and r.group.acronym != "secretariat" + ], + "ext_resources": list( + person.personextresource_set.select_related("name") + ), + "rfcs": rfcs[person.pk], + "active_drafts": list( + person.active_drafts().prefetch_related("states") + ), + "expired_drafts": [ + d for d in expired[person.pk] if d.pk not in replaced + ], + "has_drafts": person.has_drafts(), + } + ) + return profiles + + +def profile_sections(persons): + """Render each person's part of the profile page + + The rendered sections are cached, so a repeat view of a profile - including the + revalidation a conditional request makes - costs neither the queries nor the + render. Nothing in a section is tied to the moment it was rendered, so how stale + one can be is entirely PERSON_PROFILE_CACHE_SECONDS. + """ + slowpages = caches["slowpages"] + keys = {person.pk: f"person:profile:{person.pk}" for person in persons} + sections = slowpages.get_many(list(keys.values())) + + uncached = [person for person in persons if keys[person.pk] not in sections] + for profile in profile_data(uncached): + person = profile["person"] + section = { + "id": person.pk, + "name": str(person), + "has_drafts": profile["has_drafts"], + "html": render_to_string("person/profile_body.html", {"profile": profile}), + } + slowpages.set(keys[person.pk], section, settings.PERSON_PROFILE_CACHE_SECONDS) + sections[keys[person.pk]] = section + + return [sections[keys[person.pk]] for person in persons] + + def profile(request, email_or_name): persons = lookup_persons(email_or_name) - return render(request, 'person/profile.html', {'persons': persons, 'today': timezone.now()}) + return render( + request, "person/profile.html", {"sections": profile_sections(persons)} + ) + + +def profile_by_uuid(request, uuid): + person_uuid = get_person_uuid_object(uuid) + if person_uuid is None: + raise Http404("No such person identifier") + if not person_uuid.primary: + # Self-heal a link that predates a merge by sending it to the canonical address. + return redirect( + "ietf.person.views.profile_by_uuid", + uuid=person_uuid.person.primary_uuid, + ) + return render( + request, + "person/profile.html", + {"sections": profile_sections([person_uuid.person])}, + ) def photo(request, email_or_name): diff --git a/ietf/secr/announcement/forms.py b/ietf/secr/announcement/forms.py index 3776f57a49b..e00b5af4100 100644 --- a/ietf/secr/announcement/forms.py +++ b/ietf/secr/announcement/forms.py @@ -146,25 +146,31 @@ def __init__(self, *args, **kwargs): for key in list(self.fields.keys()): self.fields[key].widget = forms.HiddenInput() + def clean_cc(self): + cc_data = self.cleaned_data["cc"] + cc = [addr.strip() for addr in cc_data.split(",") if addr.strip()] + errors = [ + forms.ValidationError( + "Invalid address: %(addr)s", "invalid_cc", {"addr": addr} + ) + for addr in cc + if not is_valid_email(addr) + ] + if errors: + raise forms.ValidationError(errors) + return cc_data + def clean(self): super(AnnounceForm, self).clean() data = self.cleaned_data if self.errors: return self.cleaned_data - if data["to"] == "Other..." and not data["to_custom"]: - raise forms.ValidationError('You must enter a "To" email address') - if data["to"] == "Other..." and data["to_custom"]: - addrlist = data["to_custom"] - else: - addrlist = [data["to"]] - if data["cc"]: - cc = [email.strip() for email in data["cc"].split(",") if email.strip()] - addrlist.extend(cc) - emails = [email.strip() for email in addrlist if email.strip()] - for email in emails: - if not is_valid_email(email): - raise forms.ValidationError("An exception occurred while trying to send email to '%s'" % email) - + + if data.get("to") == "Other..." and data.get("to_custom", []) == []: + self.add_error( + None, forms.ValidationError('Must specify a "To" address', "empty_to") + ) + for k in [ "to", "frm", diff --git a/ietf/secr/announcement/tests.py b/ietf/secr/announcement/tests.py index db2dd60ead4..ebd3a6a69aa 100644 --- a/ietf/secr/announcement/tests.py +++ b/ietf/secr/announcement/tests.py @@ -113,7 +113,8 @@ def test_valid_submit(self): self.assertRedirects(response, url) self.assertEqual(len(outbox), 1) self.assertEqual(outbox[0]["subject"], "Test Subject") - self.assertEqual(outbox[0]["to"], ", , ") + self.assertEqual(outbox[0]["to"], "") + self.assertEqual(outbox[0]["cc"], "lizz@example.com, no-brackets@example.com") message = Message.objects.filter(by__user__username="secretary").last() self.assertEqual(message.subject, "Test Subject") self.assertTrue(nomcom in message.related_groups.all()) @@ -186,4 +187,4 @@ def test_invalid_submit(self): self.assertNotContains(response, "Confirm Announcement") self.assertEqual(len(outbox), 0) - \ No newline at end of file + diff --git a/ietf/settings.py b/ietf/settings.py index 483b08bff47..3e26e22e840 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -25,7 +25,6 @@ warnings.filterwarnings("ignore", message="The USE_DEPRECATED_PYTZ setting,") # https://github.com/ietf-tools/datatracker/issues/5635 warnings.filterwarnings("ignore", message="The is_dst argument to make_aware\\(\\)") # caused by django-filters when USE_DEPRECATED_PYTZ is true warnings.filterwarnings("ignore", message="The USE_L10N setting is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5648 -warnings.filterwarnings("ignore", message="django.contrib.auth.hashers.CryptPasswordHasher is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5663 # Other DeprecationWarnings warnings.filterwarnings("ignore", message="pkg_resources is deprecated as an API", module="pyang.plugin") @@ -70,11 +69,10 @@ BUG_REPORT_EMAIL = "tools-help@ietf.org" PASSWORD_HASHERS = [ - 'django.contrib.auth.hashers.Argon2PasswordHasher', - 'django.contrib.auth.hashers.BCryptSHA256PasswordHasher', - 'django.contrib.auth.hashers.PBKDF2PasswordHasher', - 'django.contrib.auth.hashers.SHA1PasswordHasher', - 'django.contrib.auth.hashers.CryptPasswordHasher', + "django.contrib.auth.hashers.Argon2PasswordHasher", + "django.contrib.auth.hashers.BCryptSHA256PasswordHasher", + "django.contrib.auth.hashers.PBKDF2PasswordHasher", + "django.contrib.auth.hashers.SHA1PasswordHasher", ] @@ -888,6 +886,10 @@ def skip_unreadable_post(record): PDFIZER_CACHE_TIME = HTMLIZER_CACHE_TIME PDFIZER_URL_PREFIX = IDTRACKER_BASE_URL+"/doc/pdf" +# How long a rendered person profile section is served from the slowpages cache. +# This is how stale a profile's roles and documents can be. +PERSON_PROFILE_CACHE_SECONDS = 60*15 # 15 minutes + # Email settings IPR_EMAIL_FROM = 'ietf-ipr@ietf.org' AUDIO_IMPORT_EMAIL = ['ietf@meetecho.com'] @@ -1291,22 +1293,6 @@ def skip_unreadable_post(record): UTILS_APIKEY_GUI_LOGIN_LIMIT_DAYS = 30 -API_KEY_TYPE="ES256" # EC / P=256 -API_PUBLIC_KEY_PEM = b""" ------BEGIN PUBLIC KEY----- -MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEqVojsaofDJScuMJN+tshumyNM5ME -garzVPqkVovmF6yE7IJ/dv4FcV+QKCtJ/rOS8e36Y8ZAEVYuukhes0yZ1w== ------END PUBLIC KEY----- -""" -API_PRIVATE_KEY_PEM = b""" ------BEGIN PRIVATE KEY----- -MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgoI6LJkopKq8XrHi9 -QqGQvE4A83TFYjqLz+8gULYecsqhRANCAASpWiOxqh8MlJy4wk362yG6bI0zkwSB -qvNU+qRWi+YXrITsgn92/gVxX5AoK0n+s5Lx7fpjxkARVi66SF6zTJnX ------END PRIVATE KEY----- -""" - - # Default timeout for HTTP requests via the requests library DEFAULT_REQUESTS_TIMEOUT = 20 # seconds diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 457462e4f2c..b331d71631f 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -60,6 +60,7 @@ from ietf.utils.timezone import date_today from ietf.utils.xmldraft import InvalidMetadataError, XMLDraft, capture_xml2rfc_output from ietf.person.name import unidecode_name +from ietf.person.utils import assign_primary_uuid def validate_submission(submission): @@ -589,7 +590,11 @@ def ensure_person_email_info_exists(name, email, docname): person.name_from_draft = name log.assertion('isinstance(person.name, str)') person.ascii = unidecode_name(person.name) - person.save() + # Atomic so a Person is never left without the primary UUID that external + # systems need to name them by. + with transaction.atomic(): + person.save() + assign_primary_uuid(person) else: person.name_from_draft = name diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index 24d3c77b3be..3a037667e28 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -11,8 +11,17 @@ from django.conf import settings from django.utils import timezone - -from ietf.doc.models import DocEvent, DocTagName, Document, RelatedDocument, RpcAssignmentDocEvent, State +from django.utils.dateparse import parse_datetime + +from ietf.doc.models import ( + DocEvent, + DocTagName, + Document, + RelatedDocument, + RpcActionHolderOpenEntry, + RpcAssignmentDocEvent, + State, +) from ietf.doc.tasks import rebuild_reference_relations_task from ietf.doc.utils import add_state_change_event, new_state_change_event, update_action_holders from ietf.person.models import Person @@ -382,6 +391,71 @@ def format_rpc_queue_status(obj): return ", ".join(parts) +# The datatracker's "(System)" person. The RPC tool substitutes a placeholder +# person of its own whenever no real person was named, and sends that person's +# datatracker id - so an action holder must never be associated with this pk. +# Which pk the RPC tool uses is configurable at its end, hence the separate +# check against the "(System)" person the datatracker actually has. +SYSTEM_PERSON_ID = 1 + + +def _rpc_action_holder_person(holder, system_person): + """Resolve the datatracker Person for one action holder, if there is one + + Returns None whenever the entry does not name a person the datatracker can + act on. The RPC tool substitutes its system person when no real person was + named, and that placeholder must never become an action holder here. Do not + use "body" to make this decision - the RPC tool's edit path can set or clear + "body" without touching the person, so it is unreliable in both directions. + """ + person_id = (holder.get("person") or {}).get("person_id") + if person_id in (SYSTEM_PERSON_ID, system_person.pk): + return None + if holder.get("body"): + return None # a body holds the action, not a person + person = Person.objects.filter(pk=person_id).first() + if person is None: + log.log( + f"process_rpc_queue_task: unknown action holder person {person_id}" + ) + return None + return person + + +def _sync_rpc_action_holders(doc, holders, rfc_number, system_person): + """Reconcile the open action holder entries for one document""" + open_purple_ids = [] + for holder in holders: + if holder.get("completed") is not None: + # The RPC tool reports completed action holders as well as open + # ones. Only open ones are kept here, so this line is the only + # place the datatracker sees a completion happen. + continue + RpcActionHolderOpenEntry.objects.update_or_create( + purple_id=holder["id"], + defaults=dict( + document=doc, + person=_rpc_action_holder_person(holder, system_person), + body=holder.get("body") or "", + display_name=holder.get("display_name") or "", + comment=holder.get("comment") or "", + rfc_number=rfc_number, + since_when=parse_datetime(holder["since_when"]), + deadline=( + parse_datetime(holder["deadline"]) + if holder.get("deadline") + else None + ), + ), + ) + open_purple_ids.append(holder["id"]) + # Anything else we were holding for this document has been completed or + # removed at the RPC tool. + RpcActionHolderOpenEntry.objects.filter(document=doc).exclude( + purple_id__in=open_purple_ids + ).delete() + + @shared_task def process_rpc_queue_task(data: list): in_progress_state = State.objects.get( @@ -485,6 +559,10 @@ def process_rpc_queue_task(data: list): d.tags.remove(*iana_ref_tags) + _sync_rpc_action_holders( + d, obj.get("actionholder_set") or [], rfc_number, system + ) + if events: d.save_with_history(events) @@ -495,3 +573,4 @@ def process_rpc_queue_task(data: list): ): d.tags.remove(*iana_ref_tags) d.unset_state("draft-rfceditor") + RpcActionHolderOpenEntry.objects.filter(document=d).delete() diff --git a/ietf/sync/tests_tasks.py b/ietf/sync/tests_tasks.py index 264a5f46bb3..7f0c5aee960 100644 --- a/ietf/sync/tests_tasks.py +++ b/ietf/sync/tests_tasks.py @@ -1,5 +1,6 @@ # Copyright The IETF Trust 2026, All Rights Reserved +import datetime import mock from django.test.utils import override_settings @@ -9,9 +10,11 @@ DocTagName, Document, DocumentURL, + RpcActionHolderOpenEntry, RpcAssignmentDocEvent, State, ) +from ietf.person.factories import PersonFactory from ietf.person.models import Person from ietf.sync import tasks from ietf.utils.mail import outbox @@ -19,7 +22,12 @@ def _make_entry( - doc_name, roles=None, blocking_reasons=None, rfc_number=None, final_approval=None + doc_name, + roles=None, + blocking_reasons=None, + rfc_number=None, + final_approval=None, + action_holders=None, ): return { "name": doc_name, @@ -27,6 +35,23 @@ def _make_entry( "blocking_reasons": blocking_reasons or [], "rfc_number": rfc_number, "final_approval": final_approval or [], + "actionholder_set": action_holders or [], + } + + +def _make_action_holder( + purple_id=1, person_id=None, body="", completed=None, deadline=None, comment="" +): + """Build one actionholder_set element as the RPC tool sends it""" + return { + "id": purple_id, + "person": {"person_id": person_id, "name": "Anybody"}, + "display_name": body or "Anybody", + "deadline": deadline, + "since_when": "2026-08-11T12:00:00Z", + "completed": completed, + "comment": comment, + "body": body, } @@ -481,6 +506,136 @@ def test_docs_in_queue_retain_rfceditor_state(self): self.assertIsNotNone(draft.get_state("draft-rfceditor")) +class RpcActionHolderSyncTests(TestCase): + def setUp(self): + super().setUp() + self.draft = WgDraftFactory(states=[("draft-iesg", "rfcqueue")]) + self.person = PersonFactory() + + def _sync(self, *holders, rfc_number=None): + tasks.process_rpc_queue_task( + [ + _make_entry( + self.draft.name, + roles=["first_editor"], + rfc_number=rfc_number, + action_holders=list(holders), + ) + ] + ) + return RpcActionHolderOpenEntry.objects.filter(document=self.draft) + + def test_person_holder_is_stored(self): + """An action holder naming a person is kept, with that person.""" + entries = self._sync( + _make_action_holder( + purple_id=17, + person_id=self.person.pk, + comment="Please confirm the change in section 4.2.", + deadline="2026-09-01T12:00:00Z", + ) + ) + self.assertEqual(entries.count(), 1) + entry = entries.first() + self.assertEqual(entry.purple_id, 17) + self.assertEqual(entry.person, self.person) + self.assertEqual(entry.body, "") + self.assertEqual(entry.comment, "Please confirm the change in section 4.2.") + self.assertEqual(entry.since_when.date(), datetime.date(2026, 8, 11)) + self.assertEqual(entry.deadline.date(), datetime.date(2026, 9, 1)) + + def test_body_holder_is_stored_without_a_person(self): + """An action held by a body is kept, but belongs to nobody.""" + entries = self._sync(_make_action_holder(person_id=self.person.pk, body="IANA")) + self.assertEqual(entries.count(), 1) + self.assertIsNone(entries.first().person) + self.assertEqual(entries.first().body, "IANA") + + def test_system_person_id_is_never_an_action_holder(self): + """The RPC tool's placeholder person never becomes an action holder. + + It arrives both with and without a body set - the RPC tool's edit path + can clear the body without changing the person - so the person id alone + has to be enough to reject it. + """ + for body in ("IANA", ""): + RpcActionHolderOpenEntry.objects.all().delete() + entries = self._sync( + _make_action_holder(person_id=tasks.SYSTEM_PERSON_ID, body=body) + ) + self.assertEqual(entries.count(), 1, f"body={body!r}") + self.assertIsNone(entries.first().person, f"body={body!r}") + + def test_person_resolving_to_system_is_never_an_action_holder(self): + """A person id that resolves to (System) is rejected on its own merits.""" + system = Person.objects.get(name="(System)") + entries = self._sync(_make_action_holder(person_id=system.pk)) + self.assertEqual(entries.count(), 1) + self.assertIsNone(entries.first().person) + + def test_unknown_person_is_stored_without_a_person(self): + """An unresolvable person is logged, and the entry kept and legible.""" + entries = self._sync(_make_action_holder(person_id=99999999)) + self.assertEqual(entries.count(), 1) + self.assertIsNone(entries.first().person) + self.assertEqual(entries.first().display_name, "Anybody") + + def test_completed_holder_is_not_stored(self): + """A completed action holder is dropped rather than kept.""" + entries = self._sync( + _make_action_holder( + person_id=self.person.pk, completed="2026-08-14T12:00:00Z" + ) + ) + self.assertEqual(entries.count(), 0) + + def test_completion_removes_a_stored_entry(self): + """An entry the RPC tool completes stops being held.""" + self._sync(_make_action_holder(purple_id=17, person_id=self.person.pk)) + entries = self._sync( + _make_action_holder( + purple_id=17, + person_id=self.person.pk, + completed="2026-08-14T12:00:00Z", + ) + ) + self.assertEqual(entries.count(), 0) + + def test_entry_dropped_from_the_payload_is_removed(self): + """An entry the RPC tool deletes stops being held.""" + self._sync(_make_action_holder(purple_id=17, person_id=self.person.pk)) + self.assertEqual(self._sync().count(), 0) + + def test_entries_are_removed_when_the_document_leaves_the_queue(self): + """Nothing is held for a document the RPC tool no longer reports.""" + self._sync(_make_action_holder(person_id=self.person.pk)) + tasks.process_rpc_queue_task([]) + self.assertFalse(RpcActionHolderOpenEntry.objects.exists()) + + def test_rfc_number_comes_from_the_queue_entry(self): + """The rfc number is captured from the entry, not the action holder.""" + entries = self._sync( + _make_action_holder(person_id=self.person.pk), rfc_number=9850 + ) + self.assertEqual(entries.first().rfc_number, 9850) + + def test_rfc_number_is_null_before_one_is_assigned(self): + """A document can hold an action before it has an rfc number.""" + entries = self._sync(_make_action_holder(person_id=self.person.pk)) + self.assertIsNone(entries.first().rfc_number) + + def test_existing_entry_is_updated_in_place(self): + """A second push updates the entry it already holds.""" + self._sync(_make_action_holder(purple_id=17, person_id=self.person.pk)) + entries = self._sync( + _make_action_holder( + purple_id=17, person_id=self.person.pk, comment="Now with a comment" + ) + ) + self.assertEqual(entries.count(), 1) + self.assertEqual(entries.first().comment, "Now with a comment") + + class FormatRpcQueueStatusTests(TestCase): """Unit tests for the queue "Status" renderer, mirroring the ietf-tools/queue site.""" diff --git a/ietf/templates/api/index.html b/ietf/templates/api/index.html index e21a50101ad..86bfcc46011 100644 --- a/ietf/templates/api/index.html +++ b/ietf/templates/api/index.html @@ -278,45 +278,5 @@

a regular login in order to activate access. -

- Signing Keys -

-

- When sending notifications to other APIs, the datatracker may sign - information with a - - RFC - 7515: JSON Web Signature (JWS) - , - using a public/private keypair with - this public key: -

-

- {{ key.export_public }} -

-

- or alternatively: -

- 
{{ key.export_to_pem }}
-

- To verify a signature and get the verified data using Python with the - - jwcrypto - - module, - you could do: -

- 
-from jwcrypto import jwk, jws
-
-# ... receive json web signed data as 'data', used below ...
-
-key = jwk.JWK()
-key.import_from_pem(API_PUBLIC_KEY_PEM)   # the key above
-jwstoken = jws.JWS()
-jwstoken.deserialize(data)
-jwstoken.verify(key)
-payload = jwstoken.payload
-
{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html index cac709021ea..cb80d8977e6 100644 --- a/ietf/templates/doc/ad_list.html +++ b/ietf/templates/doc/ad_list.html @@ -8,8 +8,8 @@ {% endblock %} {% block morecss %} table .border-bottom { border-bottom-color: var(--highcharts-neutral-color-80) !important; } - .highcharts-container .highcharts-axis-labels { - font-size: .7rem; + .highcharts-container .highcharts-axis-labels { + font-size: .7rem; fill: var(--bs-body-color) } .highcharts-container .highcharts-graph { stroke-width: 2.5; } @@ -37,6 +37,29 @@

IESG Dashboard

Documents in IESG Processing IESG view of Working Groups

+ {% if rpc_pending %} +

RPC decisions pending

+

+ Documents in the RFC Editor queue for which the RPC is currently waiting on a + decision from an Area Director. +

+ + + + + + + + + {% for row in rpc_pending %} + + + + + {% endfor %} + +
Area DirectorDecisions pending
{{ row.ad.plain_name }}{{ row.count }}
+ {% endif %} {% for dt in metadata %}

{{ dt.type.1 }} State Counts

@@ -45,7 +68,7 @@

{{ dt.type.1 }} State Counts

{% if dt.type.1 == "Internet-Draft" %} - {% endif %} + {% endif %} {% for state, state_name in dt.states %} + {% if rpc_action_holders %} + + + + + + + {% endif %}
Area DirectorPre pubreq @@ -72,7 +95,7 @@

{{ dt.type.1 }} State Counts

> {{ ad.pre_pubreq }} - {% endif %} + {% endif %} {% for state, state_name in dt.states %} 		@@ -130,7 +153,8 @@

{{ dt.type.1 }} State Counts

renderTo: element, panning: { enabled: false }, spacing: [4, 0, 5, 0], - height: "45%" + height: "45%", + reflow: false // reflow causes panels to grow on Firefox }, scrollbar: { enabled: false }, tooltip: { enabled: false }, @@ -201,7 +225,7 @@

{{ dt.type.1 }} State Counts

const sums = Array.from(table.querySelectorAll("[data-sum]")).reduce(( sumsAccumulator, cell - ) => { + ) => { const key = cell.dataset.sum const value = safeParseFloat(cell.textContent) if(key && !isNaN(value)) { @@ -290,4 +314,26 @@

{{ dt.type.1 }} State Counts

}) }) + {% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index a39fdf6eb73..543b5606430 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -592,6 +592,27 @@ {{ rfc_editor_queue_status|default:rfc_editor_state }}
+ Action Holders + + + {% for entry in rpc_action_holders %} +
+ {% person_link entry.person %} + since {{ entry.since_when|date:"Y-m-d" }} + {% if show_rpc_action_holder_comments and entry.comment %} +
{{ entry.comment }}
+ {% endif %} +
+ {% endfor %} +
diff --git a/ietf/templates/doc/drafts_for_ad.html b/ietf/templates/doc/drafts_for_ad.html index 18e5960e813..fcaf748b2b9 100644 --- a/ietf/templates/doc/drafts_for_ad.html +++ b/ietf/templates/doc/drafts_for_ad.html @@ -80,6 +80,39 @@

Missing ballot positions for {{ ad.name }}
{% endif %} + {% if rpc_action_holders %} +

RPC decisions pending from {{ ad.name }}

+ + + + + + + + + + + + {% for entry in rpc_action_holders %} + + + + + + + + {% endfor %} + +
DocumentQueue entryWaiting sinceDeadlineRequest
{{ entry.document.displayname_with_link }} + {% if entry.final_review_url %} + Final review + {% endif %} + {{ entry.since_when|date:"Y-m-d" }} + {% if entry.deadline %} + {{ entry.deadline|date:"Y-m-d" }} + {% endif %} + {{ entry.comment }}
+ {% endif %}

Documents for {{ ad.name }}

{% include "doc/search/search_results.html" with start_table=True end_table=True %} {% endblock %} diff --git a/ietf/templates/doc/search/status_columns.html b/ietf/templates/doc/search/status_columns.html index 50d70fcb379..353abf917d1 100644 --- a/ietf/templates/doc/search/status_columns.html +++ b/ietf/templates/doc/search/status_columns.html @@ -5,16 +5,17 @@
{% ballot_icon doc %}
{% if not doc.type_id == "rfc" %} - {% if '::' in doc.friendly_state %} - {{ doc.friendly_state|safe }} - {% else %} - {{ doc.friendly_state|safe }} - {% endif %} - {% if doc|state:"draft-rfceditor" %} - : {{ doc|state:"draft-rfceditor" }} - {% endif %} + {{ doc.friendly_state|safe }} {{ doc|auth48_alert_badge }} {{ doc|state_age_colored }} + {# The RFC Editor runs a state machine of its own, so its status is labeled rather than read as a substate of the state above. #} + {% with rfc_editor_state=doc|state:"draft-rfceditor" %} + {% if rfc_editor_state %} +
+ RFC Editor: + {{ doc.rfc_editor_queue_status|default:rfc_editor_state }} + {% endif %} + {% endwith %} {% if doc.telechat_date %}
IESG telechat: {{ doc.telechat_date }} diff --git a/ietf/templates/group/meetings.html b/ietf/templates/group/meetings.html index 30f478da131..e8511539cc1 100644 --- a/ietf/templates/group/meetings.html +++ b/ietf/templates/group/meetings.html @@ -1,4 +1,4 @@ -{# Copyright The IETF Trust 2025, All Rights Reserved #} +{# Copyright The IETF Trust 2026, All Rights Reserved #} {% extends "group/group_base.html" %} {% load origin static %} {% block title %} @@ -38,16 +38,19 @@

Meetings in progress

{% endwith %} {% endif %} - {% if future %} + {% if future or pending_interims %}

Future Meetings - {% for cal_action in cal_actions %} - - {{ cal_action.label }} - - {% endfor %} + {% if future %} + {% for cal_action in cal_actions %} + + {{ cal_action.label }} + + {% endfor %} + {% endif %}

+ {% if future %} @@ -63,6 +66,14 @@

{% endwith %}

+ {% endif %} + {% if pending_interims %} +

+ One or more {{ group.acronym }} interim meetings are + pending approval or + waiting to be announced. +

+ {% endif %} {% endif %} {% if past or recent %}

Past Meetings (within the last four years)

diff --git a/ietf/templates/meeting/proceedings_attendees.html b/ietf/templates/meeting/proceedings_attendees.html index 0c59d4ab155..402b807223b 100644 --- a/ietf/templates/meeting/proceedings_attendees.html +++ b/ietf/templates/meeting/proceedings_attendees.html @@ -80,7 +80,7 @@

- {{ person.name }} - {% if person.ascii != person.name %} -
- ({{ person.ascii }}) - {% endif %} - {% if person.pronouns %} -
- Pronouns: {{person.pronouns}} - {% endif %} -

-
- {% if person.photo %} -
{% include "person/photo.html" with person=person %}
- {% endif %} - {{ person.biography|apply_markup:"restructuredtext"|urlize_ietf_docs|linkify }} -
- {% if person.role_set.exists %} -

Roles

- {% if person.role_set.all|active_roles %} - - - - - - - - - - {% for role in person.role_set.all|active_roles %} - - - - - - {% endfor %} - -
RoleGroupEmail
{{ role.name.name }} - {% if role.name.name == 'Reviewer' %} - (See reviews) - {% endif %} - - {{ role.group.name }} - ({{ role.group.acronym }}) - - {{ role.email.address }} -
- {% else %} -

{{ person.first_name }} has no active roles as of {{ today|date:"Y-m-d" }}.

- {% endif %} - {% endif %} - {% if person.personextresource_set.exists %} -

External Resources

- - - - - - - - - {% for extres in person.personextresource_set.all %} - - - - - {% endfor %} - -
NameValue
- {% firstof extres.display_name extres.name.name %} - {{ extres.value|linkify }}
- {% endif %} -

- RFCs ({{ person.rfcs|length }}) -

- {% if person.rfcs %} - - - - - - - - - - - {% for doc in person.rfcs %} - - - - - - - {% endfor %} - -
RFCDateTitleCited by
- RFC {{ doc.rfc_number }} - {{ doc.pub_date|date:"b Y"|title }}{{ doc.title|urlize_ietf_docs }} - {% with doc.referenced_by_rfcs_as_rfc_or_draft.count as refbycount %} - {% if refbycount %} - - {{ refbycount }} RFC{{ refbycount|pluralize }} - - {% endif %} - {% endwith %} -
- {% else %} - {{ person.first_name }} has no RFCs as of {{ today|date:"Y-m-d" }}. - {% endif %} -

- Active Internet-Drafts ({{ person.active_drafts|length }}) -

- {% if person.active_drafts.exists %} -
    - {% for doc in person.active_drafts %} -
  • - {{ doc.name }} -
    • - {% endfor %} -
- {% else %} - {{ person.first_name }} has no active Internet-Drafts as of {{ today|date:"Y-m-d" }}. - {% endif %} -

- Expired Internet-Drafts ({{ person.expired_drafts|length }}) -

- {% if person.expired_drafts.exists %} -
    - {% for doc in person.expired_drafts %} - {% if not doc.replaced_by %} -
  • - - {{ doc.name }} - -
    • - {% endif %} - {% endfor %} -
- (Excluding replaced Internet-Drafts.) - {% else %} - {{ person.first_name }} has no expired Internet-Drafts as of {{ today|date:"Y-m-d" }}. - {% endif %} - {% if person.has_drafts %} -

- Internet-Draft Activity -

-
-
- {% endif %} + {{ section.html }} {% endfor %} {% endblock %} {% block js %} @@ -177,13 +24,13 @@

-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/templates/person/profile_body.html b/ietf/templates/person/profile_body.html new file mode 100644 index 00000000000..ff9970b1fb6 --- /dev/null +++ b/ietf/templates/person/profile_body.html @@ -0,0 +1,152 @@ +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} +{% load markup_tags %} +{% load ietf_filters textfilters %} +{% with person=profile.person %} +

+ {{ person.name }} + {% if person.ascii != person.name %} +
+ ({{ person.ascii }}) + {% endif %} + {% if person.pronouns %} +
+ Pronouns: {{person.pronouns}} + {% endif %} +

+
+ {% if person.photo %} +
{% include "person/photo.html" with person=person %}
+ {% endif %} + {{ person.biography|apply_markup:"restructuredtext"|urlize_ietf_docs|linkify }} +
+ {% if profile.has_roles %} +

Roles

+ {% if profile.roles %} + + + + + + + + + + {% for role in profile.roles %} + + + + + + {% endfor %} + +
RoleGroupEmail
{{ role.name.name }} + {% if role.name.name == 'Reviewer' %} + (See reviews) + {% endif %} + + {{ role.group.name }} + ({{ role.group.acronym }}) + + {{ role.email.address }} +
+ {% else %} +

{{ person.first_name }} does not currently have any active roles.

+ {% endif %} + {% endif %} + {% if profile.ext_resources %} +

External Resources

+ + + + + + + + + {% for extres in profile.ext_resources %} + + + + + {% endfor %} + +
NameValue
+ {% firstof extres.display_name extres.name.name %} + {{ extres.value|linkify }}
+ {% endif %} +

+ RFCs ({{ profile.rfcs|length }}) +

+ {% if profile.rfcs %} + + + + + + + + + + + {% for row in profile.rfcs %} + + + + + + + {% endfor %} + +
RFCDateTitleCited by
+ RFC {{ row.doc.rfc_number }} + {{ row.pub_date|date:"b Y"|title }}{{ row.doc.title|urlize_ietf_docs }} + {% if row.referenced_by %} + + {{ row.referenced_by }} RFC{{ row.referenced_by|pluralize }} + + {% endif %} +
+ {% else %} + {{ person.first_name }} has no RFCs. + {% endif %} +

+ Active Internet-Drafts ({{ profile.active_drafts|length }}) +

+ {% if profile.active_drafts %} +
    + {% for doc in profile.active_drafts %} +
  • + {{ doc.name }} +
    • + {% endfor %} +
+ {% else %} + {{ person.first_name }} has no active Internet-Drafts. + {% endif %} +

+ Expired Internet-Drafts ({{ profile.expired_drafts|length }}) +

+ {% if profile.expired_drafts %} + + (Excluding replaced Internet-Drafts.) + {% else %} + {{ person.first_name }} has no expired Internet-Drafts. + {% endif %} + {% if profile.has_drafts %} +

+ Internet-Draft Activity +

+
+
+ {% endif %} +{% endwith %} diff --git a/ietf/urls.py b/ietf/urls.py index e822b2042ef..61ab4b2de88 100644 --- a/ietf/urls.py +++ b/ietf/urls.py @@ -6,7 +6,7 @@ from django.contrib.sitemaps import views as sitemap_views from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.http import HttpResponse -from django.urls import include, path +from django.urls import include, path, register_converter from django.views import static as static_view from django.views.generic import TemplateView from django.views.defaults import server_error @@ -17,9 +17,16 @@ from ietf.group.urls import group_urls, grouptype_urls, stream_urls from ietf.ipr.sitemaps import IPRMap from ietf.liaisons.sitemaps import LiaisonMap +from ietf.utils.converters import AnyCaseUUIDConverter from ietf.utils.urls import url +# Register path converters here, in the root URLconf, before urlpatterns names any of +# them. Django refuses to register a converter twice, so registering at the point of +# definition would make importing that module from two URLconfs an error. +register_converter(AnyCaseUUIDConverter, "anycase_uuid") + + # sometimes, this code gets called more than once, which is an # that seems impossible to work around. try: diff --git a/ietf/utils/converters.py b/ietf/utils/converters.py new file mode 100644 index 00000000000..d4ebbbb42b3 --- /dev/null +++ b/ietf/utils/converters.py @@ -0,0 +1,21 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +"""URL path converters + +Registered in the root URLconf, not here - Django does not allow registering a converter +twice, so a module-level register_converter() would break as soon as two URLconfs +imported this module. +""" + +from django.urls.converters import UUIDConverter + + +class AnyCaseUUIDConverter(UUIDConverter): + """UUIDConverter that also accepts upper-case hex + + Django's built-in "uuid" converter matches lower case only, so an upper-cased + identifier would 404 rather than resolve. + """ + + regex = ( + "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" + ) diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index c878ba49f10..9ba3c79edd9 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -245,6 +245,17 @@ def create_default_tasks(self): ), ) + PeriodicTask.objects.get_or_create( + name="Check Person UUIDs", + task="ietf.person.tasks.check_person_uuids_task", + kwargs=json.dumps({"fix": False}), + defaults={ + "enabled": False, + "crontab": self.crontabs["daily"], + "description": "Report Persons with no primary UUID", + }, + ) + PeriodicTask.objects.get_or_create( name="Run Yang model checks", task="ietf.submit.tasks.run_yang_model_checks_task", diff --git a/ietf/utils/test_data.py b/ietf/utils/test_data.py index c5d34727511..f7c4bb3efd4 100644 --- a/ietf/utils/test_data.py +++ b/ietf/utils/test_data.py @@ -22,6 +22,7 @@ from ietf.group.utils import setup_default_community_list_for_group from ietf.review.models import (ReviewRequest, ReviewerSettings, ReviewResultName, ReviewTypeName, ReviewTeamSettings ) from ietf.person.name import unidecode_name +from ietf.person.utils import assign_primary_uuid from ietf.utils.timezone import date_today @@ -40,6 +41,7 @@ def create_person(group, role_name, name=None, username=None, email_address=None user.set_password(password) user.save() person = Person.objects.create(name=name, ascii=unidecode_name(smart_str(name)), user=user) + assign_primary_uuid(person) email = Email.objects.create(address=email_address, person=person, origin=user.username) Role.objects.create(group=group, name_id=role_name, person=person, email=email) return person @@ -63,6 +65,7 @@ def make_immutable_base_data(): # system system_person = Person.objects.create(name="(System)", ascii="(System)") + assign_primary_uuid(system_person) Email.objects.create(address="", person=system_person, origin='test') # high-level groups @@ -120,6 +123,7 @@ def make_immutable_base_data(): for i in range(1, 10): u = User.objects.create(username="ad%s" % i) p = Person.objects.create(name="Ad No%s" % i, ascii="Ad No%s" % i, user=u) + assign_primary_uuid(p) email = Email.objects.create(address="ad%s@example.org" % i, person=p, origin=u.username) if i < 6: # active @@ -148,6 +152,7 @@ def make_immutable_base_data(): for i in range(1, 5): u = User.objects.create(username="irsgmember%s" % i) p = Person.objects.create(name="IRSG Member No%s" % i, ascii="IRSG Member No%s" % i, user=u) + assign_primary_uuid(p) email = Email.objects.create(address="irsgmember%s@example.org" % i, person=p, origin=u.username) Role.objects.create(name_id="member", group=irsg, person=p, email=email) @@ -250,6 +255,7 @@ def make_test_data(): u.set_password("plain+password") u.save() plainman = Person.objects.create(name="Plain Man", ascii="Plain Man", user=u) + assign_primary_uuid(plainman) email = Email.objects.create(address="plain@example.com", person=plainman, origin=u.username) # group personnel @@ -436,6 +442,7 @@ def make_review_data(doc): u.set_password("reviewer+password") u.save() reviewer = Person.objects.create(name="Some Réviewer", ascii="Some Reviewer", user=u) + assign_primary_uuid(reviewer) email = Email.objects.create(address="reviewer@example.com", person=reviewer, origin=u.username) for team in (team1, team2, team3): @@ -459,6 +466,7 @@ def make_review_data(doc): u.set_password("reviewsecretary+password") u.save() reviewsecretary = Person.objects.create(name="Réview Secretary", ascii="Review Secretary", user=u) + assign_primary_uuid(reviewsecretary) reviewsecretary_email = Email.objects.create(address="reviewsecretary@example.com", person=reviewsecretary, origin=u.username) Role.objects.create(name_id="secr", person=reviewsecretary, email=reviewsecretary_email, group=team1) @@ -466,6 +474,7 @@ def make_review_data(doc): u.set_password("reviewsecretary3+password") u.save() reviewsecretary3 = Person.objects.create(name="Réview Secretary3", ascii="Review Secretary3", user=u) + assign_primary_uuid(reviewsecretary3) reviewsecretary3_email = Email.objects.create(address="reviewsecretary3@example.com", person=reviewsecretary, origin=u.username) Role.objects.create(name_id="secr", person=reviewsecretary3, email=reviewsecretary3_email, group=team3) diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 0a46fdf807a..f0245237670 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -251,6 +251,34 @@ def load_and_run_fixtures(verbosity): fn = getattr(module, components[-1]) fn() + check_base_data_person_uuids() + + +def check_base_data_person_uuids(): + """Every Person in the base test data must have exactly one primary UUID + + UUIDs are assigned by an explicit assign_primary_uuid() call at each site that + creates a Person (see ietf.person.utils), so a new site added without one would + otherwise go unnoticed until something asked for the Person's identifier. + """ + from django.db.models import Count, Q + + from ietf.person.models import Person + + broken = list( + Person.objects.annotate( + primary_count=Count("uuids", filter=Q(uuids__primary=True), distinct=True) + ) + .exclude(primary_count=1) + .values_list("pk", "name", "primary_count")[:10] + ) + if broken: + raise RuntimeError( + "Base test data has Persons without exactly one primary UUID - a Person " + "with none needs an assign_primary_uuid() call where it is created: " + + ", ".join(f"{pk} ({name}): {n} primary" for pk, name, n in broken) + ) + def safe_create_test_db(self, verbosity, *args, **kwargs): if old_create is None: raise RuntimeError("old_create has not been set, cannot proceed") diff --git a/k8s/README.md b/k8s/README.md index 3966101ab80..649e4058dd9 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -2,4 +2,5 @@ ## Run locally -The `secrets.yaml` file is provided as a reference only and must be referenced manually in the `kustomization.yaml` file. \ No newline at end of file +The `secrets.yaml.example` file is provided as a reference. To use it, rename it to `secrets.yaml`, update its contents, and reference it in the `kustomization.yaml` file. + diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 2a96ab63bce..5183893bc8c 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -3,7 +3,7 @@ kind: Deployment metadata: name: datatracker spec: - replicas: 2 + replicas: 1 revisionHistoryLimit: 2 selector: matchLabels: diff --git a/k8s/secrets.yaml b/k8s/secrets.yaml.example similarity index 92% rename from k8s/secrets.yaml rename to k8s/secrets.yaml.example index 4d326521467..70931018a8e 100644 --- a/k8s/secrets.yaml +++ b/k8s/secrets.yaml.example @@ -13,7 +13,7 @@ stringData: Nicolas Giard DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed # DATATRACKER_DATATRACKER_DEBUG: "false" - + # DB access details - needs to be filled in # DATATRACKER_DB_HOST: "db" # DATATRACKER_DB_PORT: "5432" @@ -22,15 +22,15 @@ stringData: # DATATRACKER_DB_PASS: "RkTkDPFnKpko" # secret # DATATRACKER_DB_CONN_MAX_AGE: "0" # connection per request if not set, no limit if set to "None" # DATATRACKER_DB_CONN_HEALTH_CHECKS: "false" - + DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret # Set this to point testing / staging at the production statics server until we # sort that out # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" - + # DATATRACKER_EMAIL_DEBUG: "true" - + # Outgoing email details # DATATRACKER_EMAIL_HOST: "localhost" # defaults to localhost # DATATRACKER_EMAIL_PORT: "2025" # defaults to 2025 @@ -51,7 +51,7 @@ stringData: ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo= # secret - value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- + DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1 F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY @@ -71,12 +71,16 @@ stringData: # Only one of these may be set # DATATRACKER_APP_API_TOKENS_JSON_B64: "e30K" # secret - # DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret + # DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret # use this to override default - one entry per line # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- # https://datatracker.staging.ietf.org + # How long a rendered person profile section is cached, in seconds. + # Defaults to 900. Set to 0 to serve every profile fresh. + # DATATRACKER_PERSON_PROFILE_CACHE_SECONDS: "900" + # Scout configuration DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" DATATRACKER_SCOUT_NAME: "StagingDatatracker" diff --git a/k8s/settings_local.py b/k8s/settings_local.py index e5f0a9af0f3..898cfb56304 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -50,30 +50,6 @@ def _multiline_to_list(s): else: raise RuntimeError("DATATRACKER_YOUTUBE_API_KEY must be set") -_GITHUB_BACKUP_API_KEY = os.environ.get("DATATRACKER_GITHUB_BACKUP_API_KEY", None) -if _GITHUB_BACKUP_API_KEY is not None: - GITHUB_BACKUP_API_KEY = _GITHUB_BACKUP_API_KEY -else: - raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set") - -_API_KEY_TYPE = os.environ.get("DATATRACKER_API_KEY_TYPE", None) -if _API_KEY_TYPE is not None: - API_KEY_TYPE = _API_KEY_TYPE -else: - raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set") - -_API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) -if _API_PUBLIC_KEY_PEM_B64 is not None: - API_PUBLIC_KEY_PEM = b64decode(_API_PUBLIC_KEY_PEM_B64) -else: - raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set") - -_API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) -if _API_PRIVATE_KEY_PEM_B64 is not None: - API_PRIVATE_KEY_PEM = b64decode(_API_PRIVATE_KEY_PEM_B64) -else: - raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") - _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get( "DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None ) @@ -388,6 +364,12 @@ def _multiline_to_list(s): }, } +_person_profile_cache_seconds = os.environ.get( + "DATATRACKER_PERSON_PROFILE_CACHE_SECONDS", None +) +if _person_profile_cache_seconds is not None: + PERSON_PROFILE_CACHE_SECONDS = int(_person_profile_cache_seconds) + _csrf_trusted_origins_str = os.environ.get("DATATRACKER_CSRF_TRUSTED_ORIGINS") if _csrf_trusted_origins_str is not None: CSRF_TRUSTED_ORIGINS = _multiline_to_list(_csrf_trusted_origins_str) diff --git a/playwright/package-lock.json b/playwright/package-lock.json index 3f354187450..143838b6c40 100644 --- a/playwright/package-lock.json +++ b/playwright/package-lock.json @@ -15,7 +15,7 @@ }, "devDependencies": { "@playwright/test": "1.62.1", - "npm-check-updates": "23.0.1" + "npm-check-updates": "23.0.2" } }, "node_modules/@faker-js/faker": { @@ -84,9 +84,9 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/npm-check-updates": { - "version": "23.0.1", - "resolved": "https://registry.npmjs.org/npm-check-updates/-/npm-check-updates-23.0.1.tgz", - "integrity": "sha512-e4hu3Rq4waj7SnhzvqAc5UG557mfP5e3JIFFQd7Fg3RiRkJl8yR90NFoZ1GSWUT3S/QacJndJb+7dLQBPeH+iA==", + "version": "23.0.2", + "resolved": "https://registry.npmjs.org/npm-check-updates/-/npm-check-updates-23.0.2.tgz", + "integrity": "sha512-t5tv+d4sP+WbSwcDAFBwDxSUJRomc+TJoURPu7q5B/19toUsR/7eshRxBdWdE7iYw80iE4O4D/7OvCvIcaG8ug==", "dev": true, "bin": { "ncu": "build/cli.js", @@ -180,9 +180,9 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "npm-check-updates": { - "version": "23.0.1", - "resolved": "https://registry.npmjs.org/npm-check-updates/-/npm-check-updates-23.0.1.tgz", - "integrity": "sha512-e4hu3Rq4waj7SnhzvqAc5UG557mfP5e3JIFFQd7Fg3RiRkJl8yR90NFoZ1GSWUT3S/QacJndJb+7dLQBPeH+iA==", + "version": "23.0.2", + "resolved": "https://registry.npmjs.org/npm-check-updates/-/npm-check-updates-23.0.2.tgz", + "integrity": "sha512-t5tv+d4sP+WbSwcDAFBwDxSUJRomc+TJoURPu7q5B/19toUsR/7eshRxBdWdE7iYw80iE4O4D/7OvCvIcaG8ug==", "dev": true }, "playwright": { diff --git a/playwright/package.json b/playwright/package.json index d33dca96930..3c9db331095 100644 --- a/playwright/package.json +++ b/playwright/package.json @@ -17,6 +17,6 @@ }, "devDependencies": { "@playwright/test": "1.62.1", - "npm-check-updates": "23.0.1" + "npm-check-updates": "23.0.2" } }